From bb039d8e4ce5b7f70c4f3e86d1327e210ef24dc3 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Wed, 25 Oct 2023 15:53:08 -0500 Subject: [PATCH 01/59] Dedupe did cache refreshes (#1773) * dedupe refreshes to did cache * handle expired cache entries as well * apply in pds as well * changeset --- .changeset/new-snails-hope.md | 5 +++ packages/bsky/src/did-cache.ts | 45 ++++++++++++++-------- packages/identity/src/did/base-resolver.ts | 29 +++++++++----- packages/identity/src/did/memory-cache.ts | 3 +- packages/identity/src/types.ts | 8 +++- packages/pds/src/did-cache.ts | 44 +++++++++++++-------- 6 files changed, 88 insertions(+), 46 deletions(-) create mode 100644 .changeset/new-snails-hope.md diff --git a/.changeset/new-snails-hope.md b/.changeset/new-snails-hope.md new file mode 100644 index 00000000000..2526d0836ed --- /dev/null +++ b/.changeset/new-snails-hope.md @@ -0,0 +1,5 @@ +--- +'@atproto/identity': minor +--- + +Pass stale did doc into refresh cache functions diff --git a/packages/bsky/src/did-cache.ts b/packages/bsky/src/did-cache.ts index b161ff73fe3..e08b09ca7e7 100644 --- a/packages/bsky/src/did-cache.ts +++ b/packages/bsky/src/did-cache.ts @@ -17,28 +17,42 @@ export class DidSqlCache implements DidCache { this.pQueue = new PQueue() } - async cacheDid(did: string, doc: DidDocument): Promise { - await this.db.db - .insertInto('did_cache') - .values({ did, doc, updatedAt: Date.now() }) - .onConflict((oc) => - oc.column('did').doUpdateSet({ - doc: excluded(this.db.db, 'doc'), - updatedAt: excluded(this.db.db, 'updatedAt'), - }), - ) - .executeTakeFirst() + async cacheDid( + did: string, + doc: DidDocument, + prevResult?: CacheResult, + ): Promise { + if (prevResult) { + await this.db.db + .updateTable('did_cache') + .set({ doc, updatedAt: Date.now() }) + .where('did', '=', did) + .where('updatedAt', '=', prevResult.updatedAt) + .execute() + } else { + await this.db.db + .insertInto('did_cache') + .values({ did, doc, updatedAt: Date.now() }) + .onConflict((oc) => + oc.column('did').doUpdateSet({ + doc: excluded(this.db.db, 'doc'), + updatedAt: excluded(this.db.db, 'updatedAt'), + }), + ) + .executeTakeFirst() + } } async refreshCache( did: string, getDoc: () => Promise, + prevResult?: CacheResult, ): Promise { this.pQueue?.add(async () => { try { const doc = await getDoc() if (doc) { - await this.cacheDid(did, doc) + await this.cacheDid(did, doc, prevResult) } else { await this.clearEntry(did) } @@ -55,20 +69,17 @@ export class DidSqlCache implements DidCache { .selectAll() .executeTakeFirst() if (!res) return null + const now = Date.now() const updatedAt = new Date(res.updatedAt).getTime() - const expired = now > updatedAt + this.maxTTL - if (expired) { - return null - } - const stale = now > updatedAt + this.staleTTL return { doc: res.doc, updatedAt, did, stale, + expired, } } diff --git a/packages/identity/src/did/base-resolver.ts b/packages/identity/src/did/base-resolver.ts index fb3d7bc57f8..765f354213c 100644 --- a/packages/identity/src/did/base-resolver.ts +++ b/packages/identity/src/did/base-resolver.ts @@ -1,6 +1,12 @@ import * as crypto from '@atproto/crypto' import { check } from '@atproto/common-web' -import { DidCache, AtprotoData, DidDocument, didDocument } from '../types' +import { + DidCache, + AtprotoData, + DidDocument, + didDocument, + CacheResult, +} from '../types' import * as atprotoData from './atproto-data' import { DidNotFoundError, PoorlyFormattedDidDocumentError } from '../errors' @@ -25,20 +31,25 @@ export abstract class BaseResolver { return this.validateDidDoc(did, got) } - async refreshCache(did: string): Promise { - await this.cache?.refreshCache(did, () => this.resolveNoCache(did)) + async refreshCache(did: string, prevResult?: CacheResult): Promise { + await this.cache?.refreshCache( + did, + () => this.resolveNoCache(did), + prevResult, + ) } async resolve( did: string, forceRefresh = false, ): Promise { + let fromCache: CacheResult | null = null if (this.cache && !forceRefresh) { - const fromCache = await this.cache.checkCache(did) - if (fromCache?.stale) { - await this.refreshCache(did) - } - if (fromCache) { + fromCache = await this.cache.checkCache(did) + if (fromCache && !fromCache.expired) { + if (fromCache?.stale) { + await this.refreshCache(did, fromCache) + } return fromCache.doc } } @@ -48,7 +59,7 @@ export abstract class BaseResolver { await this.cache?.clearEntry(did) return null } - await this.cache?.cacheDid(did, got) + await this.cache?.cacheDid(did, got, fromCache ?? undefined) return got } diff --git a/packages/identity/src/did/memory-cache.ts b/packages/identity/src/did/memory-cache.ts index c5ab8c4ec8d..42f01527529 100644 --- a/packages/identity/src/did/memory-cache.ts +++ b/packages/identity/src/did/memory-cache.ts @@ -35,13 +35,12 @@ export class MemoryCache implements DidCache { if (!val) return null const now = Date.now() const expired = now > val.updatedAt + this.maxTTL - if (expired) return null - const stale = now > val.updatedAt + this.staleTTL return { ...val, did, stale, + expired, } } diff --git a/packages/identity/src/types.ts b/packages/identity/src/types.ts index f1d983e6742..a3604f72f73 100644 --- a/packages/identity/src/types.ts +++ b/packages/identity/src/types.ts @@ -30,14 +30,20 @@ export type CacheResult = { doc: DidDocument updatedAt: number stale: boolean + expired: boolean } export interface DidCache { - cacheDid(did: string, doc: DidDocument): Promise + cacheDid( + did: string, + doc: DidDocument, + prevResult?: CacheResult, + ): Promise checkCache(did: string): Promise refreshCache( did: string, getDoc: () => Promise, + prevResult?: CacheResult, ): Promise clearEntry(did: string): Promise clear(): Promise diff --git a/packages/pds/src/did-cache.ts b/packages/pds/src/did-cache.ts index dce0252bb0f..ac719fe922c 100644 --- a/packages/pds/src/did-cache.ts +++ b/packages/pds/src/did-cache.ts @@ -15,28 +15,42 @@ export class DidSqlCache implements DidCache { this.pQueue = new PQueue() } - async cacheDid(did: string, doc: DidDocument): Promise { - await this.db.db - .insertInto('did_cache') - .values({ did, doc: JSON.stringify(doc), updatedAt: Date.now() }) - .onConflict((oc) => - oc.column('did').doUpdateSet({ - doc: excluded(this.db.db, 'doc'), - updatedAt: excluded(this.db.db, 'updatedAt'), - }), - ) - .executeTakeFirst() + async cacheDid( + did: string, + doc: DidDocument, + prevResult?: CacheResult, + ): Promise { + if (prevResult) { + await this.db.db + .updateTable('did_cache') + .set({ doc: JSON.stringify(doc), updatedAt: Date.now() }) + .where('did', '=', did) + .where('updatedAt', '=', prevResult.updatedAt) + .execute() + } else { + await this.db.db + .insertInto('did_cache') + .values({ did, doc: JSON.stringify(doc), updatedAt: Date.now() }) + .onConflict((oc) => + oc.column('did').doUpdateSet({ + doc: excluded(this.db.db, 'doc'), + updatedAt: excluded(this.db.db, 'updatedAt'), + }), + ) + .executeTakeFirst() + } } async refreshCache( did: string, getDoc: () => Promise, + prevResult?: CacheResult, ): Promise { this.pQueue?.add(async () => { try { const doc = await getDoc() if (doc) { - await this.cacheDid(did, doc) + await this.cacheDid(did, doc, prevResult) } else { await this.clearEntry(did) } @@ -55,18 +69,14 @@ export class DidSqlCache implements DidCache { if (!res) return null const now = Date.now() const updatedAt = new Date(res.updatedAt).getTime() - const expired = now > updatedAt + this.maxTTL - if (expired) { - return null - } - const stale = now > updatedAt + this.staleTTL return { doc: JSON.parse(res.doc) as DidDocument, updatedAt, did, stale, + expired, } } From 696df1e488ba916274cbb160d6a7b912a08072ab Mon Sep 17 00:00:00 2001 From: devin ivy Date: Thu, 26 Oct 2023 17:15:12 -0400 Subject: [PATCH 02/59] Lexicons to support bring-your-own-did w/ per-repo signing keys (#1739) * lexicons for per-repo signing keys * fix * rename lexicon * fix filename * rename getSigningKey to reserveSigningKey --- .../com/atproto/server/createAccount.json | 3 +- .../com/atproto/server/reserveSigningKey.json | 23 ++++++++++ packages/api/src/client/index.ts | 13 ++++++ packages/api/src/client/lexicons.ts | 27 ++++++++++++ .../types/com/atproto/server/createAccount.ts | 1 + .../com/atproto/server/reserveSigningKey.ts | 35 +++++++++++++++ packages/bsky/src/lexicon/index.ts | 12 +++++ packages/bsky/src/lexicon/lexicons.ts | 27 ++++++++++++ .../types/com/atproto/server/createAccount.ts | 1 + .../com/atproto/server/reserveSigningKey.ts | 44 +++++++++++++++++++ .../api/com/atproto/server/createAccount.ts | 3 ++ packages/pds/src/lexicon/index.ts | 12 +++++ packages/pds/src/lexicon/lexicons.ts | 27 ++++++++++++ .../types/com/atproto/server/createAccount.ts | 1 + .../com/atproto/server/reserveSigningKey.ts | 44 +++++++++++++++++++ 15 files changed, 272 insertions(+), 1 deletion(-) create mode 100644 lexicons/com/atproto/server/reserveSigningKey.json create mode 100644 packages/api/src/client/types/com/atproto/server/reserveSigningKey.ts create mode 100644 packages/bsky/src/lexicon/types/com/atproto/server/reserveSigningKey.ts create mode 100644 packages/pds/src/lexicon/types/com/atproto/server/reserveSigningKey.ts diff --git a/lexicons/com/atproto/server/createAccount.json b/lexicons/com/atproto/server/createAccount.json index 9fd09740fda..7e167a92e55 100644 --- a/lexicons/com/atproto/server/createAccount.json +++ b/lexicons/com/atproto/server/createAccount.json @@ -16,7 +16,8 @@ "did": { "type": "string", "format": "did" }, "inviteCode": { "type": "string" }, "password": { "type": "string" }, - "recoveryKey": { "type": "string" } + "recoveryKey": { "type": "string" }, + "plcOp": { "type": "bytes" } } } }, diff --git a/lexicons/com/atproto/server/reserveSigningKey.json b/lexicons/com/atproto/server/reserveSigningKey.json new file mode 100644 index 00000000000..27fb0597b0a --- /dev/null +++ b/lexicons/com/atproto/server/reserveSigningKey.json @@ -0,0 +1,23 @@ +{ + "lexicon": 1, + "id": "com.atproto.server.reserveSigningKey", + "defs": { + "main": { + "type": "procedure", + "description": "Reserve a repo signing key for account creation.", + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["signingKey"], + "properties": { + "signingKey": { + "type": "string", + "description": "Public signing key in the form of a did:key." + } + } + } + } + } + } +} diff --git a/packages/api/src/client/index.ts b/packages/api/src/client/index.ts index e5286aa2eb1..15720ad52f8 100644 --- a/packages/api/src/client/index.ts +++ b/packages/api/src/client/index.ts @@ -59,6 +59,7 @@ import * as ComAtprotoServerRequestAccountDelete from './types/com/atproto/serve import * as ComAtprotoServerRequestEmailConfirmation from './types/com/atproto/server/requestEmailConfirmation' import * as ComAtprotoServerRequestEmailUpdate from './types/com/atproto/server/requestEmailUpdate' import * as ComAtprotoServerRequestPasswordReset from './types/com/atproto/server/requestPasswordReset' +import * as ComAtprotoServerReserveSigningKey from './types/com/atproto/server/reserveSigningKey' import * as ComAtprotoServerResetPassword from './types/com/atproto/server/resetPassword' import * as ComAtprotoServerRevokeAppPassword from './types/com/atproto/server/revokeAppPassword' import * as ComAtprotoServerUpdateEmail from './types/com/atproto/server/updateEmail' @@ -192,6 +193,7 @@ export * as ComAtprotoServerRequestAccountDelete from './types/com/atproto/serve export * as ComAtprotoServerRequestEmailConfirmation from './types/com/atproto/server/requestEmailConfirmation' export * as ComAtprotoServerRequestEmailUpdate from './types/com/atproto/server/requestEmailUpdate' export * as ComAtprotoServerRequestPasswordReset from './types/com/atproto/server/requestPasswordReset' +export * as ComAtprotoServerReserveSigningKey from './types/com/atproto/server/reserveSigningKey' export * as ComAtprotoServerResetPassword from './types/com/atproto/server/resetPassword' export * as ComAtprotoServerRevokeAppPassword from './types/com/atproto/server/revokeAppPassword' export * as ComAtprotoServerUpdateEmail from './types/com/atproto/server/updateEmail' @@ -907,6 +909,17 @@ export class ServerNS { }) } + reserveSigningKey( + data?: ComAtprotoServerReserveSigningKey.InputSchema, + opts?: ComAtprotoServerReserveSigningKey.CallOptions, + ): Promise { + return this._service.xrpc + .call('com.atproto.server.reserveSigningKey', opts?.qp, data, opts) + .catch((e) => { + throw ComAtprotoServerReserveSigningKey.toKnownErr(e) + }) + } + resetPassword( data?: ComAtprotoServerResetPassword.InputSchema, opts?: ComAtprotoServerResetPassword.CallOptions, diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index ded6b1f86f6..39bc995d533 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -2315,6 +2315,9 @@ export const schemaDict = { recoveryKey: { type: 'string', }, + plcOp: { + type: 'bytes', + }, }, }, }, @@ -2955,6 +2958,29 @@ export const schemaDict = { }, }, }, + ComAtprotoServerReserveSigningKey: { + lexicon: 1, + id: 'com.atproto.server.reserveSigningKey', + defs: { + main: { + type: 'procedure', + description: 'Reserve a repo signing key for account creation.', + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['signingKey'], + properties: { + signingKey: { + type: 'string', + description: 'Public signing key in the form of a did:key.', + }, + }, + }, + }, + }, + }, + }, ComAtprotoServerResetPassword: { lexicon: 1, id: 'com.atproto.server.resetPassword', @@ -7382,6 +7408,7 @@ export const ids = { ComAtprotoServerRequestEmailUpdate: 'com.atproto.server.requestEmailUpdate', ComAtprotoServerRequestPasswordReset: 'com.atproto.server.requestPasswordReset', + ComAtprotoServerReserveSigningKey: 'com.atproto.server.reserveSigningKey', ComAtprotoServerResetPassword: 'com.atproto.server.resetPassword', ComAtprotoServerRevokeAppPassword: 'com.atproto.server.revokeAppPassword', ComAtprotoServerUpdateEmail: 'com.atproto.server.updateEmail', diff --git a/packages/api/src/client/types/com/atproto/server/createAccount.ts b/packages/api/src/client/types/com/atproto/server/createAccount.ts index 3eeaab250b4..54d1d9a1cf0 100644 --- a/packages/api/src/client/types/com/atproto/server/createAccount.ts +++ b/packages/api/src/client/types/com/atproto/server/createAccount.ts @@ -16,6 +16,7 @@ export interface InputSchema { inviteCode?: string password: string recoveryKey?: string + plcOp?: Uint8Array [k: string]: unknown } diff --git a/packages/api/src/client/types/com/atproto/server/reserveSigningKey.ts b/packages/api/src/client/types/com/atproto/server/reserveSigningKey.ts new file mode 100644 index 00000000000..e6f4f7a618a --- /dev/null +++ b/packages/api/src/client/types/com/atproto/server/reserveSigningKey.ts @@ -0,0 +1,35 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import { Headers, XRPCError } from '@atproto/xrpc' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { isObj, hasProp } from '../../../../util' +import { lexicons } from '../../../../lexicons' +import { CID } from 'multiformats/cid' + +export interface QueryParams {} + +export type InputSchema = undefined + +export interface OutputSchema { + /** Public signing key in the form of a did:key. */ + signingKey: string + [k: string]: unknown +} + +export interface CallOptions { + headers?: Headers + qp?: QueryParams +} + +export interface Response { + success: boolean + headers: Headers + data: OutputSchema +} + +export function toKnownErr(e: any) { + if (e instanceof XRPCError) { + } + return e +} diff --git a/packages/bsky/src/lexicon/index.ts b/packages/bsky/src/lexicon/index.ts index ac6ca933fcd..1fd8a1f127c 100644 --- a/packages/bsky/src/lexicon/index.ts +++ b/packages/bsky/src/lexicon/index.ts @@ -56,6 +56,7 @@ import * as ComAtprotoServerRequestAccountDelete from './types/com/atproto/serve import * as ComAtprotoServerRequestEmailConfirmation from './types/com/atproto/server/requestEmailConfirmation' import * as ComAtprotoServerRequestEmailUpdate from './types/com/atproto/server/requestEmailUpdate' import * as ComAtprotoServerRequestPasswordReset from './types/com/atproto/server/requestPasswordReset' +import * as ComAtprotoServerReserveSigningKey from './types/com/atproto/server/reserveSigningKey' import * as ComAtprotoServerResetPassword from './types/com/atproto/server/resetPassword' import * as ComAtprotoServerRevokeAppPassword from './types/com/atproto/server/revokeAppPassword' import * as ComAtprotoServerUpdateEmail from './types/com/atproto/server/updateEmail' @@ -748,6 +749,17 @@ export class ServerNS { return this._server.xrpc.method(nsid, cfg) } + reserveSigningKey( + cfg: ConfigOf< + AV, + ComAtprotoServerReserveSigningKey.Handler>, + ComAtprotoServerReserveSigningKey.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.server.reserveSigningKey' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } + resetPassword( cfg: ConfigOf< AV, diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index ded6b1f86f6..39bc995d533 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -2315,6 +2315,9 @@ export const schemaDict = { recoveryKey: { type: 'string', }, + plcOp: { + type: 'bytes', + }, }, }, }, @@ -2955,6 +2958,29 @@ export const schemaDict = { }, }, }, + ComAtprotoServerReserveSigningKey: { + lexicon: 1, + id: 'com.atproto.server.reserveSigningKey', + defs: { + main: { + type: 'procedure', + description: 'Reserve a repo signing key for account creation.', + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['signingKey'], + properties: { + signingKey: { + type: 'string', + description: 'Public signing key in the form of a did:key.', + }, + }, + }, + }, + }, + }, + }, ComAtprotoServerResetPassword: { lexicon: 1, id: 'com.atproto.server.resetPassword', @@ -7382,6 +7408,7 @@ export const ids = { ComAtprotoServerRequestEmailUpdate: 'com.atproto.server.requestEmailUpdate', ComAtprotoServerRequestPasswordReset: 'com.atproto.server.requestPasswordReset', + ComAtprotoServerReserveSigningKey: 'com.atproto.server.reserveSigningKey', ComAtprotoServerResetPassword: 'com.atproto.server.resetPassword', ComAtprotoServerRevokeAppPassword: 'com.atproto.server.revokeAppPassword', ComAtprotoServerUpdateEmail: 'com.atproto.server.updateEmail', diff --git a/packages/bsky/src/lexicon/types/com/atproto/server/createAccount.ts b/packages/bsky/src/lexicon/types/com/atproto/server/createAccount.ts index c67e7445bf9..d50d2b24025 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/server/createAccount.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/server/createAccount.ts @@ -17,6 +17,7 @@ export interface InputSchema { inviteCode?: string password: string recoveryKey?: string + plcOp?: Uint8Array [k: string]: unknown } diff --git a/packages/bsky/src/lexicon/types/com/atproto/server/reserveSigningKey.ts b/packages/bsky/src/lexicon/types/com/atproto/server/reserveSigningKey.ts new file mode 100644 index 00000000000..495b87dc03c --- /dev/null +++ b/packages/bsky/src/lexicon/types/com/atproto/server/reserveSigningKey.ts @@ -0,0 +1,44 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' + +export interface QueryParams {} + +export type InputSchema = undefined + +export interface OutputSchema { + /** Public signing key in the form of a did:key. */ + signingKey: string + [k: string]: unknown +} + +export type HandlerInput = undefined + +export interface HandlerSuccess { + encoding: 'application/json' + body: OutputSchema + headers?: { [key: string]: string } +} + +export interface HandlerError { + status: number + message?: string +} + +export type HandlerOutput = HandlerError | HandlerSuccess +export type HandlerReqCtx = { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +} +export type Handler = ( + ctx: HandlerReqCtx, +) => Promise | HandlerOutput diff --git a/packages/pds/src/api/com/atproto/server/createAccount.ts b/packages/pds/src/api/com/atproto/server/createAccount.ts index cbff7a908a8..c58e1547863 100644 --- a/packages/pds/src/api/com/atproto/server/createAccount.ts +++ b/packages/pds/src/api/com/atproto/server/createAccount.ts @@ -20,6 +20,9 @@ export default function (server: Server, ctx: AppContext) { }, handler: async ({ input, req }) => { const { email, password, inviteCode } = input.body + if (input.body.plcOp) { + throw new InvalidRequestError('Unsupported input: "plcOp"') + } if (ctx.cfg.invites.required && !inviteCode) { throw new InvalidRequestError( diff --git a/packages/pds/src/lexicon/index.ts b/packages/pds/src/lexicon/index.ts index ac6ca933fcd..1fd8a1f127c 100644 --- a/packages/pds/src/lexicon/index.ts +++ b/packages/pds/src/lexicon/index.ts @@ -56,6 +56,7 @@ import * as ComAtprotoServerRequestAccountDelete from './types/com/atproto/serve import * as ComAtprotoServerRequestEmailConfirmation from './types/com/atproto/server/requestEmailConfirmation' import * as ComAtprotoServerRequestEmailUpdate from './types/com/atproto/server/requestEmailUpdate' import * as ComAtprotoServerRequestPasswordReset from './types/com/atproto/server/requestPasswordReset' +import * as ComAtprotoServerReserveSigningKey from './types/com/atproto/server/reserveSigningKey' import * as ComAtprotoServerResetPassword from './types/com/atproto/server/resetPassword' import * as ComAtprotoServerRevokeAppPassword from './types/com/atproto/server/revokeAppPassword' import * as ComAtprotoServerUpdateEmail from './types/com/atproto/server/updateEmail' @@ -748,6 +749,17 @@ export class ServerNS { return this._server.xrpc.method(nsid, cfg) } + reserveSigningKey( + cfg: ConfigOf< + AV, + ComAtprotoServerReserveSigningKey.Handler>, + ComAtprotoServerReserveSigningKey.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.server.reserveSigningKey' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } + resetPassword( cfg: ConfigOf< AV, diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index ded6b1f86f6..39bc995d533 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -2315,6 +2315,9 @@ export const schemaDict = { recoveryKey: { type: 'string', }, + plcOp: { + type: 'bytes', + }, }, }, }, @@ -2955,6 +2958,29 @@ export const schemaDict = { }, }, }, + ComAtprotoServerReserveSigningKey: { + lexicon: 1, + id: 'com.atproto.server.reserveSigningKey', + defs: { + main: { + type: 'procedure', + description: 'Reserve a repo signing key for account creation.', + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['signingKey'], + properties: { + signingKey: { + type: 'string', + description: 'Public signing key in the form of a did:key.', + }, + }, + }, + }, + }, + }, + }, ComAtprotoServerResetPassword: { lexicon: 1, id: 'com.atproto.server.resetPassword', @@ -7382,6 +7408,7 @@ export const ids = { ComAtprotoServerRequestEmailUpdate: 'com.atproto.server.requestEmailUpdate', ComAtprotoServerRequestPasswordReset: 'com.atproto.server.requestPasswordReset', + ComAtprotoServerReserveSigningKey: 'com.atproto.server.reserveSigningKey', ComAtprotoServerResetPassword: 'com.atproto.server.resetPassword', ComAtprotoServerRevokeAppPassword: 'com.atproto.server.revokeAppPassword', ComAtprotoServerUpdateEmail: 'com.atproto.server.updateEmail', diff --git a/packages/pds/src/lexicon/types/com/atproto/server/createAccount.ts b/packages/pds/src/lexicon/types/com/atproto/server/createAccount.ts index c67e7445bf9..d50d2b24025 100644 --- a/packages/pds/src/lexicon/types/com/atproto/server/createAccount.ts +++ b/packages/pds/src/lexicon/types/com/atproto/server/createAccount.ts @@ -17,6 +17,7 @@ export interface InputSchema { inviteCode?: string password: string recoveryKey?: string + plcOp?: Uint8Array [k: string]: unknown } diff --git a/packages/pds/src/lexicon/types/com/atproto/server/reserveSigningKey.ts b/packages/pds/src/lexicon/types/com/atproto/server/reserveSigningKey.ts new file mode 100644 index 00000000000..495b87dc03c --- /dev/null +++ b/packages/pds/src/lexicon/types/com/atproto/server/reserveSigningKey.ts @@ -0,0 +1,44 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' + +export interface QueryParams {} + +export type InputSchema = undefined + +export interface OutputSchema { + /** Public signing key in the form of a did:key. */ + signingKey: string + [k: string]: unknown +} + +export type HandlerInput = undefined + +export interface HandlerSuccess { + encoding: 'application/json' + body: OutputSchema + headers?: { [key: string]: string } +} + +export interface HandlerError { + status: number + message?: string +} + +export type HandlerOutput = HandlerError | HandlerSuccess +export type HandlerReqCtx = { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +} +export type Handler = ( + ctx: HandlerReqCtx, +) => Promise | HandlerOutput From 46b108cb8672706a71c2a38bb5489c98b456fa9b Mon Sep 17 00:00:00 2001 From: devin ivy Date: Thu, 26 Oct 2023 18:29:51 -0400 Subject: [PATCH 03/59] Facilitate authing w/ PDS based on DID doc (#1727) * lexicon for did doc w/ auth credentials * include did doc w/ session when configured. configure on dev-env. * Add dynamic PDS URL adoption to the client * remove usage of did doc field from getsession in client * dry-up did doc type and validation * remove explicit dep on zod by identity package * move more did doc parsing to common-web * go back to strings * rollback breaking changes to identity package * add changeset --------- Co-authored-by: Paul Frazee Co-authored-by: dholms --- .changeset/purple-shirts-punch.md | 10 ++ .../com/atproto/server/createAccount.json | 3 +- .../com/atproto/server/createSession.json | 1 + .../com/atproto/server/refreshSession.json | 3 +- packages/api/package.json | 3 +- packages/api/src/agent.ts | 29 +++- packages/api/src/client/lexicons.ts | 9 ++ .../types/com/atproto/server/createAccount.ts | 1 + .../types/com/atproto/server/createSession.ts | 1 + .../com/atproto/server/refreshSession.ts | 1 + packages/api/tests/agent.test.ts | 3 + packages/bsky/src/lexicon/lexicons.ts | 9 ++ .../types/com/atproto/server/createAccount.ts | 1 + .../types/com/atproto/server/createSession.ts | 1 + .../com/atproto/server/refreshSession.ts | 1 + packages/common-web/src/did-doc.ts | 133 ++++++++++++++++++ packages/common-web/src/index.ts | 1 + packages/dev-env/src/bin.ts | 1 + packages/identity/package.json | 3 +- packages/identity/src/did/atproto-data.ts | 114 +++------------ packages/identity/src/types.ts | 27 +--- .../api/com/atproto/server/createAccount.ts | 10 +- .../api/com/atproto/server/createSession.ts | 12 +- .../api/com/atproto/server/refreshSession.ts | 21 +-- .../pds/src/api/com/atproto/server/util.ts | 18 +++ packages/pds/src/config/config.ts | 2 + packages/pds/src/config/env.ts | 2 + packages/pds/src/lexicon/lexicons.ts | 9 ++ .../types/com/atproto/server/createAccount.ts | 1 + .../types/com/atproto/server/createSession.ts | 1 + .../com/atproto/server/refreshSession.ts | 1 + pnpm-lock.yaml | 6 +- 32 files changed, 298 insertions(+), 140 deletions(-) create mode 100644 .changeset/purple-shirts-punch.md create mode 100644 packages/common-web/src/did-doc.ts diff --git a/.changeset/purple-shirts-punch.md b/.changeset/purple-shirts-punch.md new file mode 100644 index 00000000000..b0a6c734454 --- /dev/null +++ b/.changeset/purple-shirts-punch.md @@ -0,0 +1,10 @@ +--- +'@atproto/common-web': patch +'@atproto/identity': patch +'@atproto/dev-env': patch +'@atproto/bsky': patch +'@atproto/api': patch +'@atproto/pds': patch +--- + +Allow pds to serve did doc with credentials, API client to respect PDS listed in the did doc. diff --git a/lexicons/com/atproto/server/createAccount.json b/lexicons/com/atproto/server/createAccount.json index 7e167a92e55..4db1f31e040 100644 --- a/lexicons/com/atproto/server/createAccount.json +++ b/lexicons/com/atproto/server/createAccount.json @@ -30,7 +30,8 @@ "accessJwt": { "type": "string" }, "refreshJwt": { "type": "string" }, "handle": { "type": "string", "format": "handle" }, - "did": { "type": "string", "format": "did" } + "did": { "type": "string", "format": "did" }, + "didDoc": { "type": "unknown" } } } }, diff --git a/lexicons/com/atproto/server/createSession.json b/lexicons/com/atproto/server/createSession.json index 7d877cec91c..cef01b45b35 100644 --- a/lexicons/com/atproto/server/createSession.json +++ b/lexicons/com/atproto/server/createSession.json @@ -29,6 +29,7 @@ "refreshJwt": { "type": "string" }, "handle": { "type": "string", "format": "handle" }, "did": { "type": "string", "format": "did" }, + "didDoc": { "type": "unknown" }, "email": { "type": "string" }, "emailConfirmed": { "type": "boolean" } } diff --git a/lexicons/com/atproto/server/refreshSession.json b/lexicons/com/atproto/server/refreshSession.json index ab895a34c94..3f4d7fdf272 100644 --- a/lexicons/com/atproto/server/refreshSession.json +++ b/lexicons/com/atproto/server/refreshSession.json @@ -14,7 +14,8 @@ "accessJwt": { "type": "string" }, "refreshJwt": { "type": "string" }, "handle": { "type": "string", "format": "handle" }, - "did": { "type": "string", "format": "did" } + "did": { "type": "string", "format": "did" }, + "didDoc": { "type": "unknown" } } } }, diff --git a/packages/api/package.json b/packages/api/package.json index a8af7b6b3d7..e4a1d662ea4 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -36,7 +36,8 @@ "@atproto/xrpc": "workspace:^", "multiformats": "^9.9.0", "tlds": "^1.234.0", - "typed-emitter": "^2.1.0" + "typed-emitter": "^2.1.0", + "zod": "^3.21.4" }, "devDependencies": { "@atproto/lex-cli": "workspace:^", diff --git a/packages/api/src/agent.ts b/packages/api/src/agent.ts index 2cd4c44e7a0..ce34865c189 100644 --- a/packages/api/src/agent.ts +++ b/packages/api/src/agent.ts @@ -1,5 +1,6 @@ import { ErrorResponseBody, errorResponseBody } from '@atproto/xrpc' import { defaultFetchHandler } from '@atproto/xrpc' +import { isValidDidDoc, getPdsEndpoint } from '@atproto/common-web' import { AtpBaseClient, AtpServiceClient, @@ -30,6 +31,11 @@ export class AtpAgent { api: AtpServiceClient session?: AtpSessionData + /** + * The PDS URL, driven by the did doc. May be undefined. + */ + pdsUrl: URL | undefined + private _baseClient: AtpBaseClient private _persistSession?: AtpPersistSessionHandler private _refreshSessionPromise: Promise | undefined @@ -97,6 +103,7 @@ export class AtpAgent { email: opts.email, emailConfirmed: false, } + this._updateApiEndpoint(res.data.didDoc) return res } catch (e) { this.session = undefined @@ -129,6 +136,7 @@ export class AtpAgent { email: res.data.email, emailConfirmed: res.data.emailConfirmed, } + this._updateApiEndpoint(res.data.didDoc) return res } catch (e) { this.session = undefined @@ -253,7 +261,7 @@ export class AtpAgent { } // send the refresh request - const url = new URL(this.service.origin) + const url = new URL((this.pdsUrl || this.service).origin) url.pathname = `/xrpc/${REFRESH_SESSION}` const res = await AtpAgent.fetch( url.toString(), @@ -277,6 +285,7 @@ export class AtpAgent { handle: res.body.handle, did: res.body.did, } + this._updateApiEndpoint(res.body.didDoc) this._persistSession?.('update', this.session) } // else: other failures should be ignored - the issue will @@ -311,6 +320,24 @@ export class AtpAgent { */ createModerationReport: typeof this.api.com.atproto.moderation.createReport = (data, opts) => this.api.com.atproto.moderation.createReport(data, opts) + + /** + * Helper to update the pds endpoint dynamically. + * + * The session methods (create, resume, refresh) may respond with the user's + * did document which contains the user's canonical PDS endpoint. That endpoint + * may differ from the endpoint used to contact the server. We capture that + * PDS endpoint and update the client to use that given endpoint for future + * requests. (This helps ensure smooth migrations between PDSes, especially + * when the PDSes are operated by a single org.) + */ + private _updateApiEndpoint(didDoc: unknown) { + if (isValidDidDoc(didDoc)) { + const endpoint = getPdsEndpoint(didDoc) + this.pdsUrl = endpoint ? new URL(endpoint) : undefined + } + this.api.xrpc.uri = this.pdsUrl || this.service + } } function isErrorObject(v: unknown): v is ErrorResponseBody { diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index 39bc995d533..6c163eedca4 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -2341,6 +2341,9 @@ export const schemaDict = { type: 'string', format: 'did', }, + didDoc: { + type: 'unknown', + }, }, }, }, @@ -2566,6 +2569,9 @@ export const schemaDict = { type: 'string', format: 'did', }, + didDoc: { + type: 'unknown', + }, email: { type: 'string', }, @@ -2882,6 +2888,9 @@ export const schemaDict = { type: 'string', format: 'did', }, + didDoc: { + type: 'unknown', + }, }, }, }, diff --git a/packages/api/src/client/types/com/atproto/server/createAccount.ts b/packages/api/src/client/types/com/atproto/server/createAccount.ts index 54d1d9a1cf0..4281128cae0 100644 --- a/packages/api/src/client/types/com/atproto/server/createAccount.ts +++ b/packages/api/src/client/types/com/atproto/server/createAccount.ts @@ -25,6 +25,7 @@ export interface OutputSchema { refreshJwt: string handle: string did: string + didDoc?: {} [k: string]: unknown } diff --git a/packages/api/src/client/types/com/atproto/server/createSession.ts b/packages/api/src/client/types/com/atproto/server/createSession.ts index 08d2bcd6225..a06a7a86c6c 100644 --- a/packages/api/src/client/types/com/atproto/server/createSession.ts +++ b/packages/api/src/client/types/com/atproto/server/createSession.ts @@ -21,6 +21,7 @@ export interface OutputSchema { refreshJwt: string handle: string did: string + didDoc?: {} email?: string emailConfirmed?: boolean [k: string]: unknown diff --git a/packages/api/src/client/types/com/atproto/server/refreshSession.ts b/packages/api/src/client/types/com/atproto/server/refreshSession.ts index 5b531b19e9d..5519e352920 100644 --- a/packages/api/src/client/types/com/atproto/server/refreshSession.ts +++ b/packages/api/src/client/types/com/atproto/server/refreshSession.ts @@ -16,6 +16,7 @@ export interface OutputSchema { refreshJwt: string handle: string did: string + didDoc?: {} [k: string]: unknown } diff --git a/packages/api/tests/agent.test.ts b/packages/api/tests/agent.test.ts index 933326c43f2..7f85f3079af 100644 --- a/packages/api/tests/agent.test.ts +++ b/packages/api/tests/agent.test.ts @@ -13,6 +13,9 @@ describe('agent', () => { beforeAll(async () => { network = await TestNetworkNoAppView.create({ dbPostgresSchema: 'api_agent', + pds: { + enableDidDocWithSession: true, + }, }) }) diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index 39bc995d533..6c163eedca4 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -2341,6 +2341,9 @@ export const schemaDict = { type: 'string', format: 'did', }, + didDoc: { + type: 'unknown', + }, }, }, }, @@ -2566,6 +2569,9 @@ export const schemaDict = { type: 'string', format: 'did', }, + didDoc: { + type: 'unknown', + }, email: { type: 'string', }, @@ -2882,6 +2888,9 @@ export const schemaDict = { type: 'string', format: 'did', }, + didDoc: { + type: 'unknown', + }, }, }, }, diff --git a/packages/bsky/src/lexicon/types/com/atproto/server/createAccount.ts b/packages/bsky/src/lexicon/types/com/atproto/server/createAccount.ts index d50d2b24025..bd138919101 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/server/createAccount.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/server/createAccount.ts @@ -26,6 +26,7 @@ export interface OutputSchema { refreshJwt: string handle: string did: string + didDoc?: {} [k: string]: unknown } diff --git a/packages/bsky/src/lexicon/types/com/atproto/server/createSession.ts b/packages/bsky/src/lexicon/types/com/atproto/server/createSession.ts index 037900346a1..2cd448703a6 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/server/createSession.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/server/createSession.ts @@ -22,6 +22,7 @@ export interface OutputSchema { refreshJwt: string handle: string did: string + didDoc?: {} email?: string emailConfirmed?: boolean [k: string]: unknown diff --git a/packages/bsky/src/lexicon/types/com/atproto/server/refreshSession.ts b/packages/bsky/src/lexicon/types/com/atproto/server/refreshSession.ts index e47bf09fbc2..35874f78a69 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/server/refreshSession.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/server/refreshSession.ts @@ -17,6 +17,7 @@ export interface OutputSchema { refreshJwt: string handle: string did: string + didDoc?: {} [k: string]: unknown } diff --git a/packages/common-web/src/did-doc.ts b/packages/common-web/src/did-doc.ts new file mode 100644 index 00000000000..c2a05b796d3 --- /dev/null +++ b/packages/common-web/src/did-doc.ts @@ -0,0 +1,133 @@ +import { z } from 'zod' + +// Parsing atproto data +// -------- + +export const isValidDidDoc = (doc: unknown): doc is DidDocument => { + return didDocument.safeParse(doc).success +} + +export const getDid = (doc: DidDocument): string => { + const id = doc.id + if (typeof id !== 'string') { + throw new Error('No `id` on document') + } + return id +} + +export const getHandle = (doc: DidDocument): string | undefined => { + const aka = doc.alsoKnownAs + if (!aka) return undefined + const found = aka.find((name) => name.startsWith('at://')) + if (!found) return undefined + // strip off at:// prefix + return found.slice(5) +} + +// @NOTE we parse to type/publicKeyMultibase to avoid the dependency on @atproto/crypto +export const getSigningKey = ( + doc: DidDocument, +): { type: string; publicKeyMultibase: string } | undefined => { + const did = getDid(doc) + let keys = doc.verificationMethod + if (!keys) return undefined + if (typeof keys !== 'object') return undefined + if (!Array.isArray(keys)) { + keys = [keys] + } + const found = keys.find( + (key) => key.id === '#atproto' || key.id === `${did}#atproto`, + ) + if (!found?.publicKeyMultibase) return undefined + return { + type: found.type, + publicKeyMultibase: found.publicKeyMultibase, + } +} + +export const getPdsEndpoint = (doc: DidDocument): string | undefined => { + return getServiceEndpoint(doc, { + id: '#atproto_pds', + type: 'AtprotoPersonalDataServer', + }) +} + +export const getFeedGenEndpoint = (doc: DidDocument): string | undefined => { + return getServiceEndpoint(doc, { + id: '#bsky_fg', + type: 'BskyFeedGenerator', + }) +} + +export const getNotifEndpoint = (doc: DidDocument): string | undefined => { + return getServiceEndpoint(doc, { + id: '#bsky_notif', + type: 'BskyNotificationService', + }) +} + +export const getServiceEndpoint = ( + doc: DidDocument, + opts: { id: string; type: string }, +) => { + const did = getDid(doc) + let services = doc.service + if (!services) return undefined + if (typeof services !== 'object') return undefined + if (!Array.isArray(services)) { + services = [services] + } + const found = services.find( + (service) => service.id === opts.id || service.id === `${did}${opts.id}`, + ) + if (!found) return undefined + if (found.type !== opts.type) { + return undefined + } + if (typeof found.serviceEndpoint !== 'string') { + return undefined + } + return validateUrl(found.serviceEndpoint) +} + +// Check protocol and hostname to prevent potential SSRF +const validateUrl = (urlStr: string): string | undefined => { + let url + try { + url = new URL(urlStr) + } catch { + return undefined + } + if (!['http:', 'https:'].includes(url.protocol)) { + return undefined + } else if (!url.hostname) { + return undefined + } else { + return urlStr + } +} + +// Types +// -------- + +const verificationMethod = z.object({ + id: z.string(), + type: z.string(), + controller: z.string(), + publicKeyMultibase: z.string().optional(), +}) + +const service = z.object({ + id: z.string(), + type: z.string(), + serviceEndpoint: z.union([z.string(), z.record(z.unknown())]), +}) + +export const didDocument = z.object({ + id: z.string(), + alsoKnownAs: z.array(z.string()).optional(), + verificationMethod: z.array(verificationMethod).optional(), + service: z.array(service).optional(), +}) + +export type DidDocument = z.infer diff --git a/packages/common-web/src/index.ts b/packages/common-web/src/index.ts index e125677496f..8352123536a 100644 --- a/packages/common-web/src/index.ts +++ b/packages/common-web/src/index.ts @@ -9,3 +9,4 @@ export * from './ipld' export * from './types' export * from './times' export * from './strings' +export * from './did-doc' diff --git a/packages/dev-env/src/bin.ts b/packages/dev-env/src/bin.ts index c03f8a76900..12228579a48 100644 --- a/packages/dev-env/src/bin.ts +++ b/packages/dev-env/src/bin.ts @@ -19,6 +19,7 @@ const run = async () => { port: 2583, hostname: 'localhost', dbPostgresSchema: 'pds', + enableDidDocWithSession: true, }, bsky: { dbPostgresSchema: 'bsky', diff --git a/packages/identity/package.json b/packages/identity/package.json index 59de4cc414c..96e654c503b 100644 --- a/packages/identity/package.json +++ b/packages/identity/package.json @@ -29,8 +29,7 @@ "dependencies": { "@atproto/common-web": "workspace:^", "@atproto/crypto": "workspace:^", - "axios": "^0.27.2", - "zod": "^3.21.4" + "axios": "^0.27.2" }, "devDependencies": { "@did-plc/lib": "^0.0.1", diff --git a/packages/identity/src/did/atproto-data.ts b/packages/identity/src/did/atproto-data.ts index 3e7ee5829eb..6881bda48dc 100644 --- a/packages/identity/src/did/atproto-data.ts +++ b/packages/identity/src/did/atproto-data.ts @@ -1,73 +1,39 @@ import * as crypto from '@atproto/crypto' import { DidDocument, AtprotoData } from '../types' +import { + getDid, + getHandle, + getPdsEndpoint, + getFeedGenEndpoint, + getNotifEndpoint, + getSigningKey, +} from '@atproto/common-web' -export const getDid = (doc: DidDocument): string => { - const id = doc.id - if (typeof id !== 'string') { - throw new Error('No `id` on document') - } - return id +export { + getDid, + getHandle, + getPdsEndpoint as getPds, + getFeedGenEndpoint as getFeedGen, + getNotifEndpoint as getNotif, } export const getKey = (doc: DidDocument): string | undefined => { - const did = getDid(doc) - let keys = doc.verificationMethod - if (!keys) return undefined - if (typeof keys !== 'object') return undefined - if (!Array.isArray(keys)) { - keys = [keys] - } - const found = keys.find( - (key) => key.id === '#atproto' || key.id === `${did}#atproto`, - ) - if (!found) return undefined + const key = getSigningKey(doc) + if (!key) return undefined - // @TODO support jwk - // should we be surfacing errors here or returning undefined? - if (!found.publicKeyMultibase) return undefined - const keyBytes = crypto.multibaseToBytes(found.publicKeyMultibase) + const keyBytes = crypto.multibaseToBytes(key.publicKeyMultibase) let didKey: string | undefined = undefined - if (found.type === 'EcdsaSecp256r1VerificationKey2019') { + if (key.type === 'EcdsaSecp256r1VerificationKey2019') { didKey = crypto.formatDidKey(crypto.P256_JWT_ALG, keyBytes) - } else if (found.type === 'EcdsaSecp256k1VerificationKey2019') { + } else if (key.type === 'EcdsaSecp256k1VerificationKey2019') { didKey = crypto.formatDidKey(crypto.SECP256K1_JWT_ALG, keyBytes) - } else if (found.type === 'Multikey') { - const parsed = crypto.parseMultikey(found.publicKeyMultibase) + } else if (key.type === 'Multikey') { + const parsed = crypto.parseMultikey(key.publicKeyMultibase) didKey = crypto.formatDidKey(parsed.jwtAlg, parsed.keyBytes) } return didKey } -export const getHandle = (doc: DidDocument): string | undefined => { - const aka = doc.alsoKnownAs - if (!aka) return undefined - const found = aka.find((name) => name.startsWith('at://')) - if (!found) return undefined - // strip off at:// prefix - return found.slice(5) -} - -export const getPds = (doc: DidDocument): string | undefined => { - return getServiceEndpoint(doc, { - id: '#atproto_pds', - type: 'AtprotoPersonalDataServer', - }) -} - -export const getFeedGen = (doc: DidDocument): string | undefined => { - return getServiceEndpoint(doc, { - id: '#bsky_fg', - type: 'BskyFeedGenerator', - }) -} - -export const getNotif = (doc: DidDocument): string | undefined => { - return getServiceEndpoint(doc, { - id: '#bsky_notif', - type: 'BskyNotificationService', - }) -} - export const parseToAtprotoDocument = ( doc: DidDocument, ): Partial => { @@ -76,7 +42,7 @@ export const parseToAtprotoDocument = ( did, signingKey: getKey(doc), handle: getHandle(doc), - pds: getPds(doc), + pds: getPdsEndpoint(doc), } } @@ -96,39 +62,3 @@ export const ensureAtpDocument = (doc: DidDocument): AtprotoData => { } return { did, signingKey, handle, pds } } - -// Check protocol and hostname to prevent potential SSRF -const validateUrl = (url: string) => { - const { hostname, protocol } = new URL(url) - if (!['http:', 'https:'].includes(protocol)) { - throw new Error('Invalid pds protocol') - } - if (!hostname) { - throw new Error('Invalid pds hostname') - } -} - -const getServiceEndpoint = ( - doc: DidDocument, - opts: { id: string; type: string }, -) => { - const did = getDid(doc) - let services = doc.service - if (!services) return undefined - if (typeof services !== 'object') return undefined - if (!Array.isArray(services)) { - services = [services] - } - const found = services.find( - (service) => service.id === opts.id || service.id === `${did}${opts.id}`, - ) - if (!found) return undefined - if (found.type !== opts.type) { - return undefined - } - if (typeof found.serviceEndpoint !== 'string') { - return undefined - } - validateUrl(found.serviceEndpoint) - return found.serviceEndpoint -} diff --git a/packages/identity/src/types.ts b/packages/identity/src/types.ts index a3604f72f73..cd7996ee3cc 100644 --- a/packages/identity/src/types.ts +++ b/packages/identity/src/types.ts @@ -1,4 +1,7 @@ -import * as z from 'zod' +import { DidDocument } from '@atproto/common-web' + +export { didDocument } from '@atproto/common-web' +export type { DidDocument } from '@atproto/common-web' export type IdentityResolverOpts = { timeout?: number @@ -48,25 +51,3 @@ export interface DidCache { clearEntry(did: string): Promise clear(): Promise } - -export const verificationMethod = z.object({ - id: z.string(), - type: z.string(), - controller: z.string(), - publicKeyMultibase: z.string().optional(), -}) - -export const service = z.object({ - id: z.string(), - type: z.string(), - serviceEndpoint: z.union([z.string(), z.record(z.unknown())]), -}) - -export const didDocument = z.object({ - id: z.string(), - alsoKnownAs: z.array(z.string()).optional(), - verificationMethod: z.array(verificationMethod).optional(), - service: z.array(service).optional(), -}) - -export type DidDocument = z.infer diff --git a/packages/pds/src/api/com/atproto/server/createAccount.ts b/packages/pds/src/api/com/atproto/server/createAccount.ts index c58e1547863..36bdc7b6b86 100644 --- a/packages/pds/src/api/com/atproto/server/createAccount.ts +++ b/packages/pds/src/api/com/atproto/server/createAccount.ts @@ -1,7 +1,9 @@ +import { MINUTE } from '@atproto/common' +import { AtprotoData } from '@atproto/identity' import { InvalidRequestError } from '@atproto/xrpc-server' +import * as plc from '@did-plc/lib' import disposable from 'disposable-email' import { normalizeAndValidateHandle } from '../../../../handle' -import * as plc from '@did-plc/lib' import * as scrypt from '../../../../db/scrypt' import { Server } from '../../../../lexicon' import { InputSchema as CreateAccountInput } from '../../../../lexicon/types/com/atproto/server/createAccount' @@ -9,8 +11,7 @@ import { countAll } from '../../../../db/util' import { UserAlreadyExistsError } from '../../../../services/account' import AppContext from '../../../../context' import Database from '../../../../db' -import { AtprotoData } from '@atproto/identity' -import { MINUTE } from '@atproto/common' +import { didDocForSession } from './util' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.createAccount({ @@ -121,11 +122,14 @@ export default function (server: Server, ctx: AppContext) { } }) + const didDoc = await didDocForSession(ctx, result.did, true) + return { encoding: 'application/json', body: { handle, did: result.did, + didDoc, accessJwt: result.accessJwt, refreshJwt: result.refreshJwt, }, diff --git a/packages/pds/src/api/com/atproto/server/createSession.ts b/packages/pds/src/api/com/atproto/server/createSession.ts index 93e64d7bbcd..64872d5aae1 100644 --- a/packages/pds/src/api/com/atproto/server/createSession.ts +++ b/packages/pds/src/api/com/atproto/server/createSession.ts @@ -1,8 +1,9 @@ +import { DAY, MINUTE } from '@atproto/common' import { AuthRequiredError } from '@atproto/xrpc-server' import AppContext from '../../../../context' import { softDeleted } from '../../../../db/util' import { Server } from '../../../../lexicon' -import { DAY, MINUTE } from '@atproto/common' +import { didDocForSession } from './util' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.createSession({ @@ -54,15 +55,16 @@ export default function (server: Server, ctx: AppContext) { ) } - const { access, refresh } = await authService.createSession( - user.did, - appPasswordName, - ) + const [{ access, refresh }, didDoc] = await Promise.all([ + authService.createSession(user.did, appPasswordName), + didDocForSession(ctx, user.did), + ]) return { encoding: 'application/json', body: { did: user.did, + didDoc, handle: user.handle, email: user.email, emailConfirmed: !!user.emailConfirmedAt, diff --git a/packages/pds/src/api/com/atproto/server/refreshSession.ts b/packages/pds/src/api/com/atproto/server/refreshSession.ts index 8db33e289de..0ab39e4d8cc 100644 --- a/packages/pds/src/api/com/atproto/server/refreshSession.ts +++ b/packages/pds/src/api/com/atproto/server/refreshSession.ts @@ -2,6 +2,7 @@ import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import AppContext from '../../../../context' import { softDeleted } from '../../../../db/util' import { Server } from '../../../../lexicon' +import { didDocForSession } from './util' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.refreshSession({ @@ -21,12 +22,15 @@ export default function (server: Server, ctx: AppContext) { ) } - const res = await ctx.db.transaction((dbTxn) => { - return ctx.services - .auth(dbTxn) - .rotateRefreshToken(auth.credentials.tokenId) - }) - if (res === null) { + const [didDoc, rotated] = await Promise.all([ + didDocForSession(ctx, user.did), + ctx.db.transaction((dbTxn) => { + return ctx.services + .auth(dbTxn) + .rotateRefreshToken(auth.credentials.tokenId) + }), + ]) + if (rotated === null) { throw new InvalidRequestError('Token has been revoked', 'ExpiredToken') } @@ -34,9 +38,10 @@ export default function (server: Server, ctx: AppContext) { encoding: 'application/json', body: { did: user.did, + didDoc, handle: user.handle, - accessJwt: res.access.jwt, - refreshJwt: res.refresh.jwt, + accessJwt: rotated.access.jwt, + refreshJwt: rotated.refresh.jwt, }, } }, diff --git a/packages/pds/src/api/com/atproto/server/util.ts b/packages/pds/src/api/com/atproto/server/util.ts index 71bd1ae219c..fc3bfae8e05 100644 --- a/packages/pds/src/api/com/atproto/server/util.ts +++ b/packages/pds/src/api/com/atproto/server/util.ts @@ -1,5 +1,8 @@ import * as crypto from '@atproto/crypto' +import { DidDocument } from '@atproto/identity' import { ServerConfig } from '../../../../config' +import AppContext from '../../../../context' +import { dbLogger } from '../../../../logger' // generate an invite code preceded by the hostname // with '.'s replaced by '-'s so it is not mistakable for a link @@ -22,3 +25,18 @@ export const getRandomToken = () => { const token = crypto.randomStr(8, 'base32').slice(0, 10) return token.slice(0, 5) + '-' + token.slice(5, 10) } + +// @TODO once supporting multiple pdses, validate pds in did doc based on allow-list. +export const didDocForSession = async ( + ctx: AppContext, + did: string, + forceRefresh?: boolean, +): Promise => { + if (!ctx.cfg.identity.enableDidDocWithSession) return + try { + const didDoc = await ctx.idResolver.did.resolve(did, forceRefresh) + return didDoc ?? undefined + } catch (err) { + dbLogger.warn({ err, did }, 'failed to resolve did doc') + } +} diff --git a/packages/pds/src/config/config.ts b/packages/pds/src/config/config.ts index 68d043e6431..58040abd781 100644 --- a/packages/pds/src/config/config.ts +++ b/packages/pds/src/config/config.ts @@ -91,6 +91,7 @@ export const envToCfg = (env: ServerEnvironment): ServerConfig => { recoveryDidKey: env.recoveryDidKey ?? null, serviceHandleDomains, handleBackupNameservers: env.handleBackupNameservers, + enableDidDocWithSession: !!env.enableDidDocWithSession, } // default to being required if left undefined @@ -253,6 +254,7 @@ export type IdentityConfig = { recoveryDidKey: string | null serviceHandleDomains: string[] handleBackupNameservers?: string[] + enableDidDocWithSession: boolean } export type InvitesConfig = diff --git a/packages/pds/src/config/env.ts b/packages/pds/src/config/env.ts index 170e26d5976..2c13124b4c9 100644 --- a/packages/pds/src/config/env.ts +++ b/packages/pds/src/config/env.ts @@ -36,6 +36,7 @@ export const readEnv = (): ServerEnvironment => { recoveryDidKey: envStr('PDS_RECOVERY_DID_KEY'), serviceHandleDomains: envList('PDS_SERVICE_HANDLE_DOMAINS'), handleBackupNameservers: envList('PDS_HANDLE_BACKUP_NAMESERVERS'), + enableDidDocWithSession: envBool('PDS_ENABLE_DID_DOC_WITH_SESSION'), // invites inviteRequired: envBool('PDS_INVITE_REQUIRED'), @@ -125,6 +126,7 @@ export type ServerEnvironment = { recoveryDidKey?: string serviceHandleDomains?: string[] // public hostname by default handleBackupNameservers?: string[] + enableDidDocWithSession?: boolean // invites inviteRequired?: boolean diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index 39bc995d533..6c163eedca4 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -2341,6 +2341,9 @@ export const schemaDict = { type: 'string', format: 'did', }, + didDoc: { + type: 'unknown', + }, }, }, }, @@ -2566,6 +2569,9 @@ export const schemaDict = { type: 'string', format: 'did', }, + didDoc: { + type: 'unknown', + }, email: { type: 'string', }, @@ -2882,6 +2888,9 @@ export const schemaDict = { type: 'string', format: 'did', }, + didDoc: { + type: 'unknown', + }, }, }, }, diff --git a/packages/pds/src/lexicon/types/com/atproto/server/createAccount.ts b/packages/pds/src/lexicon/types/com/atproto/server/createAccount.ts index d50d2b24025..bd138919101 100644 --- a/packages/pds/src/lexicon/types/com/atproto/server/createAccount.ts +++ b/packages/pds/src/lexicon/types/com/atproto/server/createAccount.ts @@ -26,6 +26,7 @@ export interface OutputSchema { refreshJwt: string handle: string did: string + didDoc?: {} [k: string]: unknown } diff --git a/packages/pds/src/lexicon/types/com/atproto/server/createSession.ts b/packages/pds/src/lexicon/types/com/atproto/server/createSession.ts index 037900346a1..2cd448703a6 100644 --- a/packages/pds/src/lexicon/types/com/atproto/server/createSession.ts +++ b/packages/pds/src/lexicon/types/com/atproto/server/createSession.ts @@ -22,6 +22,7 @@ export interface OutputSchema { refreshJwt: string handle: string did: string + didDoc?: {} email?: string emailConfirmed?: boolean [k: string]: unknown diff --git a/packages/pds/src/lexicon/types/com/atproto/server/refreshSession.ts b/packages/pds/src/lexicon/types/com/atproto/server/refreshSession.ts index e47bf09fbc2..35874f78a69 100644 --- a/packages/pds/src/lexicon/types/com/atproto/server/refreshSession.ts +++ b/packages/pds/src/lexicon/types/com/atproto/server/refreshSession.ts @@ -17,6 +17,7 @@ export interface OutputSchema { refreshJwt: string handle: string did: string + didDoc?: {} [k: string]: unknown } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d48398fe9cb..1500b3ece5d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -110,6 +110,9 @@ importers: typed-emitter: specifier: ^2.1.0 version: 2.1.0 + zod: + specifier: ^3.21.4 + version: 3.21.4 devDependencies: '@atproto/dev-env': specifier: workspace:^ @@ -407,9 +410,6 @@ importers: axios: specifier: ^0.27.2 version: 0.27.2 - zod: - specifier: ^3.21.4 - version: 3.21.4 devDependencies: '@did-plc/lib': specifier: ^0.0.1 From 0587ad1dbb2a696d3961595d2a4fa108e8604295 Mon Sep 17 00:00:00 2001 From: Devin Ivy Date: Thu, 26 Oct 2023 18:35:59 -0400 Subject: [PATCH 04/59] remove services and dev-env from changeset --- .changeset/purple-shirts-punch.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/.changeset/purple-shirts-punch.md b/.changeset/purple-shirts-punch.md index b0a6c734454..c529294f308 100644 --- a/.changeset/purple-shirts-punch.md +++ b/.changeset/purple-shirts-punch.md @@ -1,10 +1,7 @@ --- '@atproto/common-web': patch '@atproto/identity': patch -'@atproto/dev-env': patch -'@atproto/bsky': patch '@atproto/api': patch -'@atproto/pds': patch --- Allow pds to serve did doc with credentials, API client to respect PDS listed in the did doc. From 22622dcdb2a66fa111c5538e28266acacdeec8fa Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Thu, 26 Oct 2023 18:37:33 -0500 Subject: [PATCH 05/59] Lengthen view maintainer interval (#1775) * lengthen view maintainer interval * dont build branch --- services/bsky/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/bsky/api.js b/services/bsky/api.js index 5363f7661f0..4ae71b17760 100644 --- a/services/bsky/api.js +++ b/services/bsky/api.js @@ -103,7 +103,7 @@ const main = async () => { schema: env.dbPostgresSchema, poolSize: 2, }) - const viewMaintainer = new ViewMaintainer(migrateDb) + const viewMaintainer = new ViewMaintainer(migrateDb, 1800) const viewMaintainerRunning = viewMaintainer.run() const periodicModerationActionReversal = new PeriodicModerationActionReversal( From 35d108ce94866ce1b3d147cd0620a0ba1c4ebcd7 Mon Sep 17 00:00:00 2001 From: Devin Ivy Date: Fri, 27 Oct 2023 12:40:02 -0400 Subject: [PATCH 06/59] bump changeset --- .changeset/{purple-shirts-punch.md => purple-shirts-poncho.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .changeset/{purple-shirts-punch.md => purple-shirts-poncho.md} (100%) diff --git a/.changeset/purple-shirts-punch.md b/.changeset/purple-shirts-poncho.md similarity index 100% rename from .changeset/purple-shirts-punch.md rename to .changeset/purple-shirts-poncho.md From 9c98a5baaf503b02238a6afe4f6e2b79c5181693 Mon Sep 17 00:00:00 2001 From: Paul Frazee Date: Fri, 27 Oct 2023 11:35:36 -0700 Subject: [PATCH 07/59] Modlist updates: add "blockingByList" information and some utility functions to the sdk (#1779) * Add blockingByList to app.bsky.actor.defs#viewerState * Add blocking-by-list behaviors to moderation sdk * Add modlist helper functions to bsky-agent * codegen * hydrate blockingByList in profile viewer state * ignore self-mutes and self-blocks in read path * format * changeset --------- Co-authored-by: Devin Ivy --- .changeset/giant-humans-argue.md | 5 +++ lexicons/app/bsky/actor/defs.json | 4 ++ packages/api/README.md | 4 ++ .../api/definitions/moderation-behaviors.d.ts | 1 + .../profile-moderation-behaviors.json | 25 +++++++++++ .../api/docs/moderation-behaviors/profiles.md | 17 ++++++++ packages/api/src/bsky-agent.ts | 43 +++++++++++++++++++ packages/api/src/client/lexicons.ts | 4 ++ .../src/client/types/app/bsky/actor/defs.ts | 1 + packages/api/src/moderation/accumulator.ts | 13 ++++++ .../api/src/moderation/subjects/account.ts | 8 +++- packages/api/tests/util/index.ts | 3 ++ .../api/tests/util/moderation-behavior.ts | 12 +++++- packages/bsky/src/lexicon/lexicons.ts | 4 ++ .../src/lexicon/types/app/bsky/actor/defs.ts | 1 + packages/bsky/src/services/actor/types.ts | 1 + packages/bsky/src/services/actor/views.ts | 34 +++++++++++---- packages/bsky/src/services/graph/index.ts | 33 +++++++++++--- .../__snapshots__/block-lists.test.ts.snap | 39 ++++++++++++++++- .../__snapshots__/mute-lists.test.ts.snap | 15 ++++++- packages/bsky/tests/views/block-lists.test.ts | 29 +++++++++++++ packages/bsky/tests/views/blocks.test.ts | 4 ++ packages/bsky/tests/views/mute-lists.test.ts | 21 ++++++++- packages/pds/src/lexicon/lexicons.ts | 4 ++ .../src/lexicon/types/app/bsky/actor/defs.ts | 1 + 25 files changed, 303 insertions(+), 23 deletions(-) create mode 100644 .changeset/giant-humans-argue.md diff --git a/.changeset/giant-humans-argue.md b/.changeset/giant-humans-argue.md new file mode 100644 index 00000000000..57efd290904 --- /dev/null +++ b/.changeset/giant-humans-argue.md @@ -0,0 +1,5 @@ +--- +'@atproto/api': patch +--- + +modlist helpers added to bsky-agent, add blockingByList to viewer state lexicon diff --git a/lexicons/app/bsky/actor/defs.json b/lexicons/app/bsky/actor/defs.json index eada4e53897..063072ba181 100644 --- a/lexicons/app/bsky/actor/defs.json +++ b/lexicons/app/bsky/actor/defs.json @@ -86,6 +86,10 @@ }, "blockedBy": { "type": "boolean" }, "blocking": { "type": "string", "format": "at-uri" }, + "blockingByList": { + "type": "ref", + "ref": "app.bsky.graph.defs#listViewBasic" + }, "following": { "type": "string", "format": "at-uri" }, "followedBy": { "type": "string", "format": "at-uri" } } diff --git a/packages/api/README.md b/packages/api/README.md index 069bdb50a5e..9f59f3fbc9b 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -91,6 +91,10 @@ await agent.searchActors(params, opts) await agent.searchActorsTypeahead(params, opts) await agent.mute(did) await agent.unmute(did) +await agent.muteModList(listUri) +await agent.unmuteModList(listUri) +await agent.blockModList(listUri) +await agent.unblockModList(listUri) // Notifications await agent.listNotifications(params, opts) diff --git a/packages/api/definitions/moderation-behaviors.d.ts b/packages/api/definitions/moderation-behaviors.d.ts index 5f6e2df81ca..8d980978623 100644 --- a/packages/api/definitions/moderation-behaviors.d.ts +++ b/packages/api/definitions/moderation-behaviors.d.ts @@ -32,6 +32,7 @@ export interface ModerationBehaviors { string, { blocking: boolean + blockingByList: boolean blockedBy: boolean muted: boolean mutedByList: boolean diff --git a/packages/api/definitions/profile-moderation-behaviors.json b/packages/api/definitions/profile-moderation-behaviors.json index 52e04761618..342592f3421 100644 --- a/packages/api/definitions/profile-moderation-behaviors.json +++ b/packages/api/definitions/profile-moderation-behaviors.json @@ -2,45 +2,59 @@ "users": { "self": { "blocking": false, + "blockingByList": false, "blockedBy": false, "muted": false, "mutedByList": false }, "alice": { "blocking": false, + "blockingByList": false, "blockedBy": false, "muted": false, "mutedByList": false }, "bob": { "blocking": true, + "blockingByList": false, "blockedBy": false, "muted": false, "mutedByList": false }, "carla": { "blocking": false, + "blockingByList": false, "blockedBy": true, "muted": false, "mutedByList": false }, "dan": { "blocking": false, + "blockingByList": false, "blockedBy": false, "muted": true, "mutedByList": false }, "elise": { "blocking": false, + "blockingByList": false, "blockedBy": false, "muted": false, "mutedByList": true }, "fern": { "blocking": true, + "blockingByList": false, "blockedBy": true, "muted": false, "mutedByList": false + }, + "georgia": { + "blocking": false, + "blockingByList": true, + "blockedBy": false, + "muted": false, + "mutedByList": false } }, "configurations": { @@ -377,6 +391,17 @@ } }, + "Mute/block: Blocking-by-list user": { + "cfg": "none", + "subject": "profile", + "author": "georgia", + "labels": {}, + "behaviors": { + "account": { "cause": "blocking-by-list", "filter": true }, + "avatar": { "blur": true, "noOverride": true } + } + }, + "Mute/block: Blocked by user": { "cfg": "none", "subject": "profile", diff --git a/packages/api/docs/moderation-behaviors/profiles.md b/packages/api/docs/moderation-behaviors/profiles.md index b8d7c94ce91..213b3bd3259 100644 --- a/packages/api/docs/moderation-behaviors/profiles.md +++ b/packages/api/docs/moderation-behaviors/profiles.md @@ -547,6 +547,23 @@ Key: + +Mute/block: Blocking-by-list user + +❌ + + + + + + + + +🚫 + + + + Mute/block: Blocked by user diff --git a/packages/api/src/bsky-agent.ts b/packages/api/src/bsky-agent.ts index 37b8d9c3620..ce166a59490 100644 --- a/packages/api/src/bsky-agent.ts +++ b/packages/api/src/bsky-agent.ts @@ -258,6 +258,49 @@ export class BskyAgent extends AtpAgent { return this.api.app.bsky.graph.unmuteActor({ actor }) } + async muteModList(uri: string) { + return this.api.app.bsky.graph.muteActorList({ + list: uri, + }) + } + + async unmuteModList(uri: string) { + return this.api.app.bsky.graph.unmuteActorList({ + list: uri, + }) + } + + async blockModList(uri: string) { + if (!this.session) { + throw new Error('Not logged in') + } + return await this.api.app.bsky.graph.listblock.create( + { repo: this.session.did }, + { + subject: uri, + createdAt: new Date().toISOString(), + }, + ) + } + + async unblockModList(uri: string) { + if (!this.session) { + throw new Error('Not logged in') + } + const listInfo = await this.api.app.bsky.graph.getList({ + list: uri, + limit: 1, + }) + if (!listInfo.data.list.viewer?.blocked) { + return + } + const { rkey } = new AtUri(listInfo.data.list.viewer.blocked) + return await this.api.app.bsky.graph.listblock.delete({ + repo: this.session.did, + rkey, + }) + } + async updateSeenNotifications(seenAt?: string) { seenAt = seenAt || new Date().toISOString() return this.api.app.bsky.notification.updateSeen({ diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index 6c163eedca4..537350b5f13 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -3827,6 +3827,10 @@ export const schemaDict = { type: 'string', format: 'at-uri', }, + blockingByList: { + type: 'ref', + ref: 'lex:app.bsky.graph.defs#listViewBasic', + }, following: { type: 'string', format: 'at-uri', diff --git a/packages/api/src/client/types/app/bsky/actor/defs.ts b/packages/api/src/client/types/app/bsky/actor/defs.ts index 340010680d0..5dd765a8373 100644 --- a/packages/api/src/client/types/app/bsky/actor/defs.ts +++ b/packages/api/src/client/types/app/bsky/actor/defs.ts @@ -87,6 +87,7 @@ export interface ViewerState { mutedByList?: AppBskyGraphDefs.ListViewBasic blockedBy?: boolean blocking?: string + blockingByList?: AppBskyGraphDefs.ListViewBasic following?: string followedBy?: string [k: string]: unknown diff --git a/packages/api/src/moderation/accumulator.ts b/packages/api/src/moderation/accumulator.ts index eebdfd59399..53180023934 100644 --- a/packages/api/src/moderation/accumulator.ts +++ b/packages/api/src/moderation/accumulator.ts @@ -1,4 +1,5 @@ import { AppBskyGraphDefs } from '../client/index' +import { AtUri } from '@atproto/syntax' import { Label, LabelPreference, @@ -28,6 +29,18 @@ export class ModerationCauseAccumulator { } } + addBlockingByList( + blockingByList: AppBskyGraphDefs.ListViewBasic | undefined, + ) { + if (blockingByList) { + this.causes.push({ + type: 'blocking', + source: { type: 'list', list: blockingByList }, + priority: 3, + }) + } + } + addBlockedBy(blockedBy: boolean | undefined) { if (blockedBy) { this.causes.push({ diff --git a/packages/api/src/moderation/subjects/account.ts b/packages/api/src/moderation/subjects/account.ts index 5b6d74369e2..8c763735a79 100644 --- a/packages/api/src/moderation/subjects/account.ts +++ b/packages/api/src/moderation/subjects/account.ts @@ -20,7 +20,13 @@ export function decideAccount( acc.addMuted(subject.viewer?.muted) } } - acc.addBlocking(subject.viewer?.blocking) + if (subject.viewer?.blocking) { + if (subject.viewer?.blockingByList) { + acc.addBlockingByList(subject.viewer?.blockingByList) + } else { + acc.addBlocking(subject.viewer?.blocking) + } + } acc.addBlockedBy(subject.viewer?.blockedBy) for (const label of filterAccountLabels(subject.labels)) { diff --git a/packages/api/tests/util/index.ts b/packages/api/tests/util/index.ts index d9cc5e90780..2a7c3ba7bbf 100644 --- a/packages/api/tests/util/index.ts +++ b/packages/api/tests/util/index.ts @@ -135,6 +135,7 @@ export const mock = { mutedByList, blockedBy, blocking, + blockingByList, following, followedBy, }: { @@ -142,6 +143,7 @@ export const mock = { mutedByList?: AppBskyGraphDefs.ListViewBasic blockedBy?: boolean blocking?: string + blockingByList?: AppBskyGraphDefs.ListViewBasic following?: string followedBy?: string }): AppBskyActorDefs.ViewerState { @@ -150,6 +152,7 @@ export const mock = { mutedByList, blockedBy, blocking, + blockingByList, following, followedBy, } diff --git a/packages/api/tests/util/moderation-behavior.ts b/packages/api/tests/util/moderation-behavior.ts index 4eda83a7b41..fd7df918153 100644 --- a/packages/api/tests/util/moderation-behavior.ts +++ b/packages/api/tests/util/moderation-behavior.ts @@ -25,6 +25,10 @@ expect.extend({ if (actual.cause.source.type === 'list') { cause = 'muted-by-list' } + } else if (actual.cause?.type === 'blocking') { + if (actual.cause.source.type === 'list') { + cause = 'blocking-by-list' + } } if (!expected) { if (!ignoreCause && actual.cause) { @@ -153,8 +157,12 @@ export class ModerationBehaviorSuiteRunner { ? m.listViewBasic({ name: 'Fake List' }) : undefined, blockedBy: def.blockedBy, - blocking: def.blocking - ? 'at://did:web:self.test/app.bsky.graph.block/fake' + blocking: + def.blocking || def.blockingByList + ? 'at://did:web:self.test/app.bsky.graph.block/fake' + : undefined, + blockingByList: def.blockingByList + ? m.listViewBasic({ name: 'Fake List' }) : undefined, }), }) diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index 6c163eedca4..537350b5f13 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -3827,6 +3827,10 @@ export const schemaDict = { type: 'string', format: 'at-uri', }, + blockingByList: { + type: 'ref', + ref: 'lex:app.bsky.graph.defs#listViewBasic', + }, following: { type: 'string', format: 'at-uri', diff --git a/packages/bsky/src/lexicon/types/app/bsky/actor/defs.ts b/packages/bsky/src/lexicon/types/app/bsky/actor/defs.ts index b24b04b34d7..171f5c5ef48 100644 --- a/packages/bsky/src/lexicon/types/app/bsky/actor/defs.ts +++ b/packages/bsky/src/lexicon/types/app/bsky/actor/defs.ts @@ -87,6 +87,7 @@ export interface ViewerState { mutedByList?: AppBskyGraphDefs.ListViewBasic blockedBy?: boolean blocking?: string + blockingByList?: AppBskyGraphDefs.ListViewBasic following?: string followedBy?: string [k: string]: unknown diff --git a/packages/bsky/src/services/actor/types.ts b/packages/bsky/src/services/actor/types.ts index e853406e22e..d622e641099 100644 --- a/packages/bsky/src/services/actor/types.ts +++ b/packages/bsky/src/services/actor/types.ts @@ -16,6 +16,7 @@ export type ActorInfo = { mutedByList?: ListViewBasic blockedBy?: boolean blocking?: string + blockingByList?: ListViewBasic following?: string followedBy?: string } diff --git a/packages/bsky/src/services/actor/views.ts b/packages/bsky/src/services/actor/views.ts index ec39805c76d..7118671bd04 100644 --- a/packages/bsky/src/services/actor/views.ts +++ b/packages/bsky/src/services/actor/views.ts @@ -137,10 +137,13 @@ export class ActorViews { ), ]) const listUris = mapDefined(profiles, ({ did }) => { - const list = viewer && bam.muteList([viewer, did]) - if (!list) return - return list - }) + const muteList = viewer && bam.muteList([viewer, did]) + const blockList = viewer && bam.blockList([viewer, did]) + const lists: string[] = [] + if (muteList) lists.push(muteList) + if (blockList) lists.push(blockList) + return lists + }).flat() const lists = await this.services.graph.getListViews(listUris, viewer) return { profilesDetailed: toMapByDid(profiles), labels, bam, lists } } @@ -168,6 +171,11 @@ export class ActorViews { mutedByListUri && lists[mutedByListUri] ? this.services.graph.formatListViewBasic(lists[mutedByListUri]) : undefined + const blockingByListUri = viewer && bam.blockList([viewer, did]) + const blockingByList = + blockingByListUri && lists[blockingByListUri] + ? this.services.graph.formatListViewBasic(lists[blockingByListUri]) + : undefined const actorLabels = labels[did] ?? [] const selfLabels = getSelfLabels({ uri: prof.profileUri, @@ -194,6 +202,7 @@ export class ActorViews { mutedByList, blockedBy: !!bam.blockedBy([viewer, did]), blocking: bam.blocking([viewer, did]) ?? undefined, + blockingByList, following: prof?.viewerFollowing && !bam.block([viewer, did]) ? prof.viewerFollowing @@ -265,10 +274,13 @@ export class ActorViews { ), ]) const listUris = mapDefined(profiles, ({ did }) => { - const list = viewer && bam.muteList([viewer, did]) - if (!list) return - return list - }) + const muteList = viewer && bam.muteList([viewer, did]) + const blockList = viewer && bam.blockList([viewer, did]) + const lists: string[] = [] + if (muteList) lists.push(muteList) + if (blockList) lists.push(blockList) + return lists + }).flat() const lists = await this.services.graph.getListViews(listUris, viewer) return { profiles: toMapByDid(profiles), labels, bam, lists } } @@ -298,6 +310,11 @@ export class ActorViews { mutedByListUri && lists[mutedByListUri] ? this.services.graph.formatListViewBasic(lists[mutedByListUri]) : undefined + const blockingByListUri = viewer && bam.blockList([viewer, did]) + const blockingByList = + blockingByListUri && lists[blockingByListUri] + ? this.services.graph.formatListViewBasic(lists[blockingByListUri]) + : undefined const actorLabels = labels[did] ?? [] const selfLabels = getSelfLabels({ uri: prof.profileUri, @@ -320,6 +337,7 @@ export class ActorViews { mutedByList, blockedBy: !!bam.blockedBy([viewer, did]), blocking: bam.blocking([viewer, did]) ?? undefined, + blockingByList, following: prof?.viewerFollowing && !bam.block([viewer, did]) ? prof.viewerFollowing diff --git a/packages/bsky/src/services/graph/index.ts b/packages/bsky/src/services/graph/index.ts index 53592ac4021..eadf035db1a 100644 --- a/packages/bsky/src/services/graph/index.ts +++ b/packages/bsky/src/services/graph/index.ts @@ -267,28 +267,44 @@ export type RelationshipPair = [didA: string, didB: string] export class BlockAndMuteState { hasIdx = new Map>() // did -> did blockIdx = new Map>() // did -> did -> block uri + blockListIdx = new Map>() // did -> did -> list uri muteIdx = new Map>() // did -> did muteListIdx = new Map>() // did -> did -> list uri constructor(items: BlockAndMuteInfo[] = []) { items.forEach((item) => this.add(item)) } add(item: BlockAndMuteInfo) { - const blocking = item.blocking || item.blockingViaList // block or list uri - if (blocking) { + if (item.source === item.target) { + return // we do not respect self-blocks or self-mutes + } + if (item.blocking) { const map = this.blockIdx.get(item.source) ?? new Map() - map.set(item.target, blocking) + map.set(item.target, item.blocking) if (!this.blockIdx.has(item.source)) { this.blockIdx.set(item.source, map) } } - const blockedBy = item.blockedBy || item.blockedByViaList // block or list uri - if (blockedBy) { + if (item.blockingViaList) { + const map = this.blockListIdx.get(item.source) ?? new Map() + map.set(item.target, item.blockingViaList) + if (!this.blockListIdx.has(item.source)) { + this.blockListIdx.set(item.source, map) + } + } + if (item.blockedBy) { const map = this.blockIdx.get(item.target) ?? new Map() - map.set(item.source, blockedBy) + map.set(item.source, item.blockedBy) if (!this.blockIdx.has(item.target)) { this.blockIdx.set(item.target, map) } } + if (item.blockedByViaList) { + const map = this.blockListIdx.get(item.target) ?? new Map() + map.set(item.source, item.blockedByViaList) + if (!this.blockListIdx.has(item.target)) { + this.blockListIdx.set(item.target, map) + } + } if (item.muting) { const set = this.muteIdx.get(item.source) ?? new Set() set.add(item.target) @@ -314,7 +330,7 @@ export class BlockAndMuteState { } // block or list uri blocking(pair: RelationshipPair): string | null { - return this.blockIdx.get(pair[0])?.get(pair[1]) ?? null + return this.blockIdx.get(pair[0])?.get(pair[1]) ?? this.blockList(pair) } // block or list uri blockedBy(pair: RelationshipPair): string | null { @@ -324,6 +340,9 @@ export class BlockAndMuteState { return !!this.muteIdx.get(pair[0])?.has(pair[1]) || !!this.muteList(pair) } // list uri + blockList(pair: RelationshipPair): string | null { + return this.blockListIdx.get(pair[0])?.get(pair[1]) ?? null + } muteList(pair: RelationshipPair): string | null { return this.muteListIdx.get(pair[0])?.get(pair[1]) ?? null } diff --git a/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap b/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap index 47648b81eac..7843adb6cc8 100644 --- a/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap +++ b/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap @@ -491,20 +491,43 @@ Object { Object { "subject": Object { "did": "user(2)", + "handle": "dan.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "muted": false, + }, + }, + }, + Object { + "subject": Object { + "did": "user(3)", "handle": "carol.test", "labels": Array [], "viewer": Object { "blockedBy": false, "blocking": "record(0)", + "blockingByList": Object { + "avatar": "https://bsky.public.url/img/avatar/plain/user(0)/cids(1)@jpeg", + "cid": "cids(0)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "name": "alice blocks", + "purpose": "app.bsky.graph.defs#modlist", + "uri": "record(0)", + "viewer": Object { + "blocked": "record(1)", + "muted": false, + }, + }, "muted": false, }, }, }, Object { "subject": Object { - "avatar": "https://bsky.public.url/img/avatar/plain/user(4)/cids(1)@jpeg", + "avatar": "https://bsky.public.url/img/avatar/plain/user(5)/cids(1)@jpeg", "description": "hi im bob label_me", - "did": "user(3)", + "did": "user(4)", "displayName": "bobby", "handle": "bob.test", "indexedAt": "1970-01-01T00:00:00.000Z", @@ -512,6 +535,18 @@ Object { "viewer": Object { "blockedBy": false, "blocking": "record(0)", + "blockingByList": Object { + "avatar": "https://bsky.public.url/img/avatar/plain/user(0)/cids(1)@jpeg", + "cid": "cids(0)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "name": "alice blocks", + "purpose": "app.bsky.graph.defs#modlist", + "uri": "record(0)", + "viewer": Object { + "blocked": "record(1)", + "muted": false, + }, + }, "muted": false, }, }, diff --git a/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap b/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap index 2824414f97b..b58e7a3734f 100644 --- a/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap +++ b/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap @@ -491,6 +491,17 @@ Object { Object { "subject": Object { "did": "user(2)", + "handle": "dan.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "muted": false, + }, + }, + }, + Object { + "subject": Object { + "did": "user(3)", "handle": "carol.test", "labels": Array [], "viewer": Object { @@ -513,9 +524,9 @@ Object { }, Object { "subject": Object { - "avatar": "https://bsky.public.url/img/avatar/plain/user(4)/cids(1)@jpeg", + "avatar": "https://bsky.public.url/img/avatar/plain/user(5)/cids(1)@jpeg", "description": "hi im bob label_me", - "did": "user(3)", + "did": "user(4)", "displayName": "bobby", "handle": "bob.test", "indexedAt": "1970-01-01T00:00:00.000Z", diff --git a/packages/bsky/tests/views/block-lists.test.ts b/packages/bsky/tests/views/block-lists.test.ts index 073590c5b34..6672d690ce1 100644 --- a/packages/bsky/tests/views/block-lists.test.ts +++ b/packages/bsky/tests/views/block-lists.test.ts @@ -84,6 +84,15 @@ describe('pds views with blocking from block lists', () => { }, sc.getHeaders(alice), ) + await pdsAgent.api.app.bsky.graph.listitem.create( + { repo: alice }, + { + subject: sc.dids.dan, + list: list.uri, + createdAt: new Date().toISOString(), + }, + sc.getHeaders(alice), + ) await network.processAll() }) @@ -192,6 +201,7 @@ describe('pds views with blocking from block lists', () => { { headers: await network.serviceHeaders(carol) }, ) expect(resCarol.data.viewer?.blocking).toBeUndefined() + expect(resCarol.data.viewer?.blockingByList).toBeUndefined() expect(resCarol.data.viewer?.blockedBy).toBe(true) const resDan = await agent.api.app.bsky.actor.getProfile( @@ -199,6 +209,9 @@ describe('pds views with blocking from block lists', () => { { headers: await network.serviceHeaders(dan) }, ) expect(resDan.data.viewer?.blocking).toBeDefined() + expect(resDan.data.viewer?.blockingByList?.uri).toEqual( + resDan.data.viewer?.blocking, + ) expect(resDan.data.viewer?.blockedBy).toBe(false) }) @@ -208,8 +221,10 @@ describe('pds views with blocking from block lists', () => { { headers: await network.serviceHeaders(carol) }, ) expect(resCarol.data.profiles[0].viewer?.blocking).toBeUndefined() + expect(resCarol.data.profiles[0].viewer?.blockingByList).toBeUndefined() expect(resCarol.data.profiles[0].viewer?.blockedBy).toBe(false) expect(resCarol.data.profiles[1].viewer?.blocking).toBeUndefined() + expect(resCarol.data.profiles[1].viewer?.blockingByList).toBeUndefined() expect(resCarol.data.profiles[1].viewer?.blockedBy).toBe(true) const resDan = await agent.api.app.bsky.actor.getProfiles( @@ -217,11 +232,25 @@ describe('pds views with blocking from block lists', () => { { headers: await network.serviceHeaders(dan) }, ) expect(resDan.data.profiles[0].viewer?.blocking).toBeUndefined() + expect(resDan.data.profiles[0].viewer?.blockingByList).toBeUndefined() expect(resDan.data.profiles[0].viewer?.blockedBy).toBe(false) expect(resDan.data.profiles[1].viewer?.blocking).toBeDefined() + expect(resDan.data.profiles[1].viewer?.blockingByList?.uri).toEqual( + resDan.data.profiles[1].viewer?.blocking, + ) expect(resDan.data.profiles[1].viewer?.blockedBy).toBe(false) }) + it('ignores self-blocks', async () => { + const res = await agent.api.app.bsky.actor.getProfile( + { actor: dan }, // dan subscribes to list that contains himself + { headers: await network.serviceHeaders(dan) }, + ) + expect(res.data.viewer?.blocking).toBeUndefined() + expect(res.data.viewer?.blockingByList).toBeUndefined() + expect(res.data.viewer?.blockedBy).toBe(false) + }) + it('does not return notifs for blocked accounts', async () => { const resCarol = await agent.api.app.bsky.notification.listNotifications( { diff --git a/packages/bsky/tests/views/blocks.test.ts b/packages/bsky/tests/views/blocks.test.ts index 312e997cb36..5d344a823d9 100644 --- a/packages/bsky/tests/views/blocks.test.ts +++ b/packages/bsky/tests/views/blocks.test.ts @@ -225,8 +225,10 @@ describe('pds views with blocking', () => { { headers: await network.serviceHeaders(carol) }, ) expect(resCarol.data.profiles[0].viewer?.blocking).toBeUndefined() + expect(resCarol.data.profiles[0].viewer?.blockingByList).toBeUndefined() expect(resCarol.data.profiles[0].viewer?.blockedBy).toBe(false) expect(resCarol.data.profiles[1].viewer?.blocking).toBeUndefined() + expect(resCarol.data.profiles[1].viewer?.blockingByList).toBeUndefined() expect(resCarol.data.profiles[1].viewer?.blockedBy).toBe(true) const resDan = await agent.api.app.bsky.actor.getProfiles( @@ -234,8 +236,10 @@ describe('pds views with blocking', () => { { headers: await network.serviceHeaders(dan) }, ) expect(resDan.data.profiles[0].viewer?.blocking).toBeUndefined() + expect(resDan.data.profiles[0].viewer?.blockingByList).toBeUndefined() expect(resDan.data.profiles[0].viewer?.blockedBy).toBe(false) expect(resDan.data.profiles[1].viewer?.blocking).toBeDefined() + expect(resDan.data.profiles[1].viewer?.blockingByList).toBeUndefined() expect(resDan.data.profiles[1].viewer?.blockedBy).toBe(false) }) diff --git a/packages/bsky/tests/views/mute-lists.test.ts b/packages/bsky/tests/views/mute-lists.test.ts index de2a047b654..07a6690f910 100644 --- a/packages/bsky/tests/views/mute-lists.test.ts +++ b/packages/bsky/tests/views/mute-lists.test.ts @@ -79,6 +79,16 @@ describe('bsky views with mutes from mute lists', () => { }, sc.getHeaders(alice), ) + await pdsAgent.api.app.bsky.graph.listitem.create( + { repo: alice }, + { + subject: sc.dids.dan, + list: list.uri, + reason: 'idk', + createdAt: new Date().toISOString(), + }, + sc.getHeaders(alice), + ) await network.processAll() }) @@ -160,6 +170,15 @@ describe('bsky views with mutes from mute lists', () => { expect(res.data.profiles[1].viewer?.mutedByList?.uri).toEqual(listUri) }) + it('ignores self-mutes', async () => { + const res = await agent.api.app.bsky.actor.getProfile( + { actor: dan }, // dan subscribes to list that contains himself + { headers: await network.serviceHeaders(dan) }, + ) + expect(res.data.viewer?.muted).toBe(false) + expect(res.data.viewer?.mutedByList).toBeUndefined() + }) + it('does not return notifs for muted accounts', async () => { const res = await agent.api.app.bsky.notification.listNotifications( { @@ -342,7 +361,7 @@ describe('bsky views with mutes from mute lists', () => { expect(got.data.list.name).toBe('updated alice mutes') expect(got.data.list.description).toBe('new descript') expect(got.data.list.avatar).toBeUndefined() - expect(got.data.items.length).toBe(2) + expect(got.data.items.length).toBe(3) }) it('embeds lists in posts', async () => { diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index 6c163eedca4..537350b5f13 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -3827,6 +3827,10 @@ export const schemaDict = { type: 'string', format: 'at-uri', }, + blockingByList: { + type: 'ref', + ref: 'lex:app.bsky.graph.defs#listViewBasic', + }, following: { type: 'string', format: 'at-uri', diff --git a/packages/pds/src/lexicon/types/app/bsky/actor/defs.ts b/packages/pds/src/lexicon/types/app/bsky/actor/defs.ts index b24b04b34d7..171f5c5ef48 100644 --- a/packages/pds/src/lexicon/types/app/bsky/actor/defs.ts +++ b/packages/pds/src/lexicon/types/app/bsky/actor/defs.ts @@ -87,6 +87,7 @@ export interface ViewerState { mutedByList?: AppBskyGraphDefs.ListViewBasic blockedBy?: boolean blocking?: string + blockingByList?: AppBskyGraphDefs.ListViewBasic following?: string followedBy?: string [k: string]: unknown From ea04096b0229a42c6aae6913a8bc5faf32ad1046 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 27 Oct 2023 14:51:50 -0400 Subject: [PATCH 08/59] Version packages (#1774) Co-authored-by: github-actions[bot] --- .changeset/giant-humans-argue.md | 5 ----- .changeset/new-snails-hope.md | 5 ----- .changeset/purple-shirts-poncho.md | 7 ------- packages/api/CHANGELOG.md | 14 ++++++++++++++ packages/api/package.json | 2 +- packages/aws/CHANGELOG.md | 8 ++++++++ packages/aws/package.json | 2 +- packages/bsky/CHANGELOG.md | 13 +++++++++++++ packages/bsky/package.json | 2 +- packages/common-web/CHANGELOG.md | 6 ++++++ packages/common-web/package.json | 2 +- packages/common/CHANGELOG.md | 7 +++++++ packages/common/package.json | 2 +- packages/dev-env/CHANGELOG.md | 14 ++++++++++++++ packages/dev-env/package.json | 2 +- packages/identity/CHANGELOG.md | 13 +++++++++++++ packages/identity/package.json | 2 +- packages/lex-cli/CHANGELOG.md | 8 ++++++++ packages/lex-cli/package.json | 2 +- packages/lexicon/CHANGELOG.md | 8 ++++++++ packages/lexicon/package.json | 2 +- packages/pds/CHANGELOG.md | 15 +++++++++++++++ packages/pds/package.json | 2 +- packages/repo/CHANGELOG.md | 11 +++++++++++ packages/repo/package.json | 2 +- packages/syntax/CHANGELOG.md | 7 +++++++ packages/syntax/package.json | 2 +- packages/xrpc-server/CHANGELOG.md | 8 ++++++++ packages/xrpc-server/package.json | 2 +- packages/xrpc/CHANGELOG.md | 7 +++++++ packages/xrpc/package.json | 2 +- 31 files changed, 153 insertions(+), 31 deletions(-) delete mode 100644 .changeset/giant-humans-argue.md delete mode 100644 .changeset/new-snails-hope.md delete mode 100644 .changeset/purple-shirts-poncho.md diff --git a/.changeset/giant-humans-argue.md b/.changeset/giant-humans-argue.md deleted file mode 100644 index 57efd290904..00000000000 --- a/.changeset/giant-humans-argue.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@atproto/api': patch ---- - -modlist helpers added to bsky-agent, add blockingByList to viewer state lexicon diff --git a/.changeset/new-snails-hope.md b/.changeset/new-snails-hope.md deleted file mode 100644 index 2526d0836ed..00000000000 --- a/.changeset/new-snails-hope.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@atproto/identity': minor ---- - -Pass stale did doc into refresh cache functions diff --git a/.changeset/purple-shirts-poncho.md b/.changeset/purple-shirts-poncho.md deleted file mode 100644 index c529294f308..00000000000 --- a/.changeset/purple-shirts-poncho.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@atproto/common-web': patch -'@atproto/identity': patch -'@atproto/api': patch ---- - -Allow pds to serve did doc with credentials, API client to respect PDS listed in the did doc. diff --git a/packages/api/CHANGELOG.md b/packages/api/CHANGELOG.md index 2a42956bc31..d3c3a00c131 100644 --- a/packages/api/CHANGELOG.md +++ b/packages/api/CHANGELOG.md @@ -1,5 +1,19 @@ # @atproto/api +## 0.6.21 + +### Patch Changes + +- [#1779](https://github.com/bluesky-social/atproto/pull/1779) [`9c98a5ba`](https://github.com/bluesky-social/atproto/commit/9c98a5baaf503b02238a6afe4f6e2b79c5181693) Thanks [@pfrazee](https://github.com/pfrazee)! - modlist helpers added to bsky-agent, add blockingByList to viewer state lexicon + +- [`35d108ce`](https://github.com/bluesky-social/atproto/commit/35d108ce94866ce1b3d147cd0620a0ba1c4ebcd7) Thanks [@devinivy](https://github.com/devinivy)! - Allow pds to serve did doc with credentials, API client to respect PDS listed in the did doc. + +- Updated dependencies [[`35d108ce`](https://github.com/bluesky-social/atproto/commit/35d108ce94866ce1b3d147cd0620a0ba1c4ebcd7)]: + - @atproto/common-web@0.2.2 + - @atproto/lexicon@0.2.3 + - @atproto/syntax@0.1.3 + - @atproto/xrpc@0.3.3 + ## 0.6.20 ### Patch Changes diff --git a/packages/api/package.json b/packages/api/package.json index e4a1d662ea4..ed373c472c4 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/api", - "version": "0.6.20", + "version": "0.6.21", "license": "MIT", "description": "Client library for atproto and Bluesky", "keywords": [ diff --git a/packages/aws/CHANGELOG.md b/packages/aws/CHANGELOG.md index 49c88194d10..a587c2e8e26 100644 --- a/packages/aws/CHANGELOG.md +++ b/packages/aws/CHANGELOG.md @@ -1,5 +1,13 @@ # @atproto/aws +## 0.1.3 + +### Patch Changes + +- Updated dependencies []: + - @atproto/repo@0.3.3 + - @atproto/common@0.3.2 + ## 0.1.2 ### Patch Changes diff --git a/packages/aws/package.json b/packages/aws/package.json index 8bf82637175..b370a69f5f4 100644 --- a/packages/aws/package.json +++ b/packages/aws/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/aws", - "version": "0.1.2", + "version": "0.1.3", "license": "MIT", "description": "Shared AWS cloud API helpers for atproto services", "keywords": [ diff --git a/packages/bsky/CHANGELOG.md b/packages/bsky/CHANGELOG.md index 13c794397af..867c1b62660 100644 --- a/packages/bsky/CHANGELOG.md +++ b/packages/bsky/CHANGELOG.md @@ -1,5 +1,18 @@ # @atproto/bsky +## 0.0.12 + +### Patch Changes + +- Updated dependencies [[`9c98a5ba`](https://github.com/bluesky-social/atproto/commit/9c98a5baaf503b02238a6afe4f6e2b79c5181693), [`bb039d8e`](https://github.com/bluesky-social/atproto/commit/bb039d8e4ce5b7f70c4f3e86d1327e210ef24dc3), [`35d108ce`](https://github.com/bluesky-social/atproto/commit/35d108ce94866ce1b3d147cd0620a0ba1c4ebcd7)]: + - @atproto/api@0.6.21 + - @atproto/identity@0.3.0 + - @atproto/repo@0.3.3 + - @atproto/common@0.3.2 + - @atproto/lexicon@0.2.3 + - @atproto/syntax@0.1.3 + - @atproto/xrpc-server@0.3.3 + ## 0.0.11 ### Patch Changes diff --git a/packages/bsky/package.json b/packages/bsky/package.json index 77fe28c7bf6..ab2b69cb5a6 100644 --- a/packages/bsky/package.json +++ b/packages/bsky/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/bsky", - "version": "0.0.11", + "version": "0.0.12", "license": "MIT", "description": "Reference implementation of app.bsky App View (Bluesky API)", "keywords": [ diff --git a/packages/common-web/CHANGELOG.md b/packages/common-web/CHANGELOG.md index ffe63bdf95d..28bc7f04df7 100644 --- a/packages/common-web/CHANGELOG.md +++ b/packages/common-web/CHANGELOG.md @@ -1,5 +1,11 @@ # @atproto/common-web +## 0.2.2 + +### Patch Changes + +- [`35d108ce`](https://github.com/bluesky-social/atproto/commit/35d108ce94866ce1b3d147cd0620a0ba1c4ebcd7) Thanks [@devinivy](https://github.com/devinivy)! - Allow pds to serve did doc with credentials, API client to respect PDS listed in the did doc. + ## 0.2.1 ### Patch Changes diff --git a/packages/common-web/package.json b/packages/common-web/package.json index 9145c4d2f5f..01d5768f91e 100644 --- a/packages/common-web/package.json +++ b/packages/common-web/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/common-web", - "version": "0.2.1", + "version": "0.2.2", "license": "MIT", "description": "Shared web-platform-friendly code for atproto libraries", "keywords": [ diff --git a/packages/common/CHANGELOG.md b/packages/common/CHANGELOG.md index 2bc819a30dc..8c9ff1b090a 100644 --- a/packages/common/CHANGELOG.md +++ b/packages/common/CHANGELOG.md @@ -1,5 +1,12 @@ # @atproto/common +## 0.3.2 + +### Patch Changes + +- Updated dependencies [[`35d108ce`](https://github.com/bluesky-social/atproto/commit/35d108ce94866ce1b3d147cd0620a0ba1c4ebcd7)]: + - @atproto/common-web@0.2.2 + ## 0.3.1 ### Patch Changes diff --git a/packages/common/package.json b/packages/common/package.json index 4803b9d0d19..beef7fe7fa4 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/common", - "version": "0.3.1", + "version": "0.3.2", "license": "MIT", "description": "Shared web-platform-friendly code for atproto libraries", "keywords": [ diff --git a/packages/dev-env/CHANGELOG.md b/packages/dev-env/CHANGELOG.md index fc9111c0a1a..3f00886dded 100644 --- a/packages/dev-env/CHANGELOG.md +++ b/packages/dev-env/CHANGELOG.md @@ -1,5 +1,19 @@ # @atproto/dev-env +## 0.2.12 + +### Patch Changes + +- Updated dependencies [[`9c98a5ba`](https://github.com/bluesky-social/atproto/commit/9c98a5baaf503b02238a6afe4f6e2b79c5181693), [`bb039d8e`](https://github.com/bluesky-social/atproto/commit/bb039d8e4ce5b7f70c4f3e86d1327e210ef24dc3), [`35d108ce`](https://github.com/bluesky-social/atproto/commit/35d108ce94866ce1b3d147cd0620a0ba1c4ebcd7)]: + - @atproto/api@0.6.21 + - @atproto/identity@0.3.0 + - @atproto/common-web@0.2.2 + - @atproto/bsky@0.0.12 + - @atproto/pds@0.3.0 + - @atproto/lexicon@0.2.3 + - @atproto/syntax@0.1.3 + - @atproto/xrpc-server@0.3.3 + ## 0.2.11 ### Patch Changes diff --git a/packages/dev-env/package.json b/packages/dev-env/package.json index 80f2b06a747..7cf3dbb2ade 100644 --- a/packages/dev-env/package.json +++ b/packages/dev-env/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/dev-env", - "version": "0.2.11", + "version": "0.2.12", "license": "MIT", "description": "Local development environment helper for atproto development", "keywords": [ diff --git a/packages/identity/CHANGELOG.md b/packages/identity/CHANGELOG.md index bcabd8e37e4..321025853e2 100644 --- a/packages/identity/CHANGELOG.md +++ b/packages/identity/CHANGELOG.md @@ -1,5 +1,18 @@ # @atproto/identity +## 0.3.0 + +### Minor Changes + +- [#1773](https://github.com/bluesky-social/atproto/pull/1773) [`bb039d8e`](https://github.com/bluesky-social/atproto/commit/bb039d8e4ce5b7f70c4f3e86d1327e210ef24dc3) Thanks [@dholms](https://github.com/dholms)! - Pass stale did doc into refresh cache functions + +### Patch Changes + +- [`35d108ce`](https://github.com/bluesky-social/atproto/commit/35d108ce94866ce1b3d147cd0620a0ba1c4ebcd7) Thanks [@devinivy](https://github.com/devinivy)! - Allow pds to serve did doc with credentials, API client to respect PDS listed in the did doc. + +- Updated dependencies [[`35d108ce`](https://github.com/bluesky-social/atproto/commit/35d108ce94866ce1b3d147cd0620a0ba1c4ebcd7)]: + - @atproto/common-web@0.2.2 + ## 0.2.1 ### Patch Changes diff --git a/packages/identity/package.json b/packages/identity/package.json index 96e654c503b..119017e77d0 100644 --- a/packages/identity/package.json +++ b/packages/identity/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/identity", - "version": "0.2.1", + "version": "0.3.0", "license": "MIT", "description": "Library for decentralized identities in atproto using DIDs and handles", "keywords": [ diff --git a/packages/lex-cli/CHANGELOG.md b/packages/lex-cli/CHANGELOG.md index 1faad66a912..02f8d6e9a8e 100644 --- a/packages/lex-cli/CHANGELOG.md +++ b/packages/lex-cli/CHANGELOG.md @@ -1,5 +1,13 @@ # @atproto/lex-cli +## 0.2.3 + +### Patch Changes + +- Updated dependencies []: + - @atproto/lexicon@0.2.3 + - @atproto/syntax@0.1.3 + ## 0.2.2 ### Patch Changes diff --git a/packages/lex-cli/package.json b/packages/lex-cli/package.json index 2f2c34c0236..ee8d7b47184 100644 --- a/packages/lex-cli/package.json +++ b/packages/lex-cli/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/lex-cli", - "version": "0.2.2", + "version": "0.2.3", "license": "MIT", "description": "TypeScript codegen tool for atproto Lexicon schemas", "keywords": [ diff --git a/packages/lexicon/CHANGELOG.md b/packages/lexicon/CHANGELOG.md index f2207b268a4..a5efe153f5c 100644 --- a/packages/lexicon/CHANGELOG.md +++ b/packages/lexicon/CHANGELOG.md @@ -1,5 +1,13 @@ # @atproto/lexicon +## 0.2.3 + +### Patch Changes + +- Updated dependencies [[`35d108ce`](https://github.com/bluesky-social/atproto/commit/35d108ce94866ce1b3d147cd0620a0ba1c4ebcd7)]: + - @atproto/common-web@0.2.2 + - @atproto/syntax@0.1.3 + ## 0.2.2 ### Patch Changes diff --git a/packages/lexicon/package.json b/packages/lexicon/package.json index e43c9c6e80a..b4eaeeedf85 100644 --- a/packages/lexicon/package.json +++ b/packages/lexicon/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/lexicon", - "version": "0.2.2", + "version": "0.2.3", "license": "MIT", "description": "atproto Lexicon schema language library", "keywords": [ diff --git a/packages/pds/CHANGELOG.md b/packages/pds/CHANGELOG.md index 75d24e70991..bd333402677 100644 --- a/packages/pds/CHANGELOG.md +++ b/packages/pds/CHANGELOG.md @@ -1,5 +1,20 @@ # @atproto/pds +## 0.3.0 + +### Patch Changes + +- Updated dependencies [[`9c98a5ba`](https://github.com/bluesky-social/atproto/commit/9c98a5baaf503b02238a6afe4f6e2b79c5181693), [`bb039d8e`](https://github.com/bluesky-social/atproto/commit/bb039d8e4ce5b7f70c4f3e86d1327e210ef24dc3), [`35d108ce`](https://github.com/bluesky-social/atproto/commit/35d108ce94866ce1b3d147cd0620a0ba1c4ebcd7)]: + - @atproto/api@0.6.21 + - @atproto/identity@0.3.0 + - @atproto/repo@0.3.3 + - @atproto/common@0.3.2 + - @atproto/lexicon@0.2.3 + - @atproto/syntax@0.1.3 + - @atproto/aws@0.1.3 + - @atproto/xrpc-server@0.3.3 + - @atproto/xrpc@0.3.3 + ## 0.1.20 ### Patch Changes diff --git a/packages/pds/package.json b/packages/pds/package.json index 4c22a61133b..a7a61e86ea2 100644 --- a/packages/pds/package.json +++ b/packages/pds/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/pds", - "version": "0.3.0-beta.3", + "version": "0.3.0", "license": "MIT", "description": "Reference implementation of atproto Personal Data Server (PDS)", "keywords": [ diff --git a/packages/repo/CHANGELOG.md b/packages/repo/CHANGELOG.md index 40fff4f310d..05258506fe9 100644 --- a/packages/repo/CHANGELOG.md +++ b/packages/repo/CHANGELOG.md @@ -1,5 +1,16 @@ # @atproto/repo +## 0.3.3 + +### Patch Changes + +- Updated dependencies [[`bb039d8e`](https://github.com/bluesky-social/atproto/commit/bb039d8e4ce5b7f70c4f3e86d1327e210ef24dc3), [`35d108ce`](https://github.com/bluesky-social/atproto/commit/35d108ce94866ce1b3d147cd0620a0ba1c4ebcd7)]: + - @atproto/identity@0.3.0 + - @atproto/common-web@0.2.2 + - @atproto/common@0.3.2 + - @atproto/lexicon@0.2.3 + - @atproto/syntax@0.1.3 + ## 0.3.2 ### Patch Changes diff --git a/packages/repo/package.json b/packages/repo/package.json index 1264656f4d4..b8839ed9f6a 100644 --- a/packages/repo/package.json +++ b/packages/repo/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/repo", - "version": "0.3.2", + "version": "0.3.3", "license": "MIT", "description": "atproto repo and MST implementation", "keywords": [ diff --git a/packages/syntax/CHANGELOG.md b/packages/syntax/CHANGELOG.md index ed367a2f2ab..4df89fb53a1 100644 --- a/packages/syntax/CHANGELOG.md +++ b/packages/syntax/CHANGELOG.md @@ -1,5 +1,12 @@ # @atproto/syntax +## 0.1.3 + +### Patch Changes + +- Updated dependencies [[`35d108ce`](https://github.com/bluesky-social/atproto/commit/35d108ce94866ce1b3d147cd0620a0ba1c4ebcd7)]: + - @atproto/common-web@0.2.2 + ## 0.1.2 ### Patch Changes diff --git a/packages/syntax/package.json b/packages/syntax/package.json index c14254f04af..ff59423f8eb 100644 --- a/packages/syntax/package.json +++ b/packages/syntax/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/syntax", - "version": "0.1.2", + "version": "0.1.3", "license": "MIT", "description": "Validation for atproto identifiers and formats: DID, handle, NSID, AT URI, etc", "keywords": [ diff --git a/packages/xrpc-server/CHANGELOG.md b/packages/xrpc-server/CHANGELOG.md index e3c8d9ae135..4be9b022bea 100644 --- a/packages/xrpc-server/CHANGELOG.md +++ b/packages/xrpc-server/CHANGELOG.md @@ -1,5 +1,13 @@ # @atproto/xrpc-server +## 0.3.3 + +### Patch Changes + +- Updated dependencies []: + - @atproto/common@0.3.2 + - @atproto/lexicon@0.2.3 + ## 0.3.2 ### Patch Changes diff --git a/packages/xrpc-server/package.json b/packages/xrpc-server/package.json index ef3c56b3bc0..d32319f301b 100644 --- a/packages/xrpc-server/package.json +++ b/packages/xrpc-server/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/xrpc-server", - "version": "0.3.2", + "version": "0.3.3", "license": "MIT", "description": "atproto HTTP API (XRPC) server library", "keywords": [ diff --git a/packages/xrpc/CHANGELOG.md b/packages/xrpc/CHANGELOG.md index e12319fa88e..2d37c765ead 100644 --- a/packages/xrpc/CHANGELOG.md +++ b/packages/xrpc/CHANGELOG.md @@ -1,5 +1,12 @@ # @atproto/xrpc +## 0.3.3 + +### Patch Changes + +- Updated dependencies []: + - @atproto/lexicon@0.2.3 + ## 0.3.2 ### Patch Changes diff --git a/packages/xrpc/package.json b/packages/xrpc/package.json index 58ed98bd88f..f9845c7fcfb 100644 --- a/packages/xrpc/package.json +++ b/packages/xrpc/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/xrpc", - "version": "0.3.2", + "version": "0.3.3", "license": "MIT", "description": "atproto HTTP API (XRPC) client library", "keywords": [ From ec0dfdc8f5a032e3fa59eea4a0c7586a749a1fca Mon Sep 17 00:00:00 2001 From: bnewbold Date: Mon, 30 Oct 2023 09:28:42 -0700 Subject: [PATCH 09/59] lexicon: maximum report "reason" length of 1000 chars (graphemes) (#1171) * lexicon: maximum report length of 500 chars (graphemes) * lexicon: bump maximum report size to 1000 chars * lexicon: bump max report size again to 2k graphemes * make codegen --- lexicons/com/atproto/moderation/createReport.json | 6 +++++- packages/api/src/client/lexicons.ts | 2 ++ packages/bsky/src/lexicon/lexicons.ts | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lexicons/com/atproto/moderation/createReport.json b/lexicons/com/atproto/moderation/createReport.json index 0f34ed4329b..161d622fcf2 100644 --- a/lexicons/com/atproto/moderation/createReport.json +++ b/lexicons/com/atproto/moderation/createReport.json @@ -43,7 +43,11 @@ "type": "ref", "ref": "com.atproto.moderation.defs#reasonType" }, - "reason": { "type": "string" }, + "reason": { + "type": "string", + "maxGraphemes": 2000, + "maxLength": 20000 + }, "subject": { "type": "union", "refs": [ diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index 537350b5f13..3911dc12497 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -1632,6 +1632,8 @@ export const schemaDict = { }, reason: { type: 'string', + maxGraphemes: 2000, + maxLength: 20000, }, subject: { type: 'union', diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index 537350b5f13..3911dc12497 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -1632,6 +1632,8 @@ export const schemaDict = { }, reason: { type: 'string', + maxGraphemes: 2000, + maxLength: 20000, }, subject: { type: 'union', From fcb19c9c51daae15e5853e194404493ec22667e9 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Mon, 30 Oct 2023 16:56:17 -0500 Subject: [PATCH 10/59] Simplify PDS moderation (#1723) * spec out new simple pds mod routes * introduce new admin state endpoints * wire up routes * clean up pds * revoke refresh tokens * getUserAccountInfo * pr tidy * fixing some tests * fixing up more tests * fanout takedowns to pds * fanout admin reqs to pds * tidy * more tidy & add more pds moderation tests * getUserAccountInfo -> getAccountInfo * dont hydrate pds info on searchRepos * fix build * port admin tests to bsky package * clean up old snaps * tests on fanout * tweak naming * missed a rename * tidy renames * fix lex name * tidy & move snap * fix build * cleanup repeat process all * skip actor search test * fix bday paradox * tidy up pds service auth * rm skipped test * retry http * tidy * improve fanout error handling * fix test * return signing key in did-web * more tests * tidy serivce auth checks * change takedownId col to takedownRef * build branch * fix bsky test * add service key to indexer * move signing key to api entry * dont build --- lexicons/com/atproto/admin/defs.json | 40 + .../com/atproto/admin/getAccountInfo.json | 24 + .../com/atproto/admin/getSubjectStatus.json | 39 + lexicons/com/atproto/admin/searchRepos.json | 1 - .../atproto/admin/updateSubjectStatus.json | 52 + packages/api/src/client/index.ts | 39 + packages/api/src/client/lexicons.ts | 203 +++- .../client/types/com/atproto/admin/defs.ts | 61 ++ .../types/com/atproto/admin/getAccountInfo.ts | 32 + .../com/atproto/admin/getSubjectStatus.ts | 44 + .../types/com/atproto/admin/searchRepos.ts | 1 - .../com/atproto/admin/updateSubjectStatus.ts | 50 + .../com/atproto/admin/getModerationAction.ts | 30 +- .../com/atproto/admin/getModerationReport.ts | 29 +- .../src/api/com/atproto/admin/getRecord.ts | 16 +- .../bsky/src/api/com/atproto/admin/getRepo.ts | 15 +- .../atproto/admin/reverseModerationAction.ts | 36 +- .../src/api/com/atproto/admin/searchRepos.ts | 7 +- .../com/atproto/admin/takeModerationAction.ts | 48 +- .../bsky/src/api/com/atproto/admin/util.ts | 50 + packages/bsky/src/api/well-known.ts | 8 + packages/bsky/src/auth.ts | 4 +- packages/bsky/src/auto-moderator/index.ts | 1 + packages/bsky/src/context.ts | 26 +- packages/bsky/src/index.ts | 5 +- packages/bsky/src/lexicon/index.ts | 36 + packages/bsky/src/lexicon/lexicons.ts | 203 +++- .../lexicon/types/com/atproto/admin/defs.ts | 61 ++ .../types/com/atproto/admin/getAccountInfo.ts | 41 + .../com/atproto/admin/getSubjectStatus.ts | 54 + .../types/com/atproto/admin/searchRepos.ts | 1 - .../com/atproto/admin/updateSubjectStatus.ts | 61 ++ .../bsky/src/services/moderation/index.ts | 108 +- .../bsky/src/services/moderation/views.ts | 1 + .../get-moderation-action.test.ts.snap | 16 +- .../get-moderation-actions.test.ts.snap | 42 +- .../get-moderation-report.test.ts.snap | 12 +- .../get-moderation-reports.test.ts.snap | 40 +- .../__snapshots__/get-record.test.ts.snap | 36 +- .../admin/__snapshots__/get-repo.test.ts.snap | 9 +- .../__snapshots__/moderation.test.ts.snap | 0 .../tests/admin/get-moderation-action.test.ts | 14 +- .../admin/get-moderation-actions.test.ts | 10 +- .../tests/admin/get-moderation-report.test.ts | 8 +- .../admin/get-moderation-reports.test.ts | 8 +- .../tests/admin/get-record.test.ts | 8 +- .../tests/admin/get-repo.test.ts | 8 +- .../bsky/tests/{ => admin}/moderation.test.ts | 110 +- .../tests/admin/repo-search.test.ts | 21 +- .../tests/auto-moderator/takedowns.test.ts | 8 +- .../bsky/tests/views/actor-search.test.ts | 4 +- packages/bsky/tests/views/posts.test.ts | 2 - packages/dev-env/src/bsky.ts | 8 +- packages/dev-env/src/const.ts | 3 + packages/dev-env/src/pds.ts | 5 +- packages/identity/src/did/atproto-data.ts | 8 + packages/identity/src/did/base-resolver.ts | 4 +- .../api/com/atproto/admin/getAccountInfo.ts | 22 + .../com/atproto/admin/getModerationAction.ts | 51 +- .../com/atproto/admin/getModerationActions.ts | 30 +- .../com/atproto/admin/getModerationReport.ts | 51 +- .../com/atproto/admin/getModerationReports.ts | 46 +- .../src/api/com/atproto/admin/getRecord.ts | 54 +- .../pds/src/api/com/atproto/admin/getRepo.ts | 50 +- .../api/com/atproto/admin/getSubjectStatus.ts | 43 + .../pds/src/api/com/atproto/admin/index.ts | 6 + .../atproto/admin/resolveModerationReports.ts | 29 +- .../atproto/admin/reverseModerationAction.ts | 105 +- .../src/api/com/atproto/admin/searchRepos.ts | 62 +- .../com/atproto/admin/takeModerationAction.ts | 151 +-- .../com/atproto/admin/updateSubjectStatus.ts | 59 ++ .../com/atproto/moderation/createReport.ts | 34 +- .../pds/src/api/com/atproto/repo/getRecord.ts | 2 +- .../api/com/atproto/server/createAccount.ts | 2 +- .../api/com/atproto/server/deleteAccount.ts | 30 +- packages/pds/src/auth-verifier.ts | 78 +- packages/pds/src/context.ts | 3 +- .../20231011T155513453Z-takedown-ref.ts | 41 + packages/pds/src/db/migrations/index.ts | 1 + .../db/periodic-moderation-action-reversal.ts | 88 -- packages/pds/src/db/tables/record.ts | 2 +- packages/pds/src/db/tables/repo-blob.ts | 2 +- packages/pds/src/db/tables/repo-root.ts | 2 +- packages/pds/src/db/util.ts | 6 +- packages/pds/src/index.ts | 1 - packages/pds/src/lexicon/index.ts | 36 + packages/pds/src/lexicon/lexicons.ts | 203 +++- .../lexicon/types/com/atproto/admin/defs.ts | 61 ++ .../types/com/atproto/admin/getAccountInfo.ts | 41 + .../com/atproto/admin/getSubjectStatus.ts | 54 + .../types/com/atproto/admin/searchRepos.ts | 1 - .../com/atproto/admin/updateSubjectStatus.ts | 61 ++ packages/pds/src/services/account/index.ts | 36 +- packages/pds/src/services/moderation/index.ts | 680 ++---------- packages/pds/src/services/moderation/views.ts | 633 ----------- packages/pds/src/services/record/index.ts | 4 +- packages/pds/src/services/repo/blobs.ts | 2 +- packages/pds/tests/account-deletion.test.ts | 9 +- .../__snapshots__/moderation.test.ts.snap | 193 ---- packages/pds/tests/admin/moderation.test.ts | 999 ------------------ packages/pds/tests/auth.test.ts | 13 +- packages/pds/tests/crud.test.ts | 78 +- packages/pds/tests/db.test.ts | 2 +- packages/pds/tests/invite-codes.test.ts | 38 +- .../invites.test.ts => invites-admin.test.ts} | 24 +- packages/pds/tests/moderation.test.ts | 357 +++++++ packages/pds/tests/proxied/admin.test.ts | 5 +- packages/pds/tests/proxied/feedgen.test.ts | 2 +- packages/pds/tests/proxied/notif.test.ts | 2 +- packages/pds/tests/proxied/procedures.test.ts | 2 +- .../tests/proxied/read-after-write.test.ts | 2 +- packages/pds/tests/proxied/views.test.ts | 2 +- packages/pds/tests/seeds/basic.ts | 39 +- packages/pds/tests/seeds/users.ts | 16 +- packages/pds/tests/sync/sync.test.ts | 27 +- packages/xrpc-server/src/auth.ts | 9 +- services/bsky/api.js | 5 + 117 files changed, 3025 insertions(+), 3473 deletions(-) create mode 100644 lexicons/com/atproto/admin/getAccountInfo.json create mode 100644 lexicons/com/atproto/admin/getSubjectStatus.json create mode 100644 lexicons/com/atproto/admin/updateSubjectStatus.json create mode 100644 packages/api/src/client/types/com/atproto/admin/getAccountInfo.ts create mode 100644 packages/api/src/client/types/com/atproto/admin/getSubjectStatus.ts create mode 100644 packages/api/src/client/types/com/atproto/admin/updateSubjectStatus.ts create mode 100644 packages/bsky/src/api/com/atproto/admin/util.ts create mode 100644 packages/bsky/src/lexicon/types/com/atproto/admin/getAccountInfo.ts create mode 100644 packages/bsky/src/lexicon/types/com/atproto/admin/getSubjectStatus.ts create mode 100644 packages/bsky/src/lexicon/types/com/atproto/admin/updateSubjectStatus.ts rename packages/{pds => bsky}/tests/admin/__snapshots__/get-moderation-action.test.ts.snap (94%) rename packages/{pds => bsky}/tests/admin/__snapshots__/get-moderation-actions.test.ts.snap (85%) rename packages/{pds => bsky}/tests/admin/__snapshots__/get-moderation-report.test.ts.snap (95%) rename packages/{pds => bsky}/tests/admin/__snapshots__/get-moderation-reports.test.ts.snap (88%) rename packages/{pds => bsky}/tests/admin/__snapshots__/get-record.test.ts.snap (91%) rename packages/{pds => bsky}/tests/admin/__snapshots__/get-repo.test.ts.snap (95%) rename packages/bsky/tests/{ => admin}/__snapshots__/moderation.test.ts.snap (100%) rename packages/{pds => bsky}/tests/admin/get-moderation-action.test.ts (89%) rename packages/{pds => bsky}/tests/admin/get-moderation-actions.test.ts (94%) rename packages/{pds => bsky}/tests/admin/get-moderation-report.test.ts (92%) rename packages/{pds => bsky}/tests/admin/get-moderation-reports.test.ts (98%) rename packages/{pds => bsky}/tests/admin/get-record.test.ts (94%) rename packages/{pds => bsky}/tests/admin/get-repo.test.ts (93%) rename packages/bsky/tests/{ => admin}/moderation.test.ts (91%) rename packages/{pds => bsky}/tests/admin/repo-search.test.ts (84%) create mode 100644 packages/dev-env/src/const.ts create mode 100644 packages/pds/src/api/com/atproto/admin/getAccountInfo.ts create mode 100644 packages/pds/src/api/com/atproto/admin/getSubjectStatus.ts create mode 100644 packages/pds/src/api/com/atproto/admin/updateSubjectStatus.ts create mode 100644 packages/pds/src/db/migrations/20231011T155513453Z-takedown-ref.ts delete mode 100644 packages/pds/src/db/periodic-moderation-action-reversal.ts create mode 100644 packages/pds/src/lexicon/types/com/atproto/admin/getAccountInfo.ts create mode 100644 packages/pds/src/lexicon/types/com/atproto/admin/getSubjectStatus.ts create mode 100644 packages/pds/src/lexicon/types/com/atproto/admin/updateSubjectStatus.ts delete mode 100644 packages/pds/src/services/moderation/views.ts delete mode 100644 packages/pds/tests/admin/__snapshots__/moderation.test.ts.snap delete mode 100644 packages/pds/tests/admin/moderation.test.ts rename packages/pds/tests/{admin/invites.test.ts => invites-admin.test.ts} (91%) create mode 100644 packages/pds/tests/moderation.test.ts diff --git a/lexicons/com/atproto/admin/defs.json b/lexicons/com/atproto/admin/defs.json index a04c77d68f8..318d1c33b5a 100644 --- a/lexicons/com/atproto/admin/defs.json +++ b/lexicons/com/atproto/admin/defs.json @@ -2,6 +2,14 @@ "lexicon": 1, "id": "com.atproto.admin.defs", "defs": { + "statusAttr": { + "type": "object", + "required": ["applied"], + "properties": { + "applied": { "type": "boolean" }, + "ref": { "type": "string" } + } + }, "actionView": { "type": "object", "required": [ @@ -243,6 +251,29 @@ "inviteNote": { "type": "string" } } }, + "accountView": { + "type": "object", + "required": ["did", "handle", "indexedAt"], + "properties": { + "did": { "type": "string", "format": "did" }, + "handle": { "type": "string", "format": "handle" }, + "email": { "type": "string" }, + "indexedAt": { "type": "string", "format": "datetime" }, + "invitedBy": { + "type": "ref", + "ref": "com.atproto.server.defs#inviteCode" + }, + "invites": { + "type": "array", + "items": { + "type": "ref", + "ref": "com.atproto.server.defs#inviteCode" + } + }, + "invitesDisabled": { "type": "boolean" }, + "inviteNote": { "type": "string" } + } + }, "repoViewNotFound": { "type": "object", "required": ["did"], @@ -257,6 +288,15 @@ "did": { "type": "string", "format": "did" } } }, + "repoBlobRef": { + "type": "object", + "required": ["did", "cid"], + "properties": { + "did": { "type": "string", "format": "did" }, + "cid": { "type": "string", "format": "cid" }, + "recordUri": { "type": "string", "format": "at-uri" } + } + }, "recordView": { "type": "object", "required": [ diff --git a/lexicons/com/atproto/admin/getAccountInfo.json b/lexicons/com/atproto/admin/getAccountInfo.json new file mode 100644 index 00000000000..da8e839fdfa --- /dev/null +++ b/lexicons/com/atproto/admin/getAccountInfo.json @@ -0,0 +1,24 @@ +{ + "lexicon": 1, + "id": "com.atproto.admin.getAccountInfo", + "defs": { + "main": { + "type": "query", + "description": "View details about an account.", + "parameters": { + "type": "params", + "required": ["did"], + "properties": { + "did": { "type": "string", "format": "did" } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "ref", + "ref": "com.atproto.admin.defs#accountView" + } + } + } + } +} diff --git a/lexicons/com/atproto/admin/getSubjectStatus.json b/lexicons/com/atproto/admin/getSubjectStatus.json new file mode 100644 index 00000000000..a6ce340c009 --- /dev/null +++ b/lexicons/com/atproto/admin/getSubjectStatus.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "com.atproto.admin.getSubjectStatus", + "defs": { + "main": { + "type": "query", + "description": "Fetch the service-specific the admin status of a subject (account, record, or blob)", + "parameters": { + "type": "params", + "properties": { + "did": { "type": "string", "format": "did" }, + "uri": { "type": "string", "format": "at-uri" }, + "blob": { "type": "string", "format": "cid" } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["subject"], + "properties": { + "subject": { + "type": "union", + "refs": [ + "com.atproto.admin.defs#repoRef", + "com.atproto.repo.strongRef", + "com.atproto.admin.defs#repoBlobRef" + ] + }, + "takedown": { + "type": "ref", + "ref": "com.atproto.admin.defs#statusAttr" + } + } + } + } + } + } +} diff --git a/lexicons/com/atproto/admin/searchRepos.json b/lexicons/com/atproto/admin/searchRepos.json index 85cc6fd482a..acc5a70f942 100644 --- a/lexicons/com/atproto/admin/searchRepos.json +++ b/lexicons/com/atproto/admin/searchRepos.json @@ -13,7 +13,6 @@ "description": "DEPRECATED: use 'q' instead" }, "q": { "type": "string" }, - "invitedBy": { "type": "string" }, "limit": { "type": "integer", "minimum": 1, diff --git a/lexicons/com/atproto/admin/updateSubjectStatus.json b/lexicons/com/atproto/admin/updateSubjectStatus.json new file mode 100644 index 00000000000..5273aea4da6 --- /dev/null +++ b/lexicons/com/atproto/admin/updateSubjectStatus.json @@ -0,0 +1,52 @@ +{ + "lexicon": 1, + "id": "com.atproto.admin.updateSubjectStatus", + "defs": { + "main": { + "type": "procedure", + "description": "Update the service-specific admin status of a subject (account, record, or blob)", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["subject"], + "properties": { + "subject": { + "type": "union", + "refs": [ + "com.atproto.admin.defs#repoRef", + "com.atproto.repo.strongRef", + "com.atproto.admin.defs#repoBlobRef" + ] + }, + "takedown": { + "type": "ref", + "ref": "com.atproto.admin.defs#statusAttr" + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["subject"], + "properties": { + "subject": { + "type": "union", + "refs": [ + "com.atproto.admin.defs#repoRef", + "com.atproto.repo.strongRef", + "com.atproto.admin.defs#repoBlobRef" + ] + }, + "takedown": { + "type": "ref", + "ref": "com.atproto.admin.defs#statusAttr" + } + } + } + } + } + } +} diff --git a/packages/api/src/client/index.ts b/packages/api/src/client/index.ts index 15720ad52f8..3fd82222639 100644 --- a/packages/api/src/client/index.ts +++ b/packages/api/src/client/index.ts @@ -11,6 +11,7 @@ import * as ComAtprotoAdminDefs from './types/com/atproto/admin/defs' import * as ComAtprotoAdminDisableAccountInvites from './types/com/atproto/admin/disableAccountInvites' import * as ComAtprotoAdminDisableInviteCodes from './types/com/atproto/admin/disableInviteCodes' import * as ComAtprotoAdminEnableAccountInvites from './types/com/atproto/admin/enableAccountInvites' +import * as ComAtprotoAdminGetAccountInfo from './types/com/atproto/admin/getAccountInfo' import * as ComAtprotoAdminGetInviteCodes from './types/com/atproto/admin/getInviteCodes' import * as ComAtprotoAdminGetModerationAction from './types/com/atproto/admin/getModerationAction' import * as ComAtprotoAdminGetModerationActions from './types/com/atproto/admin/getModerationActions' @@ -18,6 +19,7 @@ import * as ComAtprotoAdminGetModerationReport from './types/com/atproto/admin/g import * as ComAtprotoAdminGetModerationReports from './types/com/atproto/admin/getModerationReports' import * as ComAtprotoAdminGetRecord from './types/com/atproto/admin/getRecord' import * as ComAtprotoAdminGetRepo from './types/com/atproto/admin/getRepo' +import * as ComAtprotoAdminGetSubjectStatus from './types/com/atproto/admin/getSubjectStatus' import * as ComAtprotoAdminResolveModerationReports from './types/com/atproto/admin/resolveModerationReports' import * as ComAtprotoAdminReverseModerationAction from './types/com/atproto/admin/reverseModerationAction' import * as ComAtprotoAdminSearchRepos from './types/com/atproto/admin/searchRepos' @@ -25,6 +27,7 @@ import * as ComAtprotoAdminSendEmail from './types/com/atproto/admin/sendEmail' import * as ComAtprotoAdminTakeModerationAction from './types/com/atproto/admin/takeModerationAction' import * as ComAtprotoAdminUpdateAccountEmail from './types/com/atproto/admin/updateAccountEmail' import * as ComAtprotoAdminUpdateAccountHandle from './types/com/atproto/admin/updateAccountHandle' +import * as ComAtprotoAdminUpdateSubjectStatus from './types/com/atproto/admin/updateSubjectStatus' import * as ComAtprotoIdentityResolveHandle from './types/com/atproto/identity/resolveHandle' import * as ComAtprotoIdentityUpdateHandle from './types/com/atproto/identity/updateHandle' import * as ComAtprotoLabelDefs from './types/com/atproto/label/defs' @@ -145,6 +148,7 @@ export * as ComAtprotoAdminDefs from './types/com/atproto/admin/defs' export * as ComAtprotoAdminDisableAccountInvites from './types/com/atproto/admin/disableAccountInvites' export * as ComAtprotoAdminDisableInviteCodes from './types/com/atproto/admin/disableInviteCodes' export * as ComAtprotoAdminEnableAccountInvites from './types/com/atproto/admin/enableAccountInvites' +export * as ComAtprotoAdminGetAccountInfo from './types/com/atproto/admin/getAccountInfo' export * as ComAtprotoAdminGetInviteCodes from './types/com/atproto/admin/getInviteCodes' export * as ComAtprotoAdminGetModerationAction from './types/com/atproto/admin/getModerationAction' export * as ComAtprotoAdminGetModerationActions from './types/com/atproto/admin/getModerationActions' @@ -152,6 +156,7 @@ export * as ComAtprotoAdminGetModerationReport from './types/com/atproto/admin/g export * as ComAtprotoAdminGetModerationReports from './types/com/atproto/admin/getModerationReports' export * as ComAtprotoAdminGetRecord from './types/com/atproto/admin/getRecord' export * as ComAtprotoAdminGetRepo from './types/com/atproto/admin/getRepo' +export * as ComAtprotoAdminGetSubjectStatus from './types/com/atproto/admin/getSubjectStatus' export * as ComAtprotoAdminResolveModerationReports from './types/com/atproto/admin/resolveModerationReports' export * as ComAtprotoAdminReverseModerationAction from './types/com/atproto/admin/reverseModerationAction' export * as ComAtprotoAdminSearchRepos from './types/com/atproto/admin/searchRepos' @@ -159,6 +164,7 @@ export * as ComAtprotoAdminSendEmail from './types/com/atproto/admin/sendEmail' export * as ComAtprotoAdminTakeModerationAction from './types/com/atproto/admin/takeModerationAction' export * as ComAtprotoAdminUpdateAccountEmail from './types/com/atproto/admin/updateAccountEmail' export * as ComAtprotoAdminUpdateAccountHandle from './types/com/atproto/admin/updateAccountHandle' +export * as ComAtprotoAdminUpdateSubjectStatus from './types/com/atproto/admin/updateSubjectStatus' export * as ComAtprotoIdentityResolveHandle from './types/com/atproto/identity/resolveHandle' export * as ComAtprotoIdentityUpdateHandle from './types/com/atproto/identity/updateHandle' export * as ComAtprotoLabelDefs from './types/com/atproto/label/defs' @@ -396,6 +402,17 @@ export class AdminNS { }) } + getAccountInfo( + params?: ComAtprotoAdminGetAccountInfo.QueryParams, + opts?: ComAtprotoAdminGetAccountInfo.CallOptions, + ): Promise { + return this._service.xrpc + .call('com.atproto.admin.getAccountInfo', params, undefined, opts) + .catch((e) => { + throw ComAtprotoAdminGetAccountInfo.toKnownErr(e) + }) + } + getInviteCodes( params?: ComAtprotoAdminGetInviteCodes.QueryParams, opts?: ComAtprotoAdminGetInviteCodes.CallOptions, @@ -473,6 +490,17 @@ export class AdminNS { }) } + getSubjectStatus( + params?: ComAtprotoAdminGetSubjectStatus.QueryParams, + opts?: ComAtprotoAdminGetSubjectStatus.CallOptions, + ): Promise { + return this._service.xrpc + .call('com.atproto.admin.getSubjectStatus', params, undefined, opts) + .catch((e) => { + throw ComAtprotoAdminGetSubjectStatus.toKnownErr(e) + }) + } + resolveModerationReports( data?: ComAtprotoAdminResolveModerationReports.InputSchema, opts?: ComAtprotoAdminResolveModerationReports.CallOptions, @@ -549,6 +577,17 @@ export class AdminNS { throw ComAtprotoAdminUpdateAccountHandle.toKnownErr(e) }) } + + updateSubjectStatus( + data?: ComAtprotoAdminUpdateSubjectStatus.InputSchema, + opts?: ComAtprotoAdminUpdateSubjectStatus.CallOptions, + ): Promise { + return this._service.xrpc + .call('com.atproto.admin.updateSubjectStatus', opts?.qp, data, opts) + .catch((e) => { + throw ComAtprotoAdminUpdateSubjectStatus.toKnownErr(e) + }) + } } export class IdentityNS { diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index 3911dc12497..df696e5d06b 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -8,6 +8,18 @@ export const schemaDict = { lexicon: 1, id: 'com.atproto.admin.defs', defs: { + statusAttr: { + type: 'object', + required: ['applied'], + properties: { + applied: { + type: 'boolean', + }, + ref: { + type: 'string', + }, + }, + }, actionView: { type: 'object', required: [ @@ -424,6 +436,44 @@ export const schemaDict = { }, }, }, + accountView: { + type: 'object', + required: ['did', 'handle', 'indexedAt'], + properties: { + did: { + type: 'string', + format: 'did', + }, + handle: { + type: 'string', + format: 'handle', + }, + email: { + type: 'string', + }, + indexedAt: { + type: 'string', + format: 'datetime', + }, + invitedBy: { + type: 'ref', + ref: 'lex:com.atproto.server.defs#inviteCode', + }, + invites: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.server.defs#inviteCode', + }, + }, + invitesDisabled: { + type: 'boolean', + }, + inviteNote: { + type: 'string', + }, + }, + }, repoViewNotFound: { type: 'object', required: ['did'], @@ -444,6 +494,24 @@ export const schemaDict = { }, }, }, + repoBlobRef: { + type: 'object', + required: ['did', 'cid'], + properties: { + did: { + type: 'string', + format: 'did', + }, + cid: { + type: 'string', + format: 'cid', + }, + recordUri: { + type: 'string', + format: 'at-uri', + }, + }, + }, recordView: { type: 'object', required: [ @@ -730,6 +798,33 @@ export const schemaDict = { }, }, }, + ComAtprotoAdminGetAccountInfo: { + lexicon: 1, + id: 'com.atproto.admin.getAccountInfo', + defs: { + main: { + type: 'query', + description: 'View details about an account.', + parameters: { + type: 'params', + required: ['did'], + properties: { + did: { + type: 'string', + format: 'did', + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#accountView', + }, + }, + }, + }, + }, ComAtprotoAdminGetInviteCodes: { lexicon: 1, id: 'com.atproto.admin.getInviteCodes', @@ -1026,6 +1121,55 @@ export const schemaDict = { }, }, }, + ComAtprotoAdminGetSubjectStatus: { + lexicon: 1, + id: 'com.atproto.admin.getSubjectStatus', + defs: { + main: { + type: 'query', + description: + 'Fetch the service-specific the admin status of a subject (account, record, or blob)', + parameters: { + type: 'params', + properties: { + did: { + type: 'string', + format: 'did', + }, + uri: { + type: 'string', + format: 'at-uri', + }, + blob: { + type: 'string', + format: 'cid', + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['subject'], + properties: { + subject: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#repoRef', + 'lex:com.atproto.repo.strongRef', + 'lex:com.atproto.admin.defs#repoBlobRef', + ], + }, + takedown: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#statusAttr', + }, + }, + }, + }, + }, + }, + }, ComAtprotoAdminResolveModerationReports: { lexicon: 1, id: 'com.atproto.admin.resolveModerationReports', @@ -1118,9 +1262,6 @@ export const schemaDict = { q: { type: 'string', }, - invitedBy: { - type: 'string', - }, limit: { type: 'integer', minimum: 1, @@ -1326,6 +1467,59 @@ export const schemaDict = { }, }, }, + ComAtprotoAdminUpdateSubjectStatus: { + lexicon: 1, + id: 'com.atproto.admin.updateSubjectStatus', + defs: { + main: { + type: 'procedure', + description: + 'Update the service-specific admin status of a subject (account, record, or blob)', + input: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['subject'], + properties: { + subject: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#repoRef', + 'lex:com.atproto.repo.strongRef', + 'lex:com.atproto.admin.defs#repoBlobRef', + ], + }, + takedown: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#statusAttr', + }, + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['subject'], + properties: { + subject: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#repoRef', + 'lex:com.atproto.repo.strongRef', + 'lex:com.atproto.admin.defs#repoBlobRef', + ], + }, + takedown: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#statusAttr', + }, + }, + }, + }, + }, + }, + }, ComAtprotoIdentityResolveHandle: { lexicon: 1, id: 'com.atproto.identity.resolveHandle', @@ -7369,6 +7563,7 @@ export const ids = { 'com.atproto.admin.disableAccountInvites', ComAtprotoAdminDisableInviteCodes: 'com.atproto.admin.disableInviteCodes', ComAtprotoAdminEnableAccountInvites: 'com.atproto.admin.enableAccountInvites', + ComAtprotoAdminGetAccountInfo: 'com.atproto.admin.getAccountInfo', ComAtprotoAdminGetInviteCodes: 'com.atproto.admin.getInviteCodes', ComAtprotoAdminGetModerationAction: 'com.atproto.admin.getModerationAction', ComAtprotoAdminGetModerationActions: 'com.atproto.admin.getModerationActions', @@ -7376,6 +7571,7 @@ export const ids = { ComAtprotoAdminGetModerationReports: 'com.atproto.admin.getModerationReports', ComAtprotoAdminGetRecord: 'com.atproto.admin.getRecord', ComAtprotoAdminGetRepo: 'com.atproto.admin.getRepo', + ComAtprotoAdminGetSubjectStatus: 'com.atproto.admin.getSubjectStatus', ComAtprotoAdminResolveModerationReports: 'com.atproto.admin.resolveModerationReports', ComAtprotoAdminReverseModerationAction: @@ -7385,6 +7581,7 @@ export const ids = { ComAtprotoAdminTakeModerationAction: 'com.atproto.admin.takeModerationAction', ComAtprotoAdminUpdateAccountEmail: 'com.atproto.admin.updateAccountEmail', ComAtprotoAdminUpdateAccountHandle: 'com.atproto.admin.updateAccountHandle', + ComAtprotoAdminUpdateSubjectStatus: 'com.atproto.admin.updateSubjectStatus', ComAtprotoIdentityResolveHandle: 'com.atproto.identity.resolveHandle', ComAtprotoIdentityUpdateHandle: 'com.atproto.identity.updateHandle', ComAtprotoLabelDefs: 'com.atproto.label.defs', diff --git a/packages/api/src/client/types/com/atproto/admin/defs.ts b/packages/api/src/client/types/com/atproto/admin/defs.ts index f98814ca8e2..7c48fa87a3c 100644 --- a/packages/api/src/client/types/com/atproto/admin/defs.ts +++ b/packages/api/src/client/types/com/atproto/admin/defs.ts @@ -10,6 +10,24 @@ import * as ComAtprotoModerationDefs from '../moderation/defs' import * as ComAtprotoServerDefs from '../server/defs' import * as ComAtprotoLabelDefs from '../label/defs' +export interface StatusAttr { + applied: boolean + ref?: string + [k: string]: unknown +} + +export function isStatusAttr(v: unknown): v is StatusAttr { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#statusAttr' + ) +} + +export function validateStatusAttr(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#statusAttr', v) +} + export interface ActionView { id: number action: ActionType @@ -238,6 +256,30 @@ export function validateRepoViewDetail(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#repoViewDetail', v) } +export interface AccountView { + did: string + handle: string + email?: string + indexedAt: string + invitedBy?: ComAtprotoServerDefs.InviteCode + invites?: ComAtprotoServerDefs.InviteCode[] + invitesDisabled?: boolean + inviteNote?: string + [k: string]: unknown +} + +export function isAccountView(v: unknown): v is AccountView { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#accountView' + ) +} + +export function validateAccountView(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#accountView', v) +} + export interface RepoViewNotFound { did: string [k: string]: unknown @@ -272,6 +314,25 @@ export function validateRepoRef(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#repoRef', v) } +export interface RepoBlobRef { + did: string + cid: string + recordUri?: string + [k: string]: unknown +} + +export function isRepoBlobRef(v: unknown): v is RepoBlobRef { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#repoBlobRef' + ) +} + +export function validateRepoBlobRef(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#repoBlobRef', v) +} + export interface RecordView { uri: string cid: string diff --git a/packages/api/src/client/types/com/atproto/admin/getAccountInfo.ts b/packages/api/src/client/types/com/atproto/admin/getAccountInfo.ts new file mode 100644 index 00000000000..a6d2b97bb63 --- /dev/null +++ b/packages/api/src/client/types/com/atproto/admin/getAccountInfo.ts @@ -0,0 +1,32 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import { Headers, XRPCError } from '@atproto/xrpc' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { isObj, hasProp } from '../../../../util' +import { lexicons } from '../../../../lexicons' +import { CID } from 'multiformats/cid' +import * as ComAtprotoAdminDefs from './defs' + +export interface QueryParams { + did: string +} + +export type InputSchema = undefined +export type OutputSchema = ComAtprotoAdminDefs.AccountView + +export interface CallOptions { + headers?: Headers +} + +export interface Response { + success: boolean + headers: Headers + data: OutputSchema +} + +export function toKnownErr(e: any) { + if (e instanceof XRPCError) { + } + return e +} diff --git a/packages/api/src/client/types/com/atproto/admin/getSubjectStatus.ts b/packages/api/src/client/types/com/atproto/admin/getSubjectStatus.ts new file mode 100644 index 00000000000..26986e5dde7 --- /dev/null +++ b/packages/api/src/client/types/com/atproto/admin/getSubjectStatus.ts @@ -0,0 +1,44 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import { Headers, XRPCError } from '@atproto/xrpc' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { isObj, hasProp } from '../../../../util' +import { lexicons } from '../../../../lexicons' +import { CID } from 'multiformats/cid' +import * as ComAtprotoAdminDefs from './defs' +import * as ComAtprotoRepoStrongRef from '../repo/strongRef' + +export interface QueryParams { + did?: string + uri?: string + blob?: string +} + +export type InputSchema = undefined + +export interface OutputSchema { + subject: + | ComAtprotoAdminDefs.RepoRef + | ComAtprotoRepoStrongRef.Main + | ComAtprotoAdminDefs.RepoBlobRef + | { $type: string; [k: string]: unknown } + takedown?: ComAtprotoAdminDefs.StatusAttr + [k: string]: unknown +} + +export interface CallOptions { + headers?: Headers +} + +export interface Response { + success: boolean + headers: Headers + data: OutputSchema +} + +export function toKnownErr(e: any) { + if (e instanceof XRPCError) { + } + return e +} diff --git a/packages/api/src/client/types/com/atproto/admin/searchRepos.ts b/packages/api/src/client/types/com/atproto/admin/searchRepos.ts index 372cc98ff13..451077479b9 100644 --- a/packages/api/src/client/types/com/atproto/admin/searchRepos.ts +++ b/packages/api/src/client/types/com/atproto/admin/searchRepos.ts @@ -12,7 +12,6 @@ export interface QueryParams { /** DEPRECATED: use 'q' instead */ term?: string q?: string - invitedBy?: string limit?: number cursor?: string } diff --git a/packages/api/src/client/types/com/atproto/admin/updateSubjectStatus.ts b/packages/api/src/client/types/com/atproto/admin/updateSubjectStatus.ts new file mode 100644 index 00000000000..c7e17b50582 --- /dev/null +++ b/packages/api/src/client/types/com/atproto/admin/updateSubjectStatus.ts @@ -0,0 +1,50 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import { Headers, XRPCError } from '@atproto/xrpc' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { isObj, hasProp } from '../../../../util' +import { lexicons } from '../../../../lexicons' +import { CID } from 'multiformats/cid' +import * as ComAtprotoAdminDefs from './defs' +import * as ComAtprotoRepoStrongRef from '../repo/strongRef' + +export interface QueryParams {} + +export interface InputSchema { + subject: + | ComAtprotoAdminDefs.RepoRef + | ComAtprotoRepoStrongRef.Main + | ComAtprotoAdminDefs.RepoBlobRef + | { $type: string; [k: string]: unknown } + takedown?: ComAtprotoAdminDefs.StatusAttr + [k: string]: unknown +} + +export interface OutputSchema { + subject: + | ComAtprotoAdminDefs.RepoRef + | ComAtprotoRepoStrongRef.Main + | ComAtprotoAdminDefs.RepoBlobRef + | { $type: string; [k: string]: unknown } + takedown?: ComAtprotoAdminDefs.StatusAttr + [k: string]: unknown +} + +export interface CallOptions { + headers?: Headers + qp?: QueryParams + encoding: 'application/json' +} + +export interface Response { + success: boolean + headers: Headers + data: OutputSchema +} + +export function toKnownErr(e: any) { + if (e instanceof XRPCError) { + } + return e +} diff --git a/packages/bsky/src/api/com/atproto/admin/getModerationAction.ts b/packages/bsky/src/api/com/atproto/admin/getModerationAction.ts index 55ff9b9ccf8..51218077bcf 100644 --- a/packages/bsky/src/api/com/atproto/admin/getModerationAction.ts +++ b/packages/bsky/src/api/com/atproto/admin/getModerationAction.ts @@ -1,17 +1,43 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' +import { addAccountInfoToRepoView, getPdsAccountInfo } from './util' +import { + isRecordView, + isRepoView, +} from '../../../../lexicon/types/com/atproto/admin/defs' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.getModerationAction({ auth: ctx.roleVerifier, - handler: async ({ params }) => { + handler: async ({ params, auth }) => { const { id } = params const db = ctx.db.getPrimary() const moderationService = ctx.services.moderation(db) const result = await moderationService.getActionOrThrow(id) + + const [action, accountInfo] = await Promise.all([ + moderationService.views.actionDetail(result), + getPdsAccountInfo(ctx, result.subjectDid), + ]) + + // add in pds account info if available + if (isRepoView(action.subject)) { + action.subject = addAccountInfoToRepoView( + action.subject, + accountInfo, + auth.credentials.moderator, + ) + } else if (isRecordView(action.subject)) { + action.subject.repo = addAccountInfoToRepoView( + action.subject.repo, + accountInfo, + auth.credentials.moderator, + ) + } + return { encoding: 'application/json', - body: await moderationService.views.actionDetail(result), + body: action, } }, }) diff --git a/packages/bsky/src/api/com/atproto/admin/getModerationReport.ts b/packages/bsky/src/api/com/atproto/admin/getModerationReport.ts index e3faaa04436..814d1069e3f 100644 --- a/packages/bsky/src/api/com/atproto/admin/getModerationReport.ts +++ b/packages/bsky/src/api/com/atproto/admin/getModerationReport.ts @@ -1,17 +1,42 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' +import { + isRecordView, + isRepoView, +} from '../../../../lexicon/types/com/atproto/admin/defs' +import { addAccountInfoToRepoView, getPdsAccountInfo } from './util' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.getModerationReport({ auth: ctx.roleVerifier, - handler: async ({ params }) => { + handler: async ({ params, auth }) => { const { id } = params const db = ctx.db.getPrimary() const moderationService = ctx.services.moderation(db) const result = await moderationService.getReportOrThrow(id) + const [report, accountInfo] = await Promise.all([ + moderationService.views.reportDetail(result), + getPdsAccountInfo(ctx, result.subjectDid), + ]) + + // add in pds account info if available + if (isRepoView(report.subject)) { + report.subject = addAccountInfoToRepoView( + report.subject, + accountInfo, + auth.credentials.moderator, + ) + } else if (isRecordView(report.subject)) { + report.subject.repo = addAccountInfoToRepoView( + report.subject.repo, + accountInfo, + auth.credentials.moderator, + ) + } + return { encoding: 'application/json', - body: await moderationService.views.reportDetail(result), + body: report, } }, }) diff --git a/packages/bsky/src/api/com/atproto/admin/getRecord.ts b/packages/bsky/src/api/com/atproto/admin/getRecord.ts index 80e79fd94a2..245ce2b8f26 100644 --- a/packages/bsky/src/api/com/atproto/admin/getRecord.ts +++ b/packages/bsky/src/api/com/atproto/admin/getRecord.ts @@ -1,11 +1,12 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' +import { addAccountInfoToRepoView, getPdsAccountInfo } from './util' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.getRecord({ auth: ctx.roleVerifier, - handler: async ({ params }) => { + handler: async ({ params, auth }) => { const { uri, cid } = params const db = ctx.db.getPrimary() const result = await db.db @@ -17,9 +18,20 @@ export default function (server: Server, ctx: AppContext) { if (!result) { throw new InvalidRequestError('Record not found', 'RecordNotFound') } + const [record, accountInfo] = await Promise.all([ + ctx.services.moderation(db).views.recordDetail(result), + getPdsAccountInfo(ctx, result.did), + ]) + + record.repo = addAccountInfoToRepoView( + record.repo, + accountInfo, + auth.credentials.moderator, + ) + return { encoding: 'application/json', - body: await ctx.services.moderation(db).views.recordDetail(result), + body: record, } }, }) diff --git a/packages/bsky/src/api/com/atproto/admin/getRepo.ts b/packages/bsky/src/api/com/atproto/admin/getRepo.ts index 5febdfcdd0c..314b345b5e9 100644 --- a/packages/bsky/src/api/com/atproto/admin/getRepo.ts +++ b/packages/bsky/src/api/com/atproto/admin/getRepo.ts @@ -1,20 +1,31 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' +import { addAccountInfoToRepoViewDetail, getPdsAccountInfo } from './util' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.getRepo({ auth: ctx.roleVerifier, - handler: async ({ params }) => { + handler: async ({ params, auth }) => { const { did } = params const db = ctx.db.getPrimary() const result = await ctx.services.actor(db).getActor(did, true) if (!result) { throw new InvalidRequestError('Repo not found', 'RepoNotFound') } + const [partialRepo, accountInfo] = await Promise.all([ + ctx.services.moderation(db).views.repoDetail(result), + getPdsAccountInfo(ctx, result.did), + ]) + + const repo = addAccountInfoToRepoViewDetail( + partialRepo, + accountInfo, + auth.credentials.moderator, + ) return { encoding: 'application/json', - body: await ctx.services.moderation(db).views.repoDetail(result), + body: repo, } }, }) diff --git a/packages/bsky/src/api/com/atproto/admin/reverseModerationAction.ts b/packages/bsky/src/api/com/atproto/admin/reverseModerationAction.ts index bd478285204..ae76df5b0c7 100644 --- a/packages/bsky/src/api/com/atproto/admin/reverseModerationAction.ts +++ b/packages/bsky/src/api/com/atproto/admin/reverseModerationAction.ts @@ -1,4 +1,8 @@ -import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' +import { + AuthRequiredError, + InvalidRequestError, + UpstreamFailureError, +} from '@atproto/xrpc-server' import { ACKNOWLEDGE, ESCALATE, @@ -6,6 +10,7 @@ import { } from '../../../../lexicon/types/com/atproto/admin/defs' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' +import { retryHttp } from '../../../../util/retry' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.reverseModerationAction({ @@ -16,7 +21,7 @@ export default function (server: Server, ctx: AppContext) { const moderationService = ctx.services.moderation(db) const { id, createdBy, reason } = input.body - const moderationAction = await db.transaction(async (dbTxn) => { + const { result, restored } = await db.transaction(async (dbTxn) => { const moderationTxn = ctx.services.moderation(dbTxn) const labelTxn = ctx.services.label(dbTxn) const now = new Date() @@ -53,7 +58,7 @@ export default function (server: Server, ctx: AppContext) { ) } - const result = await moderationTxn.revertAction({ + const { result, restored } = await moderationTxn.revertAction({ id, createdAt: now, createdBy, @@ -77,12 +82,33 @@ export default function (server: Server, ctx: AppContext) { { create, negate }, ) - return result + return { result, restored } }) + if (restored) { + const { did, subjects } = restored + const agent = await ctx.pdsAdminAgent(did) + const results = await Promise.allSettled( + subjects.map((subject) => + retryHttp(() => + agent.api.com.atproto.admin.updateSubjectStatus({ + subject, + takedown: { + applied: false, + }, + }), + ), + ), + ) + const hadFailure = results.some((r) => r.status === 'rejected') + if (hadFailure) { + throw new UpstreamFailureError('failed to revert action on PDS') + } + } + return { encoding: 'application/json', - body: await moderationService.views.action(moderationAction), + body: await moderationService.views.action(result), } }, }) diff --git a/packages/bsky/src/api/com/atproto/admin/searchRepos.ts b/packages/bsky/src/api/com/atproto/admin/searchRepos.ts index 8faf041f589..ef580f30d67 100644 --- a/packages/bsky/src/api/com/atproto/admin/searchRepos.ts +++ b/packages/bsky/src/api/com/atproto/admin/searchRepos.ts @@ -1,4 +1,3 @@ -import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' @@ -8,16 +7,14 @@ export default function (server: Server, ctx: AppContext) { handler: async ({ params }) => { const db = ctx.db.getPrimary() const moderationService = ctx.services.moderation(db) - const { invitedBy, limit, cursor } = params - if (invitedBy) { - throw new InvalidRequestError('The invitedBy parameter is unsupported') - } + const { limit, cursor } = params // prefer new 'q' query param over deprecated 'term' const query = params.q ?? params.term const { results, cursor: resCursor } = await ctx.services .actor(db) .getSearchResults({ query, limit, cursor, includeSoftDeleted: true }) + return { encoding: 'application/json', body: { diff --git a/packages/bsky/src/api/com/atproto/admin/takeModerationAction.ts b/packages/bsky/src/api/com/atproto/admin/takeModerationAction.ts index fc49a9c14ff..a8d67fced9f 100644 --- a/packages/bsky/src/api/com/atproto/admin/takeModerationAction.ts +++ b/packages/bsky/src/api/com/atproto/admin/takeModerationAction.ts @@ -1,6 +1,10 @@ import { CID } from 'multiformats/cid' import { AtUri } from '@atproto/syntax' -import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' +import { + AuthRequiredError, + InvalidRequestError, + UpstreamFailureError, +} from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { @@ -9,6 +13,8 @@ import { TAKEDOWN, } from '../../../../lexicon/types/com/atproto/admin/defs' import { getSubject, getAction } from '../moderation/util' +import { TakedownSubjects } from '../../../../services/moderation' +import { retryHttp } from '../../../../util/retry' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.takeModerationAction({ @@ -52,7 +58,7 @@ export default function (server: Server, ctx: AppContext) { validateLabels([...(createLabelVals ?? []), ...(negateLabelVals ?? [])]) - const moderationAction = await db.transaction(async (dbTxn) => { + const { result, takenDown } = await db.transaction(async (dbTxn) => { const moderationTxn = ctx.services.moderation(dbTxn) const labelTxn = ctx.services.label(dbTxn) @@ -67,13 +73,15 @@ export default function (server: Server, ctx: AppContext) { durationInHours, }) + let takenDown: TakedownSubjects | undefined + if ( result.action === TAKEDOWN && result.subjectType === 'com.atproto.admin.defs#repoRef' && result.subjectDid ) { // No credentials to revoke on appview - await moderationTxn.takedownRepo({ + takenDown = await moderationTxn.takedownRepo({ takedownId: result.id, did: result.subjectDid, }) @@ -82,11 +90,13 @@ export default function (server: Server, ctx: AppContext) { if ( result.action === TAKEDOWN && result.subjectType === 'com.atproto.repo.strongRef' && - result.subjectUri + result.subjectUri && + result.subjectCid ) { - await moderationTxn.takedownRecord({ + takenDown = await moderationTxn.takedownRecord({ takedownId: result.id, uri: new AtUri(result.subjectUri), + cid: CID.parse(result.subjectCid), blobCids: subjectBlobCids?.map((cid) => CID.parse(cid)) ?? [], }) } @@ -98,12 +108,36 @@ export default function (server: Server, ctx: AppContext) { { create: createLabelVals, negate: negateLabelVals }, ) - return result + return { result, takenDown } }) + if (takenDown) { + const { did, subjects } = takenDown + if (did && subjects.length > 0) { + const agent = await ctx.pdsAdminAgent(did) + const results = await Promise.allSettled( + subjects.map((subject) => + retryHttp(() => + agent.api.com.atproto.admin.updateSubjectStatus({ + subject, + takedown: { + applied: true, + ref: result.id.toString(), + }, + }), + ), + ), + ) + const hadFailure = results.some((r) => r.status === 'rejected') + if (hadFailure) { + throw new UpstreamFailureError('failed to apply action on PDS') + } + } + } + return { encoding: 'application/json', - body: await moderationService.views.action(moderationAction), + body: await moderationService.views.action(result), } }, }) diff --git a/packages/bsky/src/api/com/atproto/admin/util.ts b/packages/bsky/src/api/com/atproto/admin/util.ts new file mode 100644 index 00000000000..eba3eaa1d1e --- /dev/null +++ b/packages/bsky/src/api/com/atproto/admin/util.ts @@ -0,0 +1,50 @@ +import AppContext from '../../../../context' +import { + RepoView, + RepoViewDetail, + AccountView, +} from '../../../../lexicon/types/com/atproto/admin/defs' + +export const getPdsAccountInfo = async ( + ctx: AppContext, + did: string, +): Promise => { + try { + const agent = await ctx.pdsAdminAgent(did) + const res = await agent.api.com.atproto.admin.getAccountInfo({ did }) + return res.data + } catch (err) { + return null + } +} + +export const addAccountInfoToRepoViewDetail = ( + repoView: RepoViewDetail, + accountInfo: AccountView | null, + includeEmail = false, +): RepoViewDetail => { + if (!accountInfo) return repoView + return { + ...repoView, + email: includeEmail ? accountInfo.email : undefined, + invitedBy: accountInfo.invitedBy, + invitesDisabled: accountInfo.invitesDisabled, + inviteNote: accountInfo.inviteNote, + invites: accountInfo.invites, + } +} + +export const addAccountInfoToRepoView = ( + repoView: RepoView, + accountInfo: AccountView | null, + includeEmail = false, +): RepoView => { + if (!accountInfo) return repoView + return { + ...repoView, + email: includeEmail ? accountInfo.email : undefined, + invitedBy: accountInfo.invitedBy, + invitesDisabled: accountInfo.invitesDisabled, + inviteNote: accountInfo.inviteNote, + } +} diff --git a/packages/bsky/src/api/well-known.ts b/packages/bsky/src/api/well-known.ts index b6813751605..0c0802620e1 100644 --- a/packages/bsky/src/api/well-known.ts +++ b/packages/bsky/src/api/well-known.ts @@ -12,6 +12,14 @@ export const createRouter = (ctx: AppContext): express.Router => { res.json({ '@context': ['https://www.w3.org/ns/did/v1'], id: ctx.cfg.serverDid, + verificationMethod: [ + { + id: `${ctx.cfg.serverDid}#atproto`, + type: 'Multikey', + controller: ctx.cfg.serverDid, + publicKeyMultibase: ctx.signingKey.did().replace('did:key:', ''), + }, + ], service: [ { id: '#bsky_notif', diff --git a/packages/bsky/src/auth.ts b/packages/bsky/src/auth.ts index a92023d55f5..290ef3c7a42 100644 --- a/packages/bsky/src/auth.ts +++ b/packages/bsky/src/auth.ts @@ -14,11 +14,11 @@ export const authVerifier = if (!jwtStr) { throw new AuthRequiredError('missing jwt', 'MissingJwt') } - const did = await verifyJwt(jwtStr, opts.aud, async (did: string) => { + const payload = await verifyJwt(jwtStr, opts.aud, async (did: string) => { const atprotoData = await idResolver.did.resolveAtprotoData(did) return atprotoData.signingKey }) - return { credentials: { did }, artifacts: { aud: opts.aud } } + return { credentials: { did: payload.iss }, artifacts: { aud: opts.aud } } } export const authOptionalVerifier = diff --git a/packages/bsky/src/auto-moderator/index.ts b/packages/bsky/src/auto-moderator/index.ts index 30befc19110..7118b95ac62 100644 --- a/packages/bsky/src/auto-moderator/index.ts +++ b/packages/bsky/src/auto-moderator/index.ts @@ -271,6 +271,7 @@ export class AutoModerator { await modSrvc.takedownRecord({ takedownId: action.id, uri: uri, + cid: recordCid, blobCids: takedownCids, }) }) diff --git a/packages/bsky/src/context.ts b/packages/bsky/src/context.ts index 42cbfecf218..90e6cf60014 100644 --- a/packages/bsky/src/context.ts +++ b/packages/bsky/src/context.ts @@ -1,5 +1,8 @@ import * as plc from '@did-plc/lib' import { IdResolver } from '@atproto/identity' +import { AtpAgent } from '@atproto/api' +import { Keypair } from '@atproto/crypto' +import { createServiceJwt } from '@atproto/xrpc-server' import { DatabaseCoordinator } from './db' import { ServerConfig } from './config' import { ImageUriBuilder } from './image/uri' @@ -10,7 +13,6 @@ import { BackgroundQueue } from './background' import { MountedAlgos } from './feed-gen/types' import { LabelCache } from './label-cache' import { NotificationServer } from './notifications' -import { AtpAgent } from '@atproto/api' export class AppContext { constructor( @@ -19,6 +21,7 @@ export class AppContext { imgUriBuilder: ImageUriBuilder cfg: ServerConfig services: Services + signingKey: Keypair idResolver: IdResolver didCache: DidSqlCache labelCache: LabelCache @@ -45,6 +48,10 @@ export class AppContext { return this.opts.services } + get signingKey(): Keypair { + return this.opts.signingKey + } + get plcClient(): plc.Client { return new plc.Client(this.cfg.didPlcUrl) } @@ -91,6 +98,23 @@ export class AppContext { return auth.roleVerifier(this.cfg) } + async serviceAuthJwt(aud: string) { + const iss = this.cfg.serverDid + return createServiceJwt({ + iss, + aud, + keypair: this.signingKey, + }) + } + + async pdsAdminAgent(did: string): Promise { + const data = await this.idResolver.did.resolveAtprotoData(did) + const agent = new AtpAgent({ service: data.pds }) + const jwt = await this.serviceAuthJwt(did) + agent.api.setHeader('authorization', `Bearer ${jwt}`) + return agent + } + get backgroundQueue(): BackgroundQueue { return this.opts.backgroundQueue } diff --git a/packages/bsky/src/index.ts b/packages/bsky/src/index.ts index 8ef2109218e..938d634356c 100644 --- a/packages/bsky/src/index.ts +++ b/packages/bsky/src/index.ts @@ -26,6 +26,7 @@ import { MountedAlgos } from './feed-gen/types' import { LabelCache } from './label-cache' import { NotificationServer } from './notifications' import { AtpAgent } from '@atproto/api' +import { Keypair } from '@atproto/crypto' export type { ServerConfigValues } from './config' export type { MountedAlgos } from './feed-gen/types' @@ -54,10 +55,11 @@ export class BskyAppView { static create(opts: { db: DatabaseCoordinator config: ServerConfig + signingKey: Keypair imgInvalidator?: ImageInvalidator algos?: MountedAlgos }): BskyAppView { - const { db, config, algos = {} } = opts + const { db, config, signingKey, algos = {} } = opts let maybeImgInvalidator = opts.imgInvalidator const app = express() app.use(cors()) @@ -116,6 +118,7 @@ export class BskyAppView { cfg: config, services, imgUriBuilder, + signingKey, idResolver, didCache, labelCache, diff --git a/packages/bsky/src/lexicon/index.ts b/packages/bsky/src/lexicon/index.ts index 1fd8a1f127c..bf69ebafa68 100644 --- a/packages/bsky/src/lexicon/index.ts +++ b/packages/bsky/src/lexicon/index.ts @@ -12,6 +12,7 @@ import { schemas } from './lexicons' import * as ComAtprotoAdminDisableAccountInvites from './types/com/atproto/admin/disableAccountInvites' import * as ComAtprotoAdminDisableInviteCodes from './types/com/atproto/admin/disableInviteCodes' import * as ComAtprotoAdminEnableAccountInvites from './types/com/atproto/admin/enableAccountInvites' +import * as ComAtprotoAdminGetAccountInfo from './types/com/atproto/admin/getAccountInfo' import * as ComAtprotoAdminGetInviteCodes from './types/com/atproto/admin/getInviteCodes' import * as ComAtprotoAdminGetModerationAction from './types/com/atproto/admin/getModerationAction' import * as ComAtprotoAdminGetModerationActions from './types/com/atproto/admin/getModerationActions' @@ -19,6 +20,7 @@ import * as ComAtprotoAdminGetModerationReport from './types/com/atproto/admin/g import * as ComAtprotoAdminGetModerationReports from './types/com/atproto/admin/getModerationReports' import * as ComAtprotoAdminGetRecord from './types/com/atproto/admin/getRecord' import * as ComAtprotoAdminGetRepo from './types/com/atproto/admin/getRepo' +import * as ComAtprotoAdminGetSubjectStatus from './types/com/atproto/admin/getSubjectStatus' import * as ComAtprotoAdminResolveModerationReports from './types/com/atproto/admin/resolveModerationReports' import * as ComAtprotoAdminReverseModerationAction from './types/com/atproto/admin/reverseModerationAction' import * as ComAtprotoAdminSearchRepos from './types/com/atproto/admin/searchRepos' @@ -26,6 +28,7 @@ import * as ComAtprotoAdminSendEmail from './types/com/atproto/admin/sendEmail' import * as ComAtprotoAdminTakeModerationAction from './types/com/atproto/admin/takeModerationAction' import * as ComAtprotoAdminUpdateAccountEmail from './types/com/atproto/admin/updateAccountEmail' import * as ComAtprotoAdminUpdateAccountHandle from './types/com/atproto/admin/updateAccountHandle' +import * as ComAtprotoAdminUpdateSubjectStatus from './types/com/atproto/admin/updateSubjectStatus' import * as ComAtprotoIdentityResolveHandle from './types/com/atproto/identity/resolveHandle' import * as ComAtprotoIdentityUpdateHandle from './types/com/atproto/identity/updateHandle' import * as ComAtprotoLabelQueryLabels from './types/com/atproto/label/queryLabels' @@ -225,6 +228,17 @@ export class AdminNS { return this._server.xrpc.method(nsid, cfg) } + getAccountInfo( + cfg: ConfigOf< + AV, + ComAtprotoAdminGetAccountInfo.Handler>, + ComAtprotoAdminGetAccountInfo.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.admin.getAccountInfo' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } + getInviteCodes( cfg: ConfigOf< AV, @@ -302,6 +316,17 @@ export class AdminNS { return this._server.xrpc.method(nsid, cfg) } + getSubjectStatus( + cfg: ConfigOf< + AV, + ComAtprotoAdminGetSubjectStatus.Handler>, + ComAtprotoAdminGetSubjectStatus.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.admin.getSubjectStatus' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } + resolveModerationReports( cfg: ConfigOf< AV, @@ -378,6 +403,17 @@ export class AdminNS { const nsid = 'com.atproto.admin.updateAccountHandle' // @ts-ignore return this._server.xrpc.method(nsid, cfg) } + + updateSubjectStatus( + cfg: ConfigOf< + AV, + ComAtprotoAdminUpdateSubjectStatus.Handler>, + ComAtprotoAdminUpdateSubjectStatus.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.admin.updateSubjectStatus' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } } export class IdentityNS { diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index 3911dc12497..df696e5d06b 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -8,6 +8,18 @@ export const schemaDict = { lexicon: 1, id: 'com.atproto.admin.defs', defs: { + statusAttr: { + type: 'object', + required: ['applied'], + properties: { + applied: { + type: 'boolean', + }, + ref: { + type: 'string', + }, + }, + }, actionView: { type: 'object', required: [ @@ -424,6 +436,44 @@ export const schemaDict = { }, }, }, + accountView: { + type: 'object', + required: ['did', 'handle', 'indexedAt'], + properties: { + did: { + type: 'string', + format: 'did', + }, + handle: { + type: 'string', + format: 'handle', + }, + email: { + type: 'string', + }, + indexedAt: { + type: 'string', + format: 'datetime', + }, + invitedBy: { + type: 'ref', + ref: 'lex:com.atproto.server.defs#inviteCode', + }, + invites: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.server.defs#inviteCode', + }, + }, + invitesDisabled: { + type: 'boolean', + }, + inviteNote: { + type: 'string', + }, + }, + }, repoViewNotFound: { type: 'object', required: ['did'], @@ -444,6 +494,24 @@ export const schemaDict = { }, }, }, + repoBlobRef: { + type: 'object', + required: ['did', 'cid'], + properties: { + did: { + type: 'string', + format: 'did', + }, + cid: { + type: 'string', + format: 'cid', + }, + recordUri: { + type: 'string', + format: 'at-uri', + }, + }, + }, recordView: { type: 'object', required: [ @@ -730,6 +798,33 @@ export const schemaDict = { }, }, }, + ComAtprotoAdminGetAccountInfo: { + lexicon: 1, + id: 'com.atproto.admin.getAccountInfo', + defs: { + main: { + type: 'query', + description: 'View details about an account.', + parameters: { + type: 'params', + required: ['did'], + properties: { + did: { + type: 'string', + format: 'did', + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#accountView', + }, + }, + }, + }, + }, ComAtprotoAdminGetInviteCodes: { lexicon: 1, id: 'com.atproto.admin.getInviteCodes', @@ -1026,6 +1121,55 @@ export const schemaDict = { }, }, }, + ComAtprotoAdminGetSubjectStatus: { + lexicon: 1, + id: 'com.atproto.admin.getSubjectStatus', + defs: { + main: { + type: 'query', + description: + 'Fetch the service-specific the admin status of a subject (account, record, or blob)', + parameters: { + type: 'params', + properties: { + did: { + type: 'string', + format: 'did', + }, + uri: { + type: 'string', + format: 'at-uri', + }, + blob: { + type: 'string', + format: 'cid', + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['subject'], + properties: { + subject: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#repoRef', + 'lex:com.atproto.repo.strongRef', + 'lex:com.atproto.admin.defs#repoBlobRef', + ], + }, + takedown: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#statusAttr', + }, + }, + }, + }, + }, + }, + }, ComAtprotoAdminResolveModerationReports: { lexicon: 1, id: 'com.atproto.admin.resolveModerationReports', @@ -1118,9 +1262,6 @@ export const schemaDict = { q: { type: 'string', }, - invitedBy: { - type: 'string', - }, limit: { type: 'integer', minimum: 1, @@ -1326,6 +1467,59 @@ export const schemaDict = { }, }, }, + ComAtprotoAdminUpdateSubjectStatus: { + lexicon: 1, + id: 'com.atproto.admin.updateSubjectStatus', + defs: { + main: { + type: 'procedure', + description: + 'Update the service-specific admin status of a subject (account, record, or blob)', + input: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['subject'], + properties: { + subject: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#repoRef', + 'lex:com.atproto.repo.strongRef', + 'lex:com.atproto.admin.defs#repoBlobRef', + ], + }, + takedown: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#statusAttr', + }, + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['subject'], + properties: { + subject: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#repoRef', + 'lex:com.atproto.repo.strongRef', + 'lex:com.atproto.admin.defs#repoBlobRef', + ], + }, + takedown: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#statusAttr', + }, + }, + }, + }, + }, + }, + }, ComAtprotoIdentityResolveHandle: { lexicon: 1, id: 'com.atproto.identity.resolveHandle', @@ -7369,6 +7563,7 @@ export const ids = { 'com.atproto.admin.disableAccountInvites', ComAtprotoAdminDisableInviteCodes: 'com.atproto.admin.disableInviteCodes', ComAtprotoAdminEnableAccountInvites: 'com.atproto.admin.enableAccountInvites', + ComAtprotoAdminGetAccountInfo: 'com.atproto.admin.getAccountInfo', ComAtprotoAdminGetInviteCodes: 'com.atproto.admin.getInviteCodes', ComAtprotoAdminGetModerationAction: 'com.atproto.admin.getModerationAction', ComAtprotoAdminGetModerationActions: 'com.atproto.admin.getModerationActions', @@ -7376,6 +7571,7 @@ export const ids = { ComAtprotoAdminGetModerationReports: 'com.atproto.admin.getModerationReports', ComAtprotoAdminGetRecord: 'com.atproto.admin.getRecord', ComAtprotoAdminGetRepo: 'com.atproto.admin.getRepo', + ComAtprotoAdminGetSubjectStatus: 'com.atproto.admin.getSubjectStatus', ComAtprotoAdminResolveModerationReports: 'com.atproto.admin.resolveModerationReports', ComAtprotoAdminReverseModerationAction: @@ -7385,6 +7581,7 @@ export const ids = { ComAtprotoAdminTakeModerationAction: 'com.atproto.admin.takeModerationAction', ComAtprotoAdminUpdateAccountEmail: 'com.atproto.admin.updateAccountEmail', ComAtprotoAdminUpdateAccountHandle: 'com.atproto.admin.updateAccountHandle', + ComAtprotoAdminUpdateSubjectStatus: 'com.atproto.admin.updateSubjectStatus', ComAtprotoIdentityResolveHandle: 'com.atproto.identity.resolveHandle', ComAtprotoIdentityUpdateHandle: 'com.atproto.identity.updateHandle', ComAtprotoLabelDefs: 'com.atproto.label.defs', diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts index 968252a4c2c..ea463368f8e 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts @@ -10,6 +10,24 @@ import * as ComAtprotoModerationDefs from '../moderation/defs' import * as ComAtprotoServerDefs from '../server/defs' import * as ComAtprotoLabelDefs from '../label/defs' +export interface StatusAttr { + applied: boolean + ref?: string + [k: string]: unknown +} + +export function isStatusAttr(v: unknown): v is StatusAttr { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#statusAttr' + ) +} + +export function validateStatusAttr(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#statusAttr', v) +} + export interface ActionView { id: number action: ActionType @@ -238,6 +256,30 @@ export function validateRepoViewDetail(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#repoViewDetail', v) } +export interface AccountView { + did: string + handle: string + email?: string + indexedAt: string + invitedBy?: ComAtprotoServerDefs.InviteCode + invites?: ComAtprotoServerDefs.InviteCode[] + invitesDisabled?: boolean + inviteNote?: string + [k: string]: unknown +} + +export function isAccountView(v: unknown): v is AccountView { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#accountView' + ) +} + +export function validateAccountView(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#accountView', v) +} + export interface RepoViewNotFound { did: string [k: string]: unknown @@ -272,6 +314,25 @@ export function validateRepoRef(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#repoRef', v) } +export interface RepoBlobRef { + did: string + cid: string + recordUri?: string + [k: string]: unknown +} + +export function isRepoBlobRef(v: unknown): v is RepoBlobRef { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#repoBlobRef' + ) +} + +export function validateRepoBlobRef(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#repoBlobRef', v) +} + export interface RecordView { uri: string cid: string diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/getAccountInfo.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/getAccountInfo.ts new file mode 100644 index 00000000000..88a2b17a4b8 --- /dev/null +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/getAccountInfo.ts @@ -0,0 +1,41 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' +import * as ComAtprotoAdminDefs from './defs' + +export interface QueryParams { + did: string +} + +export type InputSchema = undefined +export type OutputSchema = ComAtprotoAdminDefs.AccountView +export type HandlerInput = undefined + +export interface HandlerSuccess { + encoding: 'application/json' + body: OutputSchema + headers?: { [key: string]: string } +} + +export interface HandlerError { + status: number + message?: string +} + +export type HandlerOutput = HandlerError | HandlerSuccess +export type HandlerReqCtx = { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +} +export type Handler = ( + ctx: HandlerReqCtx, +) => Promise | HandlerOutput diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/getSubjectStatus.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/getSubjectStatus.ts new file mode 100644 index 00000000000..7315e51e8c2 --- /dev/null +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/getSubjectStatus.ts @@ -0,0 +1,54 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' +import * as ComAtprotoAdminDefs from './defs' +import * as ComAtprotoRepoStrongRef from '../repo/strongRef' + +export interface QueryParams { + did?: string + uri?: string + blob?: string +} + +export type InputSchema = undefined + +export interface OutputSchema { + subject: + | ComAtprotoAdminDefs.RepoRef + | ComAtprotoRepoStrongRef.Main + | ComAtprotoAdminDefs.RepoBlobRef + | { $type: string; [k: string]: unknown } + takedown?: ComAtprotoAdminDefs.StatusAttr + [k: string]: unknown +} + +export type HandlerInput = undefined + +export interface HandlerSuccess { + encoding: 'application/json' + body: OutputSchema + headers?: { [key: string]: string } +} + +export interface HandlerError { + status: number + message?: string +} + +export type HandlerOutput = HandlerError | HandlerSuccess +export type HandlerReqCtx = { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +} +export type Handler = ( + ctx: HandlerReqCtx, +) => Promise | HandlerOutput diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/searchRepos.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/searchRepos.ts index 32266fd66fd..1e7e1a36bb6 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/searchRepos.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/searchRepos.ts @@ -13,7 +13,6 @@ export interface QueryParams { /** DEPRECATED: use 'q' instead */ term?: string q?: string - invitedBy?: string limit: number cursor?: string } diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/updateSubjectStatus.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/updateSubjectStatus.ts new file mode 100644 index 00000000000..559ee948380 --- /dev/null +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/updateSubjectStatus.ts @@ -0,0 +1,61 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' +import * as ComAtprotoAdminDefs from './defs' +import * as ComAtprotoRepoStrongRef from '../repo/strongRef' + +export interface QueryParams {} + +export interface InputSchema { + subject: + | ComAtprotoAdminDefs.RepoRef + | ComAtprotoRepoStrongRef.Main + | ComAtprotoAdminDefs.RepoBlobRef + | { $type: string; [k: string]: unknown } + takedown?: ComAtprotoAdminDefs.StatusAttr + [k: string]: unknown +} + +export interface OutputSchema { + subject: + | ComAtprotoAdminDefs.RepoRef + | ComAtprotoRepoStrongRef.Main + | ComAtprotoAdminDefs.RepoBlobRef + | { $type: string; [k: string]: unknown } + takedown?: ComAtprotoAdminDefs.StatusAttr + [k: string]: unknown +} + +export interface HandlerInput { + encoding: 'application/json' + body: InputSchema +} + +export interface HandlerSuccess { + encoding: 'application/json' + body: OutputSchema + headers?: { [key: string]: string } +} + +export interface HandlerError { + status: number + message?: string +} + +export type HandlerOutput = HandlerError | HandlerSuccess +export type HandlerReqCtx = { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +} +export type Handler = ( + ctx: HandlerReqCtx, +) => Promise | HandlerOutput diff --git a/packages/bsky/src/services/moderation/index.ts b/packages/bsky/src/services/moderation/index.ts index 0abf8f348eb..e85f1218470 100644 --- a/packages/bsky/src/services/moderation/index.ts +++ b/packages/bsky/src/services/moderation/index.ts @@ -7,7 +7,12 @@ import { ModerationAction, ModerationReport } from '../../db/tables/moderation' import { ModerationViews } from './views' import { ImageUriBuilder } from '../../image/uri' import { ImageInvalidator } from '../../image/invalidator' -import { TAKEDOWN } from '../../lexicon/types/com/atproto/admin/defs' +import { + RepoRef, + RepoBlobRef, + TAKEDOWN, +} from '../../lexicon/types/com/atproto/admin/defs' +import { Main as StrongRef } from '../../lexicon/types/com/atproto/repo/strongRef' import { addHoursToDate } from '../../util/date' export class ModerationService { @@ -355,7 +360,10 @@ export class ModerationService { createdBy, createdAt, reason, - }: ReversibleModerationAction) { + }: ReversibleModerationAction): Promise<{ + result: ModerationActionRow + restored?: TakedownSubjects + }> { this.db.assertTransaction() const result = await this.logReverseAction({ id, @@ -364,6 +372,8 @@ export class ModerationService { reason, }) + let restored: TakedownSubjects | undefined + if ( result.action === TAKEDOWN && result.subjectType === 'com.atproto.admin.defs#repoRef' && @@ -372,6 +382,15 @@ export class ModerationService { await this.reverseTakedownRepo({ did: result.subjectDid, }) + restored = { + did: result.subjectDid, + subjects: [ + { + $type: 'com.atproto.admin.defs#repoRef', + did: result.subjectDid, + }, + ], + } } if ( @@ -379,12 +398,35 @@ export class ModerationService { result.subjectType === 'com.atproto.repo.strongRef' && result.subjectUri ) { + const uri = new AtUri(result.subjectUri) await this.reverseTakedownRecord({ - uri: new AtUri(result.subjectUri), + uri, }) + const did = uri.hostname + const actionBlobs = await this.db.db + .selectFrom('moderation_action_subject_blob') + .where('actionId', '=', id) + .select('cid') + .execute() + restored = { + did, + subjects: [ + { + $type: 'com.atproto.repo.strongRef', + uri: result.subjectUri, + cid: result.subjectCid ?? '', + }, + ...actionBlobs.map((row) => ({ + $type: 'com.atproto.admin.defs#repoBlobRef', + did, + cid: row.cid, + recordUri: result.subjectUri, + })), + ], + } } - return result + return { result, restored } } async logReverseAction( @@ -410,13 +452,27 @@ export class ModerationService { return result } - async takedownRepo(info: { takedownId: number; did: string }) { + async takedownRepo(info: { + takedownId: number + did: string + }): Promise { + const { takedownId, did } = info await this.db.db .updateTable('actor') - .set({ takedownId: info.takedownId }) - .where('did', '=', info.did) + .set({ takedownId }) + .where('did', '=', did) .where('takedownId', 'is', null) .executeTakeFirst() + + return { + did, + subjects: [ + { + $type: 'com.atproto.admin.defs#repoRef', + did, + }, + ], + } } async reverseTakedownRepo(info: { did: string }) { @@ -430,26 +486,45 @@ export class ModerationService { async takedownRecord(info: { takedownId: number uri: AtUri + cid: CID blobCids?: CID[] - }) { + }): Promise { + const { takedownId, uri, cid, blobCids } = info + const did = uri.hostname this.db.assertTransaction() await this.db.db .updateTable('record') - .set({ takedownId: info.takedownId }) - .where('uri', '=', info.uri.toString()) + .set({ takedownId }) + .where('uri', '=', uri.toString()) .where('takedownId', 'is', null) .executeTakeFirst() - if (info.blobCids) { + if (blobCids) { await Promise.all( - info.blobCids.map(async (cid) => { + blobCids.map(async (cid) => { const paths = ImageUriBuilder.presets.map((id) => { - const uri = this.imgUriBuilder.getPresetUri(id, info.uri.host, cid) - return uri.replace(this.imgUriBuilder.endpoint, '') + const imgUri = this.imgUriBuilder.getPresetUri(id, uri.host, cid) + return imgUri.replace(this.imgUriBuilder.endpoint, '') }) await this.imgInvalidator.invalidate(cid.toString(), paths) }), ) } + return { + did, + subjects: [ + { + $type: 'com.atproto.repo.strongRef', + uri: uri.toString(), + cid: cid.toString(), + }, + ...(blobCids || []).map((cid) => ({ + $type: 'com.atproto.admin.defs#repoBlobRef', + did, + cid: cid.toString(), + recordUri: uri.toString(), + })), + ], + } } async reverseTakedownRecord(info: { uri: AtUri }) { @@ -563,6 +638,11 @@ export class ModerationService { } } +export type TakedownSubjects = { + did: string + subjects: (RepoRef | RepoBlobRef | StrongRef)[] +} + export type ModerationActionRow = Selectable export type ReversibleModerationAction = Pick< ModerationActionRow, diff --git a/packages/bsky/src/services/moderation/views.ts b/packages/bsky/src/services/moderation/views.ts index b8d745a594d..06398c3427e 100644 --- a/packages/bsky/src/services/moderation/views.ts +++ b/packages/bsky/src/services/moderation/views.ts @@ -211,6 +211,7 @@ export class ModerationViews { .selectFrom('moderation_report') .where('subjectType', '=', 'com.atproto.repo.strongRef') .where('subjectUri', '=', result.uri) + .leftJoin('actor', 'actor.did', 'moderation_report.subjectDid') .orderBy('id', 'desc') .selectAll() .execute(), diff --git a/packages/pds/tests/admin/__snapshots__/get-moderation-action.test.ts.snap b/packages/bsky/tests/admin/__snapshots__/get-moderation-action.test.ts.snap similarity index 94% rename from packages/pds/tests/admin/__snapshots__/get-moderation-action.test.ts.snap rename to packages/bsky/tests/admin/__snapshots__/get-moderation-action.test.ts.snap index aedd7a5a7ea..fffc5678d9b 100644 --- a/packages/pds/tests/admin/__snapshots__/get-moderation-action.test.ts.snap +++ b/packages/bsky/tests/admin/__snapshots__/get-moderation-action.test.ts.snap @@ -1,11 +1,11 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`pds admin get moderation action view gets moderation action for a record. 1`] = ` +exports[`admin get moderation action view gets moderation action for a record. 1`] = ` Object { "action": "com.atproto.admin.defs#takedown", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 3, + "id": 2, "reason": "X", "resolvedReports": Array [ Object { @@ -15,8 +15,8 @@ Object { "reasonType": "com.atproto.moderation.defs#reasonOther", "reportedBy": "user(1)", "resolvedByActionIds": Array [ - 3, 2, + 1, ], "subject": Object { "$type": "com.atproto.repo.strongRef", @@ -33,7 +33,7 @@ Object { "moderation": Object { "currentAction": Object { "action": "com.atproto.admin.defs#takedown", - "id": 3, + "id": 2, }, }, "repo": Object { @@ -89,12 +89,12 @@ Object { } `; -exports[`pds admin get moderation action view gets moderation action for a repo. 1`] = ` +exports[`admin get moderation action view gets moderation action for a repo. 1`] = ` Object { "action": "com.atproto.admin.defs#flag", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 2, + "id": 1, "reason": "X", "resolvedReports": Array [ Object { @@ -104,8 +104,8 @@ Object { "reasonType": "com.atproto.moderation.defs#reasonOther", "reportedBy": "user(1)", "resolvedByActionIds": Array [ - 3, 2, + 1, ], "subject": Object { "$type": "com.atproto.repo.strongRef", @@ -119,7 +119,7 @@ Object { "reasonType": "com.atproto.moderation.defs#reasonSpam", "reportedBy": "user(2)", "resolvedByActionIds": Array [ - 2, + 1, ], "subject": Object { "$type": "com.atproto.admin.defs#repoRef", diff --git a/packages/pds/tests/admin/__snapshots__/get-moderation-actions.test.ts.snap b/packages/bsky/tests/admin/__snapshots__/get-moderation-actions.test.ts.snap similarity index 85% rename from packages/pds/tests/admin/__snapshots__/get-moderation-actions.test.ts.snap rename to packages/bsky/tests/admin/__snapshots__/get-moderation-actions.test.ts.snap index 67ef8d45700..625df2076d8 100644 --- a/packages/pds/tests/admin/__snapshots__/get-moderation-actions.test.ts.snap +++ b/packages/bsky/tests/admin/__snapshots__/get-moderation-actions.test.ts.snap @@ -1,12 +1,12 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`pds admin get moderation actions view gets all moderation actions for a record. 1`] = ` +exports[`admin get moderation actions view gets all moderation actions for a record. 1`] = ` Array [ Object { "action": "com.atproto.admin.defs#flag", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 2, + "id": 1, "reason": "X", "resolvedReportIds": Array [ 1, @@ -26,13 +26,13 @@ Array [ ] `; -exports[`pds admin get moderation actions view gets all moderation actions for a repo. 1`] = ` +exports[`admin get moderation actions view gets all moderation actions for a repo. 1`] = ` Array [ Object { "action": "com.atproto.admin.defs#flag", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 6, + "id": 5, "reason": "X", "resolvedReportIds": Array [ 3, @@ -47,7 +47,7 @@ Array [ "action": "com.atproto.admin.defs#acknowledge", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 3, + "id": 2, "reason": "X", "resolvedReportIds": Array [], "subject": Object { @@ -61,7 +61,7 @@ Array [ "action": "com.atproto.admin.defs#flag", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 2, + "id": 1, "reason": "X", "resolvedReportIds": Array [ 1, @@ -81,13 +81,13 @@ Array [ ] `; -exports[`pds admin get moderation actions view gets all moderation actions. 1`] = ` +exports[`admin get moderation actions view gets all moderation actions. 1`] = ` Array [ Object { "action": "com.atproto.admin.defs#acknowledge", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 7, + "id": 6, "reason": "X", "resolvedReportIds": Array [], "subject": Object { @@ -100,7 +100,7 @@ Array [ "action": "com.atproto.admin.defs#flag", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 6, + "id": 5, "reason": "X", "resolvedReportIds": Array [ 3, @@ -115,7 +115,7 @@ Array [ "action": "com.atproto.admin.defs#flag", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 5, + "id": 4, "reason": "X", "resolvedReportIds": Array [], "subject": Object { @@ -129,7 +129,7 @@ Array [ "action": "com.atproto.admin.defs#takedown", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 4, + "id": 3, "reason": "X", "resolvedReportIds": Array [], "subject": Object { @@ -143,7 +143,7 @@ Array [ "action": "com.atproto.admin.defs#acknowledge", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 3, + "id": 2, "reason": "X", "resolvedReportIds": Array [], "subject": Object { @@ -157,7 +157,7 @@ Array [ "action": "com.atproto.admin.defs#flag", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 2, + "id": 1, "reason": "X", "resolvedReportIds": Array [ 1, @@ -174,21 +174,5 @@ Array [ }, "subjectBlobCids": Array [], }, - Object { - "action": "com.atproto.admin.defs#flag", - "createLabelVals": Array [ - "repo-action-label", - ], - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 1, - "reason": "test", - "resolvedReportIds": Array [], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(2)", - }, - "subjectBlobCids": Array [], - }, ] `; diff --git a/packages/pds/tests/admin/__snapshots__/get-moderation-report.test.ts.snap b/packages/bsky/tests/admin/__snapshots__/get-moderation-report.test.ts.snap similarity index 95% rename from packages/pds/tests/admin/__snapshots__/get-moderation-report.test.ts.snap rename to packages/bsky/tests/admin/__snapshots__/get-moderation-report.test.ts.snap index 70e829d0ab0..44a42b129e7 100644 --- a/packages/pds/tests/admin/__snapshots__/get-moderation-report.test.ts.snap +++ b/packages/bsky/tests/admin/__snapshots__/get-moderation-report.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`pds admin get moderation action view gets moderation report for a record. 1`] = ` +exports[`admin get moderation action view gets moderation report for a record. 1`] = ` Object { "createdAt": "1970-01-01T00:00:00.000Z", "id": 2, @@ -12,7 +12,7 @@ Object { "action": "com.atproto.admin.defs#takedown", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 3, + "id": 2, "reason": "X", "resolvedReportIds": Array [ 2, @@ -28,7 +28,7 @@ Object { "action": "com.atproto.admin.defs#flag", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 2, + "id": 1, "reason": "X", "resolvedReportIds": Array [ 2, @@ -54,7 +54,7 @@ Object { "moderation": Object { "currentAction": Object { "action": "com.atproto.admin.defs#takedown", - "id": 3, + "id": 2, }, }, "repo": Object { @@ -109,7 +109,7 @@ Object { } `; -exports[`pds admin get moderation action view gets moderation report for a repo. 1`] = ` +exports[`admin get moderation action view gets moderation report for a repo. 1`] = ` Object { "createdAt": "1970-01-01T00:00:00.000Z", "id": 1, @@ -120,7 +120,7 @@ Object { "action": "com.atproto.admin.defs#flag", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 2, + "id": 1, "reason": "X", "resolvedReportIds": Array [ 2, diff --git a/packages/pds/tests/admin/__snapshots__/get-moderation-reports.test.ts.snap b/packages/bsky/tests/admin/__snapshots__/get-moderation-reports.test.ts.snap similarity index 88% rename from packages/pds/tests/admin/__snapshots__/get-moderation-reports.test.ts.snap rename to packages/bsky/tests/admin/__snapshots__/get-moderation-reports.test.ts.snap index 9cfb5ae3c34..9708df52cc6 100644 --- a/packages/pds/tests/admin/__snapshots__/get-moderation-reports.test.ts.snap +++ b/packages/bsky/tests/admin/__snapshots__/get-moderation-reports.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`pds admin get moderation reports view gets all moderation reports actioned by a certain moderator. 1`] = ` +exports[`admin get moderation reports view gets all moderation reports actioned by a certain moderator. 1`] = ` Array [ Object { "createdAt": "1970-01-01T00:00:00.000Z", @@ -8,7 +8,7 @@ Array [ "reasonType": "com.atproto.moderation.defs#reasonOther", "reportedBy": "user(0)", "resolvedByActionIds": Array [ - 2, + 1, ], "subject": Object { "$type": "com.atproto.repo.strongRef", @@ -20,7 +20,7 @@ Array [ ] `; -exports[`pds admin get moderation reports view gets all moderation reports actioned by a certain moderator. 2`] = ` +exports[`admin get moderation reports view gets all moderation reports actioned by a certain moderator. 2`] = ` Array [ Object { "createdAt": "1970-01-01T00:00:00.000Z", @@ -28,7 +28,7 @@ Array [ "reasonType": "com.atproto.moderation.defs#reasonOther", "reportedBy": "user(0)", "resolvedByActionIds": Array [ - 4, + 3, ], "subject": Object { "$type": "com.atproto.repo.strongRef", @@ -40,7 +40,7 @@ Array [ ] `; -exports[`pds admin get moderation reports view gets all moderation reports by active resolution action type. 1`] = ` +exports[`admin get moderation reports view gets all moderation reports by active resolution action type. 1`] = ` Array [ Object { "createdAt": "1970-01-01T00:00:00.000Z", @@ -48,7 +48,7 @@ Array [ "reasonType": "com.atproto.moderation.defs#reasonOther", "reportedBy": "user(0)", "resolvedByActionIds": Array [ - 4, + 3, ], "subject": Object { "$type": "com.atproto.repo.strongRef", @@ -60,7 +60,7 @@ Array [ ] `; -exports[`pds admin get moderation reports view gets all moderation reports for a record. 1`] = ` +exports[`admin get moderation reports view gets all moderation reports for a record. 1`] = ` Array [ Object { "createdAt": "1970-01-01T00:00:00.000Z", @@ -68,7 +68,7 @@ Array [ "reasonType": "com.atproto.moderation.defs#reasonOther", "reportedBy": "user(0)", "resolvedByActionIds": Array [ - 2, + 1, ], "subject": Object { "$type": "com.atproto.repo.strongRef", @@ -80,7 +80,7 @@ Array [ ] `; -exports[`pds admin get moderation reports view gets all moderation reports for a repo. 1`] = ` +exports[`admin get moderation reports view gets all moderation reports for a repo. 1`] = ` Array [ Object { "createdAt": "1970-01-01T00:00:00.000Z", @@ -88,7 +88,7 @@ Array [ "reasonType": "com.atproto.moderation.defs#reasonOther", "reportedBy": "user(0)", "resolvedByActionIds": Array [ - 6, + 5, ], "subject": Object { "$type": "com.atproto.admin.defs#repoRef", @@ -115,7 +115,7 @@ Array [ "reasonType": "com.atproto.moderation.defs#reasonOther", "reportedBy": "user(0)", "resolvedByActionIds": Array [ - 2, + 1, ], "subject": Object { "$type": "com.atproto.repo.strongRef", @@ -127,7 +127,7 @@ Array [ ] `; -exports[`pds admin get moderation reports view gets all moderation reports. 1`] = ` +exports[`admin get moderation reports view gets all moderation reports. 1`] = ` Array [ Object { "createdAt": "1970-01-01T00:00:00.000Z", @@ -147,7 +147,7 @@ Array [ "reasonType": "com.atproto.moderation.defs#reasonOther", "reportedBy": "user(1)", "resolvedByActionIds": Array [ - 6, + 5, ], "subject": Object { "$type": "com.atproto.admin.defs#repoRef", @@ -174,7 +174,7 @@ Array [ "reasonType": "com.atproto.moderation.defs#reasonOther", "reportedBy": "user(1)", "resolvedByActionIds": Array [ - 4, + 3, ], "subject": Object { "$type": "com.atproto.repo.strongRef", @@ -202,7 +202,7 @@ Array [ "reasonType": "com.atproto.moderation.defs#reasonOther", "reportedBy": "user(1)", "resolvedByActionIds": Array [ - 2, + 1, ], "subject": Object { "$type": "com.atproto.repo.strongRef", @@ -214,7 +214,7 @@ Array [ ] `; -exports[`pds admin get moderation reports view gets all resolved/unresolved moderation reports. 1`] = ` +exports[`admin get moderation reports view gets all resolved/unresolved moderation reports. 1`] = ` Array [ Object { "createdAt": "1970-01-01T00:00:00.000Z", @@ -222,7 +222,7 @@ Array [ "reasonType": "com.atproto.moderation.defs#reasonOther", "reportedBy": "user(0)", "resolvedByActionIds": Array [ - 6, + 5, ], "subject": Object { "$type": "com.atproto.admin.defs#repoRef", @@ -236,7 +236,7 @@ Array [ "reasonType": "com.atproto.moderation.defs#reasonOther", "reportedBy": "user(0)", "resolvedByActionIds": Array [ - 4, + 3, ], "subject": Object { "$type": "com.atproto.repo.strongRef", @@ -251,7 +251,7 @@ Array [ "reasonType": "com.atproto.moderation.defs#reasonOther", "reportedBy": "user(0)", "resolvedByActionIds": Array [ - 2, + 1, ], "subject": Object { "$type": "com.atproto.repo.strongRef", @@ -263,7 +263,7 @@ Array [ ] `; -exports[`pds admin get moderation reports view gets all resolved/unresolved moderation reports. 2`] = ` +exports[`admin get moderation reports view gets all resolved/unresolved moderation reports. 2`] = ` Array [ Object { "createdAt": "1970-01-01T00:00:00.000Z", diff --git a/packages/pds/tests/admin/__snapshots__/get-record.test.ts.snap b/packages/bsky/tests/admin/__snapshots__/get-record.test.ts.snap similarity index 91% rename from packages/pds/tests/admin/__snapshots__/get-record.test.ts.snap rename to packages/bsky/tests/admin/__snapshots__/get-record.test.ts.snap index 00fbc5bda1c..cbb922003cb 100644 --- a/packages/pds/tests/admin/__snapshots__/get-record.test.ts.snap +++ b/packages/bsky/tests/admin/__snapshots__/get-record.test.ts.snap @@ -1,18 +1,28 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`pds admin get record view gets a record by uri and cid. 1`] = ` +exports[`admin get record view gets a record by uri and cid. 1`] = ` Object { "blobCids": Array [], "blobs": Array [], "cid": "cids(0)", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [ + Object { + "cid": "cids(0)", + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "user(0)", + "uri": "record(0)", + "val": "self-label", + }, + ], "moderation": Object { "actions": Array [ Object { "action": "com.atproto.admin.defs#takedown", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 3, + "id": 2, "reason": "X", "resolvedReportIds": Array [], "subject": Object { @@ -26,7 +36,7 @@ Object { "action": "com.atproto.admin.defs#acknowledge", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 2, + "id": 1, "reason": "X", "resolvedReportIds": Array [], "reversal": Object { @@ -44,7 +54,7 @@ Object { ], "currentAction": Object { "action": "com.atproto.admin.defs#takedown", - "id": 3, + "id": 2, }, "reports": Array [ Object { @@ -127,19 +137,29 @@ Object { } `; -exports[`pds admin get record view gets a record by uri, even when taken down. 1`] = ` +exports[`admin get record view gets a record by uri, even when taken down. 1`] = ` Object { "blobCids": Array [], "blobs": Array [], "cid": "cids(0)", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [ + Object { + "cid": "cids(0)", + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "user(0)", + "uri": "record(0)", + "val": "self-label", + }, + ], "moderation": Object { "actions": Array [ Object { "action": "com.atproto.admin.defs#takedown", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 3, + "id": 2, "reason": "X", "resolvedReportIds": Array [], "subject": Object { @@ -153,7 +173,7 @@ Object { "action": "com.atproto.admin.defs#acknowledge", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 2, + "id": 1, "reason": "X", "resolvedReportIds": Array [], "reversal": Object { @@ -171,7 +191,7 @@ Object { ], "currentAction": Object { "action": "com.atproto.admin.defs#takedown", - "id": 3, + "id": 2, }, "reports": Array [ Object { diff --git a/packages/pds/tests/admin/__snapshots__/get-repo.test.ts.snap b/packages/bsky/tests/admin/__snapshots__/get-repo.test.ts.snap similarity index 95% rename from packages/pds/tests/admin/__snapshots__/get-repo.test.ts.snap rename to packages/bsky/tests/admin/__snapshots__/get-repo.test.ts.snap index c90b1a070b2..1a60b27b069 100644 --- a/packages/pds/tests/admin/__snapshots__/get-repo.test.ts.snap +++ b/packages/bsky/tests/admin/__snapshots__/get-repo.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`pds admin get repo view gets a repo by did, even when taken down. 1`] = ` +exports[`admin get repo view gets a repo by did, even when taken down. 1`] = ` Object { "did": "user(0)", "email": "alice@test.com", @@ -8,13 +8,14 @@ Object { "indexedAt": "1970-01-01T00:00:00.000Z", "invites": Array [], "invitesDisabled": false, + "labels": Array [], "moderation": Object { "actions": Array [ Object { "action": "com.atproto.admin.defs#takedown", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 3, + "id": 2, "reason": "X", "resolvedReportIds": Array [], "subject": Object { @@ -27,7 +28,7 @@ Object { "action": "com.atproto.admin.defs#acknowledge", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 2, + "id": 1, "reason": "X", "resolvedReportIds": Array [], "reversal": Object { @@ -44,7 +45,7 @@ Object { ], "currentAction": Object { "action": "com.atproto.admin.defs#takedown", - "id": 3, + "id": 2, }, "reports": Array [ Object { diff --git a/packages/bsky/tests/__snapshots__/moderation.test.ts.snap b/packages/bsky/tests/admin/__snapshots__/moderation.test.ts.snap similarity index 100% rename from packages/bsky/tests/__snapshots__/moderation.test.ts.snap rename to packages/bsky/tests/admin/__snapshots__/moderation.test.ts.snap diff --git a/packages/pds/tests/admin/get-moderation-action.test.ts b/packages/bsky/tests/admin/get-moderation-action.test.ts similarity index 89% rename from packages/pds/tests/admin/get-moderation-action.test.ts rename to packages/bsky/tests/admin/get-moderation-action.test.ts index 11a64799db3..5c7fe3401db 100644 --- a/packages/pds/tests/admin/get-moderation-action.test.ts +++ b/packages/bsky/tests/admin/get-moderation-action.test.ts @@ -1,4 +1,4 @@ -import { TestNetworkNoAppView, SeedClient } from '@atproto/dev-env' +import { TestNetwork, SeedClient } from '@atproto/dev-env' import AtpAgent from '@atproto/api' import { FLAG, @@ -11,13 +11,13 @@ import { import { forSnapshot } from '../_util' import basicSeed from '../seeds/basic' -describe('pds admin get moderation action view', () => { - let network: TestNetworkNoAppView +describe('admin get moderation action view', () => { + let network: TestNetwork let agent: AtpAgent let sc: SeedClient beforeAll(async () => { - network = await TestNetworkNoAppView.create({ + network = await TestNetwork.create({ dbPostgresSchema: 'views_admin_get_moderation_action', }) agent = network.pds.getClient() @@ -75,18 +75,16 @@ describe('pds admin get moderation action view', () => { }) it('gets moderation action for a repo.', async () => { - // id 2 because id 1 is in seed client const result = await agent.api.com.atproto.admin.getModerationAction( - { id: 2 }, + { id: 1 }, { headers: { authorization: network.pds.adminAuth() } }, ) expect(forSnapshot(result.data)).toMatchSnapshot() }) it('gets moderation action for a record.', async () => { - // id 3 because id 1 is in seed client const result = await agent.api.com.atproto.admin.getModerationAction( - { id: 3 }, + { id: 2 }, { headers: { authorization: network.pds.adminAuth() } }, ) expect(forSnapshot(result.data)).toMatchSnapshot() diff --git a/packages/pds/tests/admin/get-moderation-actions.test.ts b/packages/bsky/tests/admin/get-moderation-actions.test.ts similarity index 94% rename from packages/pds/tests/admin/get-moderation-actions.test.ts rename to packages/bsky/tests/admin/get-moderation-actions.test.ts index 01a934c32e0..dfc08aa82b5 100644 --- a/packages/pds/tests/admin/get-moderation-actions.test.ts +++ b/packages/bsky/tests/admin/get-moderation-actions.test.ts @@ -1,4 +1,4 @@ -import { TestNetworkNoAppView, SeedClient } from '@atproto/dev-env' +import { SeedClient, TestNetwork } from '@atproto/dev-env' import AtpAgent from '@atproto/api' import { ACKNOWLEDGE, @@ -12,13 +12,13 @@ import { import { forSnapshot, paginateAll } from '../_util' import basicSeed from '../seeds/basic' -describe('pds admin get moderation actions view', () => { - let network: TestNetworkNoAppView +describe('admin get moderation actions view', () => { + let network: TestNetwork let agent: AtpAgent let sc: SeedClient beforeAll(async () => { - network = await TestNetworkNoAppView.create({ + network = await TestNetwork.create({ dbPostgresSchema: 'views_admin_get_moderation_actions', }) agent = network.pds.getClient() @@ -158,7 +158,7 @@ describe('pds admin get moderation actions view', () => { { headers: network.pds.adminAuthHeaders() }, ) - expect(full.data.actions.length).toEqual(7) // extra one because of seed client + expect(full.data.actions.length).toEqual(6) expect(results(paginatedAll)).toEqual(results([full.data])) }) }) diff --git a/packages/pds/tests/admin/get-moderation-report.test.ts b/packages/bsky/tests/admin/get-moderation-report.test.ts similarity index 92% rename from packages/pds/tests/admin/get-moderation-report.test.ts rename to packages/bsky/tests/admin/get-moderation-report.test.ts index 714596e352f..4a77750aa0a 100644 --- a/packages/pds/tests/admin/get-moderation-report.test.ts +++ b/packages/bsky/tests/admin/get-moderation-report.test.ts @@ -1,4 +1,4 @@ -import { TestNetworkNoAppView, SeedClient } from '@atproto/dev-env' +import { SeedClient, TestNetwork } from '@atproto/dev-env' import AtpAgent from '@atproto/api' import { FLAG, @@ -11,13 +11,13 @@ import { import { forSnapshot } from '../_util' import basicSeed from '../seeds/basic' -describe('pds admin get moderation action view', () => { - let network: TestNetworkNoAppView +describe('admin get moderation action view', () => { + let network: TestNetwork let agent: AtpAgent let sc: SeedClient beforeAll(async () => { - network = await TestNetworkNoAppView.create({ + network = await TestNetwork.create({ dbPostgresSchema: 'views_admin_get_moderation_report', }) agent = network.pds.getClient() diff --git a/packages/pds/tests/admin/get-moderation-reports.test.ts b/packages/bsky/tests/admin/get-moderation-reports.test.ts similarity index 98% rename from packages/pds/tests/admin/get-moderation-reports.test.ts rename to packages/bsky/tests/admin/get-moderation-reports.test.ts index aac3560c048..64313130047 100644 --- a/packages/pds/tests/admin/get-moderation-reports.test.ts +++ b/packages/bsky/tests/admin/get-moderation-reports.test.ts @@ -1,4 +1,4 @@ -import { TestNetworkNoAppView, SeedClient } from '@atproto/dev-env' +import { SeedClient, TestNetwork } from '@atproto/dev-env' import AtpAgent from '@atproto/api' import { ACKNOWLEDGE, @@ -12,13 +12,13 @@ import { import { forSnapshot, paginateAll } from '../_util' import basicSeed from '../seeds/basic' -describe('pds admin get moderation reports view', () => { - let network: TestNetworkNoAppView +describe('admin get moderation reports view', () => { + let network: TestNetwork let agent: AtpAgent let sc: SeedClient beforeAll(async () => { - network = await TestNetworkNoAppView.create({ + network = await TestNetwork.create({ dbPostgresSchema: 'views_admin_get_moderation_reports', }) agent = network.pds.getClient() diff --git a/packages/pds/tests/admin/get-record.test.ts b/packages/bsky/tests/admin/get-record.test.ts similarity index 94% rename from packages/pds/tests/admin/get-record.test.ts rename to packages/bsky/tests/admin/get-record.test.ts index 350709971fc..94ae22b1694 100644 --- a/packages/pds/tests/admin/get-record.test.ts +++ b/packages/bsky/tests/admin/get-record.test.ts @@ -1,4 +1,4 @@ -import { TestNetworkNoAppView, SeedClient } from '@atproto/dev-env' +import { SeedClient, TestNetwork } from '@atproto/dev-env' import AtpAgent from '@atproto/api' import { AtUri } from '@atproto/syntax' import { @@ -12,13 +12,13 @@ import { import { forSnapshot } from '../_util' import basicSeed from '../seeds/basic' -describe('pds admin get record view', () => { - let network: TestNetworkNoAppView +describe('admin get record view', () => { + let network: TestNetwork let agent: AtpAgent let sc: SeedClient beforeAll(async () => { - network = await TestNetworkNoAppView.create({ + network = await TestNetwork.create({ dbPostgresSchema: 'views_admin_get_record', }) agent = network.pds.getClient() diff --git a/packages/pds/tests/admin/get-repo.test.ts b/packages/bsky/tests/admin/get-repo.test.ts similarity index 93% rename from packages/pds/tests/admin/get-repo.test.ts rename to packages/bsky/tests/admin/get-repo.test.ts index 9467643973e..3c1e909a4ab 100644 --- a/packages/pds/tests/admin/get-repo.test.ts +++ b/packages/bsky/tests/admin/get-repo.test.ts @@ -1,4 +1,4 @@ -import { TestNetworkNoAppView, SeedClient } from '@atproto/dev-env' +import { SeedClient, TestNetwork } from '@atproto/dev-env' import AtpAgent from '@atproto/api' import { ACKNOWLEDGE, @@ -11,13 +11,13 @@ import { import { forSnapshot } from '../_util' import basicSeed from '../seeds/basic' -describe('pds admin get repo view', () => { - let network: TestNetworkNoAppView +describe('admin get repo view', () => { + let network: TestNetwork let agent: AtpAgent let sc: SeedClient beforeAll(async () => { - network = await TestNetworkNoAppView.create({ + network = await TestNetwork.create({ dbPostgresSchema: 'views_admin_get_repo', }) agent = network.pds.getClient() diff --git a/packages/bsky/tests/moderation.test.ts b/packages/bsky/tests/admin/moderation.test.ts similarity index 91% rename from packages/bsky/tests/moderation.test.ts rename to packages/bsky/tests/admin/moderation.test.ts index e1af045693b..05200087e3c 100644 --- a/packages/bsky/tests/moderation.test.ts +++ b/packages/bsky/tests/admin/moderation.test.ts @@ -2,23 +2,24 @@ import { TestNetwork, ImageRef, RecordRef, SeedClient } from '@atproto/dev-env' import { TID, cidForCbor } from '@atproto/common' import AtpAgent, { ComAtprotoAdminTakeModerationAction } from '@atproto/api' import { AtUri } from '@atproto/syntax' -import { forSnapshot } from './_util' -import basicSeed from './seeds/basic' +import { forSnapshot } from '../_util' +import basicSeed from '../seeds/basic' import { ACKNOWLEDGE, ESCALATE, FLAG, TAKEDOWN, -} from '../src/lexicon/types/com/atproto/admin/defs' +} from '../../src/lexicon/types/com/atproto/admin/defs' import { REASONOTHER, REASONSPAM, -} from '../src/lexicon/types/com/atproto/moderation/defs' -import { PeriodicModerationActionReversal } from '../src' +} from '../../src/lexicon/types/com/atproto/moderation/defs' +import { PeriodicModerationActionReversal } from '../../src' describe('moderation', () => { let network: TestNetwork let agent: AtpAgent + let pdsAgent: AtpAgent let sc: SeedClient beforeAll(async () => { @@ -26,6 +27,7 @@ describe('moderation', () => { dbPostgresSchema: 'bsky_moderation', }) agent = network.bsky.getClient() + pdsAgent = network.pds.getClient() sc = network.getSeedClient() await basicSeed(sc) await network.processAll() @@ -960,6 +962,82 @@ describe('moderation', () => { ) }) + it('fans out repo takedowns to pds', async () => { + const { data: action } = + await agent.api.com.atproto.admin.takeModerationAction( + { + action: TAKEDOWN, + createdBy: 'did:example:moderator', + reason: 'Y', + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.bob, + }, + }, + { + encoding: 'application/json', + headers: network.bsky.adminAuthHeaders(), + }, + ) + + const res1 = await pdsAgent.api.com.atproto.admin.getSubjectStatus( + { + did: sc.dids.bob, + }, + { headers: network.pds.adminAuthHeaders() }, + ) + expect(res1.data.takedown?.applied).toBe(true) + + // cleanup + await reverse(action.id) + + const res2 = await pdsAgent.api.com.atproto.admin.getSubjectStatus( + { + did: sc.dids.bob, + }, + { headers: network.pds.adminAuthHeaders() }, + ) + expect(res2.data.takedown?.applied).toBe(false) + }) + + it('fans out record takedowns to pds', async () => { + const post = sc.posts[sc.dids.bob][0] + const uri = post.ref.uriStr + const cid = post.ref.cidStr + const { data: action } = + await agent.api.com.atproto.admin.takeModerationAction( + { + action: TAKEDOWN, + createdBy: 'did:example:moderator', + reason: 'Y', + subject: { + $type: 'com.atproto.repo.strongRef', + uri, + cid, + }, + }, + { + encoding: 'application/json', + headers: network.bsky.adminAuthHeaders(), + }, + ) + + const res1 = await pdsAgent.api.com.atproto.admin.getSubjectStatus( + { uri }, + { headers: network.pds.adminAuthHeaders() }, + ) + expect(res1.data.takedown?.applied).toBe(true) + + // cleanup + await reverse(action.id) + + const res2 = await pdsAgent.api.com.atproto.admin.getSubjectStatus( + { uri }, + { headers: network.pds.adminAuthHeaders() }, + ) + expect(res2.data.takedown?.applied).toBe(false) + }) + it('allows full moderators to takedown.', async () => { const { data: action } = await agent.api.com.atproto.admin.takeModerationAction( @@ -1159,6 +1237,17 @@ describe('moderation', () => { expect(await fetchImage.json()).toEqual({ message: 'Image not found' }) }) + it('fans takedown out to pds', async () => { + const res = await pdsAgent.api.com.atproto.admin.getSubjectStatus( + { + did: sc.dids.carol, + blob: blob.image.ref.toString(), + }, + { headers: network.pds.adminAuthHeaders() }, + ) + expect(res.data.takedown?.applied).toBe(true) + }) + it('restores blob when action is reversed.', async () => { await agent.api.com.atproto.admin.reverseModerationAction( { @@ -1183,5 +1272,16 @@ describe('moderation', () => { const size = Number(fetchImage.headers.get('content-length')) expect(size).toBeGreaterThan(9000) }) + + it('fans reversal out to pds', async () => { + const res = await pdsAgent.api.com.atproto.admin.getSubjectStatus( + { + did: sc.dids.carol, + blob: blob.image.ref.toString(), + }, + { headers: network.pds.adminAuthHeaders() }, + ) + expect(res.data.takedown?.applied).toBe(false) + }) }) }) diff --git a/packages/pds/tests/admin/repo-search.test.ts b/packages/bsky/tests/admin/repo-search.test.ts similarity index 84% rename from packages/pds/tests/admin/repo-search.test.ts rename to packages/bsky/tests/admin/repo-search.test.ts index b95dde6063d..fab63257147 100644 --- a/packages/pds/tests/admin/repo-search.test.ts +++ b/packages/bsky/tests/admin/repo-search.test.ts @@ -1,17 +1,17 @@ -import { TestNetworkNoAppView, SeedClient } from '@atproto/dev-env' +import { SeedClient, TestNetwork } from '@atproto/dev-env' import AtpAgent from '@atproto/api' import { TAKEDOWN } from '@atproto/api/src/client/types/com/atproto/admin/defs' import { paginateAll } from '../_util' import usersBulkSeed from '../seeds/users-bulk' -describe('pds admin repo search view', () => { - let network: TestNetworkNoAppView +describe('admin repo search view', () => { + let network: TestNetwork let agent: AtpAgent let sc: SeedClient let headers: { [s: string]: string } beforeAll(async () => { - network = await TestNetworkNoAppView.create({ + network = await TestNetwork.create({ dbPostgresSchema: 'views_admin_repo_search', }) agent = network.pds.getClient() @@ -74,19 +74,6 @@ describe('pds admin repo search view', () => { expect(res.data.repos[0].did).toEqual(term) }) - it('finds repo by email', async () => { - const did = sc.dids['cara-wiegand69.test'] - const { email } = sc.accounts[did] - const res = await agent.api.com.atproto.admin.searchRepos( - { term: email }, - { headers }, - ) - - expect(res.data.repos.length).toEqual(1) - expect(res.data.repos[0].did).toEqual(did) - expect(res.data.repos[0].email).toEqual(email) - }) - it('paginates with term', async () => { const results = (results) => results.flatMap((res) => res.users) const paginator = async (cursor?: string) => { diff --git a/packages/bsky/tests/auto-moderator/takedowns.test.ts b/packages/bsky/tests/auto-moderator/takedowns.test.ts index 6c7b0669b77..d2bc8d4a2a2 100644 --- a/packages/bsky/tests/auto-moderator/takedowns.test.ts +++ b/packages/bsky/tests/auto-moderator/takedowns.test.ts @@ -96,9 +96,9 @@ describe('takedowner', () => { const recordPds = await network.pds.ctx.db.db .selectFrom('record') .where('uri', '=', post.ref.uriStr) - .select('takedownId') + .select('takedownRef') .executeTakeFirst() - expect(recordPds?.takedownId).toEqual(modAction.id) + expect(recordPds?.takedownRef).toEqual(modAction.id.toString()) expect(testInvalidator.invalidated.length).toBe(1) expect(testInvalidator.invalidated[0].subject).toBe( @@ -138,9 +138,9 @@ describe('takedowner', () => { const recordPds = await network.pds.ctx.db.db .selectFrom('record') .where('uri', '=', res.data.uri) - .select('takedownId') + .select('takedownRef') .executeTakeFirst() - expect(recordPds?.takedownId).toEqual(modAction.id) + expect(recordPds?.takedownRef).toEqual(modAction.id.toString()) expect(testInvalidator.invalidated.length).toBe(2) expect(testInvalidator.invalidated[1].subject).toBe( diff --git a/packages/bsky/tests/views/actor-search.test.ts b/packages/bsky/tests/views/actor-search.test.ts index 5562f747700..0f22eff0513 100644 --- a/packages/bsky/tests/views/actor-search.test.ts +++ b/packages/bsky/tests/views/actor-search.test.ts @@ -5,7 +5,9 @@ import { TAKEDOWN } from '@atproto/api/src/client/types/com/atproto/admin/defs' import { forSnapshot, paginateAll, stripViewer } from '../_util' import usersBulkSeed from '../seeds/users-bulk' -describe('pds actor search views', () => { +// @NOTE skipped to help with CI failures +// The search code is not used in production & we should switch it out for tests on the search proxy interface +describe.skip('pds actor search views', () => { let network: TestNetwork let agent: AtpAgent let sc: SeedClient diff --git a/packages/bsky/tests/views/posts.test.ts b/packages/bsky/tests/views/posts.test.ts index a2710a02cf7..69bade5b91a 100644 --- a/packages/bsky/tests/views/posts.test.ts +++ b/packages/bsky/tests/views/posts.test.ts @@ -18,7 +18,6 @@ describe('pds posts views', () => { sc = network.getSeedClient() await basicSeed(sc) await network.processAll() - await network.bsky.processAll() }) afterAll(async () => { @@ -98,7 +97,6 @@ describe('pds posts views', () => { ) await network.processAll() - await network.bsky.processAll() const { data } = await agent.api.app.bsky.feed.getPosts({ uris: [uri] }) diff --git a/packages/dev-env/src/bsky.ts b/packages/dev-env/src/bsky.ts index ac09e88419d..968bceb9536 100644 --- a/packages/dev-env/src/bsky.ts +++ b/packages/dev-env/src/bsky.ts @@ -9,6 +9,7 @@ import { Client as PlcClient } from '@did-plc/lib' import { BskyConfig } from './types' import { uniqueLockId } from './util' import { TestNetworkNoAppView } from './network-no-appview' +import { ADMIN_PASSWORD, MOD_PASSWORD, TRIAGE_PASSWORD } from './const' export class TestBsky { constructor( @@ -43,9 +44,9 @@ export class TestBsky { didCacheMaxTTL: DAY, ...cfg, // Each test suite gets its own lock id for the repo subscription - adminPassword: 'admin-pass', - moderatorPassword: 'moderator-pass', - triagePassword: 'triage-pass', + adminPassword: ADMIN_PASSWORD, + moderatorPassword: MOD_PASSWORD, + triagePassword: TRIAGE_PASSWORD, labelerDid: 'did:example:labeler', feedGenDid: 'did:example:feedGen', }) @@ -78,6 +79,7 @@ export class TestBsky { config, algos: cfg.algos, imgInvalidator: cfg.imgInvalidator, + signingKey: serviceKeypair, }) // indexer const ns = cfg.dbPostgresSchema diff --git a/packages/dev-env/src/const.ts b/packages/dev-env/src/const.ts new file mode 100644 index 00000000000..137b8efd2c5 --- /dev/null +++ b/packages/dev-env/src/const.ts @@ -0,0 +1,3 @@ +export const ADMIN_PASSWORD = 'admin-pass' +export const MOD_PASSWORD = 'mod-pass' +export const TRIAGE_PASSWORD = 'triage-pass' diff --git a/packages/dev-env/src/pds.ts b/packages/dev-env/src/pds.ts index 501ae390cdb..ada6dbec0a4 100644 --- a/packages/dev-env/src/pds.ts +++ b/packages/dev-env/src/pds.ts @@ -7,10 +7,7 @@ import { Secp256k1Keypair, randomStr } from '@atproto/crypto' import { AtpAgent } from '@atproto/api' import { PdsConfig } from './types' import { uniqueLockId } from './util' - -const ADMIN_PASSWORD = 'admin-pass' -const MOD_PASSWORD = 'mod-pass' -const TRIAGE_PASSWORD = 'triage-pass' +import { ADMIN_PASSWORD, MOD_PASSWORD, TRIAGE_PASSWORD } from './const' export class TestPds { constructor( diff --git a/packages/identity/src/did/atproto-data.ts b/packages/identity/src/did/atproto-data.ts index 6881bda48dc..c03f76ef598 100644 --- a/packages/identity/src/did/atproto-data.ts +++ b/packages/identity/src/did/atproto-data.ts @@ -62,3 +62,11 @@ export const ensureAtpDocument = (doc: DidDocument): AtprotoData => { } return { did, signingKey, handle, pds } } + +export const ensureAtprotoKey = (doc: DidDocument): string => { + const { signingKey } = parseToAtprotoDocument(doc) + if (!signingKey) { + throw new Error(`Could not parse signingKey from doc: ${doc}`) + } + return signingKey +} diff --git a/packages/identity/src/did/base-resolver.ts b/packages/identity/src/did/base-resolver.ts index 765f354213c..825dddac566 100644 --- a/packages/identity/src/did/base-resolver.ts +++ b/packages/identity/src/did/base-resolver.ts @@ -83,8 +83,8 @@ export abstract class BaseResolver { if (did.startsWith('did:key:')) { return did } else { - const data = await this.resolveAtprotoData(did, forceRefresh) - return data.signingKey + const didDocument = await this.ensureResolve(did, forceRefresh) + return atprotoData.ensureAtprotoKey(didDocument) } } diff --git a/packages/pds/src/api/com/atproto/admin/getAccountInfo.ts b/packages/pds/src/api/com/atproto/admin/getAccountInfo.ts new file mode 100644 index 00000000000..cf751d08df4 --- /dev/null +++ b/packages/pds/src/api/com/atproto/admin/getAccountInfo.ts @@ -0,0 +1,22 @@ +import { Server } from '../../../../lexicon' +import AppContext from '../../../../context' +import { InvalidRequestError } from '@atproto/xrpc-server' +import { ensureValidAdminAud } from '../../../../auth-verifier' + +export default function (server: Server, ctx: AppContext) { + server.com.atproto.admin.getAccountInfo({ + auth: ctx.authVerifier.roleOrAdminService, + handler: async ({ params, auth }) => { + // any admin role auth can get account info, but verify aud on service jwt + ensureValidAdminAud(auth, params.did) + const view = await ctx.services.account(ctx.db).adminView(params.did) + if (!view) { + throw new InvalidRequestError('Account not found', 'NotFound') + } + return { + encoding: 'application/json', + body: view, + } + }, + }) +} diff --git a/packages/pds/src/api/com/atproto/admin/getModerationAction.ts b/packages/pds/src/api/com/atproto/admin/getModerationAction.ts index 02d14a0ce1c..50b9fcde5ad 100644 --- a/packages/pds/src/api/com/atproto/admin/getModerationAction.ts +++ b/packages/pds/src/api/com/atproto/admin/getModerationAction.ts @@ -1,54 +1,19 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { authPassthru, mergeRepoViewPdsDetails } from './util' -import { isRepoView } from '../../../../lexicon/types/com/atproto/admin/defs' +import { authPassthru } from './util' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.getModerationAction({ auth: ctx.authVerifier.role, - handler: async ({ req, params, auth }) => { - const access = auth.credentials - const { db, services } = ctx - const accountService = services.account(db) - const moderationService = services.moderation(db) - - if (ctx.cfg.bskyAppView.proxyModeration) { - const { data: resultAppview } = - await ctx.appViewAgent.com.atproto.admin.getModerationAction( - params, - authPassthru(req), - ) - // merge local repo state for subject if available - if (isRepoView(resultAppview.subject)) { - const account = await accountService.getAccount( - resultAppview.subject.did, - true, - ) - const repo = - account && - (await moderationService.views.repo(account, { - includeEmails: access.moderator, - })) - if (repo) { - resultAppview.subject = mergeRepoViewPdsDetails( - resultAppview.subject, - repo, - ) - } - } - return { - encoding: 'application/json', - body: resultAppview, - } - } - - const { id } = params - const result = await moderationService.getActionOrThrow(id) + handler: async ({ req, params }) => { + const { data: resultAppview } = + await ctx.appViewAgent.com.atproto.admin.getModerationAction( + params, + authPassthru(req), + ) return { encoding: 'application/json', - body: await moderationService.views.actionDetail(result, { - includeEmails: access.moderator, - }), + body: resultAppview, } }, }) diff --git a/packages/pds/src/api/com/atproto/admin/getModerationActions.ts b/packages/pds/src/api/com/atproto/admin/getModerationActions.ts index f36f44c4917..d9cf61ba1ee 100644 --- a/packages/pds/src/api/com/atproto/admin/getModerationActions.ts +++ b/packages/pds/src/api/com/atproto/admin/getModerationActions.ts @@ -6,32 +6,14 @@ export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.getModerationActions({ auth: ctx.authVerifier.role, handler: async ({ req, params }) => { - if (ctx.cfg.bskyAppView.proxyModeration) { - const { data: result } = - await ctx.appViewAgent.com.atproto.admin.getModerationActions( - params, - authPassthru(req), - ) - return { - encoding: 'application/json', - body: result, - } - } - - const { db, services } = ctx - const { subject, limit = 50, cursor } = params - const moderationService = services.moderation(db) - const results = await moderationService.getActions({ - subject, - limit, - cursor, - }) + const { data: result } = + await ctx.appViewAgent.com.atproto.admin.getModerationActions( + params, + authPassthru(req), + ) return { encoding: 'application/json', - body: { - cursor: results.at(-1)?.id.toString() ?? undefined, - actions: await moderationService.views.action(results), - }, + body: result, } }, }) diff --git a/packages/pds/src/api/com/atproto/admin/getModerationReport.ts b/packages/pds/src/api/com/atproto/admin/getModerationReport.ts index 11a7a943542..681679c87db 100644 --- a/packages/pds/src/api/com/atproto/admin/getModerationReport.ts +++ b/packages/pds/src/api/com/atproto/admin/getModerationReport.ts @@ -1,54 +1,19 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { authPassthru, mergeRepoViewPdsDetails } from './util' -import { isRepoView } from '../../../../lexicon/types/com/atproto/admin/defs' +import { authPassthru } from './util' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.getModerationReport({ auth: ctx.authVerifier.role, - handler: async ({ req, params, auth }) => { - const access = auth.credentials - const { db, services } = ctx - const accountService = services.account(db) - const moderationService = services.moderation(db) - - if (ctx.cfg.bskyAppView.proxyModeration) { - const { data: resultAppview } = - await ctx.appViewAgent.com.atproto.admin.getModerationReport( - params, - authPassthru(req), - ) - // merge local repo state for subject if available - if (isRepoView(resultAppview.subject)) { - const account = await accountService.getAccount( - resultAppview.subject.did, - true, - ) - const repo = - account && - (await moderationService.views.repo(account, { - includeEmails: access.moderator, - })) - if (repo) { - resultAppview.subject = mergeRepoViewPdsDetails( - resultAppview.subject, - repo, - ) - } - } - return { - encoding: 'application/json', - body: resultAppview, - } - } - - const { id } = params - const result = await moderationService.getReportOrThrow(id) + handler: async ({ req, params }) => { + const { data: resultAppview } = + await ctx.appViewAgent.com.atproto.admin.getModerationReport( + params, + authPassthru(req), + ) return { encoding: 'application/json', - body: await moderationService.views.reportDetail(result, { - includeEmails: access.moderator, - }), + body: resultAppview, } }, }) diff --git a/packages/pds/src/api/com/atproto/admin/getModerationReports.ts b/packages/pds/src/api/com/atproto/admin/getModerationReports.ts index 20a7bb6c88d..a213504d840 100644 --- a/packages/pds/src/api/com/atproto/admin/getModerationReports.ts +++ b/packages/pds/src/api/com/atproto/admin/getModerationReports.ts @@ -6,48 +6,14 @@ export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.getModerationReports({ auth: ctx.authVerifier.role, handler: async ({ req, params }) => { - if (ctx.cfg.bskyAppView.proxyModeration) { - const { data: result } = - await ctx.appViewAgent.com.atproto.admin.getModerationReports( - params, - authPassthru(req), - ) - return { - encoding: 'application/json', - body: result, - } - } - - const { db, services } = ctx - const { - subject, - resolved, - actionType, - limit = 50, - cursor, - ignoreSubjects = [], - reverse = false, - reporters = [], - actionedBy, - } = params - const moderationService = services.moderation(db) - const results = await moderationService.getReports({ - subject, - resolved, - actionType, - limit, - cursor, - ignoreSubjects, - reverse, - reporters, - actionedBy, - }) + const { data: result } = + await ctx.appViewAgent.com.atproto.admin.getModerationReports( + params, + authPassthru(req), + ) return { encoding: 'application/json', - body: { - cursor: results.at(-1)?.id.toString() ?? undefined, - reports: await moderationService.views.report(results), - }, + body: result, } }, }) diff --git a/packages/pds/src/api/com/atproto/admin/getRecord.ts b/packages/pds/src/api/com/atproto/admin/getRecord.ts index 9ec3a0606ae..30075a1d2ab 100644 --- a/packages/pds/src/api/com/atproto/admin/getRecord.ts +++ b/packages/pds/src/api/com/atproto/admin/getRecord.ts @@ -1,57 +1,19 @@ -import { AtUri } from '@atproto/syntax' -import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { authPassthru, mergeRepoViewPdsDetails } from './util' +import { authPassthru } from './util' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.getRecord({ auth: ctx.authVerifier.role, - handler: async ({ req, params, auth }) => { - const access = auth.credentials - const { db, services } = ctx - const { uri, cid } = params - const result = await services - .record(db) - .getRecord(new AtUri(uri), cid ?? null, true) - const recordDetail = - result && - (await services.moderation(db).views.recordDetail(result, { - includeEmails: access.moderator, - })) - - if (ctx.cfg.bskyAppView.proxyModeration) { - try { - const { data: recordDetailAppview } = - await ctx.appViewAgent.com.atproto.admin.getRecord( - params, - authPassthru(req), - ) - if (recordDetail) { - recordDetailAppview.repo = mergeRepoViewPdsDetails( - recordDetailAppview.repo, - recordDetail.repo, - ) - } - return { - encoding: 'application/json', - body: recordDetailAppview, - } - } catch (err) { - if (err && err['error'] === 'RecordNotFound') { - throw new InvalidRequestError('Record not found', 'RecordNotFound') - } else { - throw err - } - } - } - - if (!recordDetail) { - throw new InvalidRequestError('Record not found', 'RecordNotFound') - } + handler: async ({ req, params }) => { + const { data: recordDetailAppview } = + await ctx.appViewAgent.com.atproto.admin.getRecord( + params, + authPassthru(req), + ) return { encoding: 'application/json', - body: recordDetail, + body: recordDetailAppview, } }, }) diff --git a/packages/pds/src/api/com/atproto/admin/getRepo.ts b/packages/pds/src/api/com/atproto/admin/getRepo.ts index 9c786772a36..3eb2e7c14c8 100644 --- a/packages/pds/src/api/com/atproto/admin/getRepo.ts +++ b/packages/pds/src/api/com/atproto/admin/getRepo.ts @@ -1,54 +1,18 @@ -import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { authPassthru, mergeRepoViewPdsDetails } from './util' +import { authPassthru } from './util' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.getRepo({ auth: ctx.authVerifier.role, - handler: async ({ req, params, auth }) => { - const access = auth.credentials - const { db, services } = ctx - const { did } = params - const result = await services.account(db).getAccount(did, true) - const repoDetail = - result && - (await services.moderation(db).views.repoDetail(result, { - includeEmails: access.moderator, - })) - - if (ctx.cfg.bskyAppView.proxyModeration) { - try { - let { data: repoDetailAppview } = - await ctx.appViewAgent.com.atproto.admin.getRepo( - params, - authPassthru(req), - ) - if (repoDetail) { - repoDetailAppview = mergeRepoViewPdsDetails( - repoDetailAppview, - repoDetail, - ) - } - return { - encoding: 'application/json', - body: repoDetailAppview, - } - } catch (err) { - if (err && err['error'] === 'RepoNotFound') { - throw new InvalidRequestError('Repo not found', 'RepoNotFound') - } else { - throw err - } - } - } - - if (!repoDetail) { - throw new InvalidRequestError('Repo not found', 'RepoNotFound') - } + handler: async ({ req, params }) => { + const res = await ctx.appViewAgent.com.atproto.admin.getRepo( + params, + authPassthru(req), + ) return { encoding: 'application/json', - body: repoDetail, + body: res.data, } }, }) diff --git a/packages/pds/src/api/com/atproto/admin/getSubjectStatus.ts b/packages/pds/src/api/com/atproto/admin/getSubjectStatus.ts new file mode 100644 index 00000000000..20ded7bc747 --- /dev/null +++ b/packages/pds/src/api/com/atproto/admin/getSubjectStatus.ts @@ -0,0 +1,43 @@ +import { CID } from 'multiformats/cid' +import { AtUri } from '@atproto/syntax' +import { InvalidRequestError } from '@atproto/xrpc-server' +import { Server } from '../../../../lexicon' +import AppContext from '../../../../context' +import { OutputSchema } from '../../../../lexicon/types/com/atproto/admin/getSubjectStatus' +import { ensureValidAdminAud } from '../../../../auth-verifier' + +export default function (server: Server, ctx: AppContext) { + server.com.atproto.admin.getSubjectStatus({ + auth: ctx.authVerifier.roleOrAdminService, + handler: async ({ params, auth }) => { + const { did, uri, blob } = params + const modSrvc = ctx.services.moderation(ctx.db) + let body: OutputSchema | null + if (blob) { + if (!did) { + throw new InvalidRequestError( + 'Must provide a did to request blob state', + ) + } + ensureValidAdminAud(auth, did) + body = await modSrvc.getBlobTakedownState(did, CID.parse(blob)) + } else if (uri) { + const parsedUri = new AtUri(uri) + ensureValidAdminAud(auth, parsedUri.hostname) + body = await modSrvc.getRecordTakedownState(parsedUri) + } else if (did) { + ensureValidAdminAud(auth, did) + body = await modSrvc.getRepoTakedownState(did) + } else { + throw new InvalidRequestError('No provided subject') + } + if (body === null) { + throw new InvalidRequestError('Subject not found', 'NotFound') + } + return { + encoding: 'application/json', + body, + } + }, + }) +} diff --git a/packages/pds/src/api/com/atproto/admin/index.ts b/packages/pds/src/api/com/atproto/admin/index.ts index 84d1fe3218a..29fdec10efe 100644 --- a/packages/pds/src/api/com/atproto/admin/index.ts +++ b/packages/pds/src/api/com/atproto/admin/index.ts @@ -3,6 +3,9 @@ import { Server } from '../../../../lexicon' import resolveModerationReports from './resolveModerationReports' import reverseModerationAction from './reverseModerationAction' import takeModerationAction from './takeModerationAction' +import updateSubjectStatus from './updateSubjectStatus' +import getSubjectStatus from './getSubjectStatus' +import getAccountInfo from './getAccountInfo' import searchRepos from './searchRepos' import getRecord from './getRecord' import getRepo from './getRepo' @@ -22,6 +25,9 @@ export default function (server: Server, ctx: AppContext) { resolveModerationReports(server, ctx) reverseModerationAction(server, ctx) takeModerationAction(server, ctx) + updateSubjectStatus(server, ctx) + getSubjectStatus(server, ctx) + getAccountInfo(server, ctx) searchRepos(server, ctx) getRecord(server, ctx) getRepo(server, ctx) diff --git a/packages/pds/src/api/com/atproto/admin/resolveModerationReports.ts b/packages/pds/src/api/com/atproto/admin/resolveModerationReports.ts index 229433fa50c..1246a2364b1 100644 --- a/packages/pds/src/api/com/atproto/admin/resolveModerationReports.ts +++ b/packages/pds/src/api/com/atproto/admin/resolveModerationReports.ts @@ -6,31 +6,14 @@ export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.resolveModerationReports({ auth: ctx.authVerifier.role, handler: async ({ req, input }) => { - if (ctx.cfg.bskyAppView.proxyModeration) { - const { data: result } = - await ctx.appViewAgent.com.atproto.admin.resolveModerationReports( - input.body, - authPassthru(req, true), - ) - return { - encoding: 'application/json', - body: result, - } - } - - const { db, services } = ctx - const moderationService = services.moderation(db) - const { actionId, reportIds, createdBy } = input.body - - const moderationAction = await db.transaction(async (dbTxn) => { - const moderationTxn = services.moderation(dbTxn) - await moderationTxn.resolveReports({ reportIds, actionId, createdBy }) - return await moderationTxn.getActionOrThrow(actionId) - }) - + const { data: result } = + await ctx.appViewAgent.com.atproto.admin.resolveModerationReports( + input.body, + authPassthru(req, true), + ) return { encoding: 'application/json', - body: await moderationService.views.action(moderationAction), + body: result, } }, }) diff --git a/packages/pds/src/api/com/atproto/admin/reverseModerationAction.ts b/packages/pds/src/api/com/atproto/admin/reverseModerationAction.ts index dc5a22e600e..ec52e2c36c6 100644 --- a/packages/pds/src/api/com/atproto/admin/reverseModerationAction.ts +++ b/packages/pds/src/api/com/atproto/admin/reverseModerationAction.ts @@ -1,111 +1,20 @@ -import { AtUri } from '@atproto/syntax' -import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { - isRepoRef, - ACKNOWLEDGE, - ESCALATE, - TAKEDOWN, -} from '../../../../lexicon/types/com/atproto/admin/defs' -import { isMain as isStrongRef } from '../../../../lexicon/types/com/atproto/repo/strongRef' import { authPassthru } from './util' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.reverseModerationAction({ auth: ctx.authVerifier.role, - handler: async ({ req, input, auth }) => { - const access = auth.credentials - const { db, services } = ctx - if (ctx.cfg.bskyAppView.proxyModeration) { - const { data: result } = - await ctx.appViewAgent.com.atproto.admin.reverseModerationAction( - input.body, - authPassthru(req, true), - ) - - const transact = db.transaction(async (dbTxn) => { - const moderationTxn = services.moderation(dbTxn) - // reverse takedowns - if (result.action === TAKEDOWN && isRepoRef(result.subject)) { - await moderationTxn.reverseTakedownRepo({ - did: result.subject.did, - }) - } - if (result.action === TAKEDOWN && isStrongRef(result.subject)) { - await moderationTxn.reverseTakedownRecord({ - uri: new AtUri(result.subject.uri), - }) - } - }) - - try { - await transact - } catch (err) { - req.log.error( - { err, actionId: input.body.id }, - 'proxied moderation action reversal failed', - ) - } - - return { - encoding: 'application/json', - body: result, - } - } - - const moderationService = services.moderation(db) - const { id, createdBy, reason } = input.body - - const moderationAction = await db.transaction(async (dbTxn) => { - const moderationTxn = services.moderation(dbTxn) - const now = new Date() - - const existing = await moderationTxn.getAction(id) - if (!existing) { - throw new InvalidRequestError('Moderation action does not exist') - } - if (existing.reversedAt !== null) { - throw new InvalidRequestError( - 'Moderation action has already been reversed', - ) - } - - // apply access rules - - // if less than moderator access then can only reverse ack and escalation actions - if ( - !access.moderator && - ![ACKNOWLEDGE, ESCALATE].includes(existing.action) - ) { - throw new AuthRequiredError( - 'Must be a full moderator to reverse this type of action', - ) - } - // if less than moderator access then cannot reverse takedown on an account - if ( - !access.moderator && - existing.action === TAKEDOWN && - existing.subjectType === 'com.atproto.admin.defs#repoRef' - ) { - throw new AuthRequiredError( - 'Must be an admin to reverse an account takedown', - ) - } - - const result = await moderationTxn.revertAction({ - id, - createdAt: now, - createdBy, - reason, - }) - - return result - }) + handler: async ({ req, input }) => { + const { data: result } = + await ctx.appViewAgent.com.atproto.admin.reverseModerationAction( + input.body, + authPassthru(req, true), + ) return { encoding: 'application/json', - body: await moderationService.views.action(moderationAction), + body: result, } }, }) diff --git a/packages/pds/src/api/com/atproto/admin/searchRepos.ts b/packages/pds/src/api/com/atproto/admin/searchRepos.ts index 8195c8c2d98..bf1ab92e3c3 100644 --- a/packages/pds/src/api/com/atproto/admin/searchRepos.ts +++ b/packages/pds/src/api/com/atproto/admin/searchRepos.ts @@ -1,65 +1,21 @@ -import { sql } from 'kysely' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { ListKeyset } from '../../../../services/account' import { authPassthru } from './util' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.searchRepos({ auth: ctx.authVerifier.role, - handler: async ({ req, params, auth }) => { - if (ctx.cfg.bskyAppView.proxyModeration) { - // @TODO merge invite details to this list view. could also add - // support for invitedBy param, which is not supported by appview. - const { data: result } = - await ctx.appViewAgent.com.atproto.admin.searchRepos( - params, - authPassthru(req), - ) - return { - encoding: 'application/json', - body: result, - } - } - - const access = auth.credentials - const { db, services } = ctx - const moderationService = services.moderation(db) - const { limit, cursor, invitedBy } = params - const query = params.q?.trim() ?? params.term?.trim() ?? '' - - const keyset = new ListKeyset(sql``, sql``) - - if (!query) { - const results = await services - .account(db) - .list({ limit, cursor, includeSoftDeleted: true, invitedBy }) - return { - encoding: 'application/json', - body: { - cursor: keyset.packFromResult(results), - repos: await moderationService.views.repo(results, { - includeEmails: access.moderator, - }), - }, - } - } - - const results = await services - .account(db) - .search({ query, limit, cursor, includeSoftDeleted: true }) - + handler: async ({ req, params }) => { + // @TODO merge invite details to this list view. could also add + // support for invitedBy param, which is not supported by appview. + const { data: result } = + await ctx.appViewAgent.com.atproto.admin.searchRepos( + params, + authPassthru(req), + ) return { encoding: 'application/json', - body: { - // For did search, we can only find 1 or no match, cursors can be ignored entirely - cursor: query.startsWith('did:') - ? undefined - : keyset.packFromResult(results), - repos: await moderationService.views.repo(results, { - includeEmails: access.moderator, - }), - }, + body: result, } }, }) diff --git a/packages/pds/src/api/com/atproto/admin/takeModerationAction.ts b/packages/pds/src/api/com/atproto/admin/takeModerationAction.ts index d81bd0233f3..c07e8d04f08 100644 --- a/packages/pds/src/api/com/atproto/admin/takeModerationAction.ts +++ b/packages/pds/src/api/com/atproto/admin/takeModerationAction.ts @@ -1,160 +1,21 @@ -import { CID } from 'multiformats/cid' -import { AtUri } from '@atproto/syntax' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { - isRepoRef, - ACKNOWLEDGE, - ESCALATE, - TAKEDOWN, -} from '../../../../lexicon/types/com/atproto/admin/defs' -import { isMain as isStrongRef } from '../../../../lexicon/types/com/atproto/repo/strongRef' -import { getSubject, getAction } from '../moderation/util' -import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import { authPassthru } from './util' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.takeModerationAction({ auth: ctx.authVerifier.role, - handler: async ({ req, input, auth }) => { - const access = auth.credentials - const { db, services } = ctx - if (ctx.cfg.bskyAppView.proxyModeration) { - const { data: result } = - await ctx.appViewAgent.com.atproto.admin.takeModerationAction( - input.body, - authPassthru(req, true), - ) - - const transact = db.transaction(async (dbTxn) => { - const authTxn = services.auth(dbTxn) - const moderationTxn = services.moderation(dbTxn) - // perform takedowns - if (result.action === TAKEDOWN && isRepoRef(result.subject)) { - await authTxn.revokeRefreshTokensByDid(result.subject.did) - await moderationTxn.takedownRepo({ - takedownId: result.id, - did: result.subject.did, - }) - } - if (result.action === TAKEDOWN && isStrongRef(result.subject)) { - await moderationTxn.takedownRecord({ - takedownId: result.id, - uri: new AtUri(result.subject.uri), - blobCids: result.subjectBlobCids.map((cid) => CID.parse(cid)), - }) - } - }) - - try { - await transact - } catch (err) { - req.log.error( - { err, actionId: result.id }, - 'proxied moderation action failed', - ) - } - - return { - encoding: 'application/json', - body: result, - } - } - - const moderationService = services.moderation(db) - const { - action, - subject, - reason, - createdBy, - createLabelVals, - negateLabelVals, - subjectBlobCids, - durationInHours, - } = input.body - - // apply access rules - - // if less than admin access then can not takedown an account - if (!access.moderator && action === TAKEDOWN && 'did' in subject) { - throw new AuthRequiredError( - 'Must be a full moderator to perform an account takedown', - ) - } - // if less than moderator access then can only take ack and escalation actions - if (!access.moderator && ![ACKNOWLEDGE, ESCALATE].includes(action)) { - throw new AuthRequiredError( - 'Must be a full moderator to take this type of action', + handler: async ({ req, input }) => { + const { data: result } = + await ctx.appViewAgent.com.atproto.admin.takeModerationAction( + input.body, + authPassthru(req, true), ) - } - // if less than moderator access then can not apply labels - if ( - !access.moderator && - (createLabelVals?.length || negateLabelVals?.length) - ) { - throw new AuthRequiredError('Must be a full moderator to label content') - } - - validateLabels([...(createLabelVals ?? []), ...(negateLabelVals ?? [])]) - - const moderationAction = await db.transaction(async (dbTxn) => { - const authTxn = services.auth(dbTxn) - const moderationTxn = services.moderation(dbTxn) - - const result = await moderationTxn.logAction({ - action: getAction(action), - subject: getSubject(subject), - subjectBlobCids: subjectBlobCids?.map((cid) => CID.parse(cid)) ?? [], - createLabelVals, - negateLabelVals, - createdBy, - reason, - durationInHours, - }) - - if ( - result.action === TAKEDOWN && - result.subjectType === 'com.atproto.admin.defs#repoRef' && - result.subjectDid - ) { - await authTxn.revokeRefreshTokensByDid(result.subjectDid) - await moderationTxn.takedownRepo({ - takedownId: result.id, - did: result.subjectDid, - }) - } - - if ( - result.action === TAKEDOWN && - result.subjectType === 'com.atproto.repo.strongRef' && - result.subjectUri - ) { - await moderationTxn.takedownRecord({ - takedownId: result.id, - uri: new AtUri(result.subjectUri), - blobCids: subjectBlobCids?.map((cid) => CID.parse(cid)) ?? [], - }) - } - - return result - }) return { encoding: 'application/json', - body: await moderationService.views.action(moderationAction), + body: result, } }, }) } - -const validateLabels = (labels: string[]) => { - for (const label of labels) { - for (const char of badChars) { - if (label.includes(char)) { - throw new InvalidRequestError(`Invalid label: ${label}`) - } - } - } -} - -const badChars = [' ', ',', ';', `'`, `"`] diff --git a/packages/pds/src/api/com/atproto/admin/updateSubjectStatus.ts b/packages/pds/src/api/com/atproto/admin/updateSubjectStatus.ts new file mode 100644 index 00000000000..920debba986 --- /dev/null +++ b/packages/pds/src/api/com/atproto/admin/updateSubjectStatus.ts @@ -0,0 +1,59 @@ +import { CID } from 'multiformats/cid' +import { AtUri } from '@atproto/syntax' +import { Server } from '../../../../lexicon' +import AppContext from '../../../../context' +import { + isRepoRef, + isRepoBlobRef, +} from '../../../../lexicon/types/com/atproto/admin/defs' +import { isMain as isStrongRef } from '../../../../lexicon/types/com/atproto/repo/strongRef' +import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' +import { ensureValidAdminAud } from '../../../../auth-verifier' + +export default function (server: Server, ctx: AppContext) { + server.com.atproto.admin.updateSubjectStatus({ + auth: ctx.authVerifier.roleOrAdminService, + handler: async ({ input, auth }) => { + // if less than moderator access then cannot perform a takedown + if (auth.credentials.type === 'role' && !auth.credentials.moderator) { + throw new AuthRequiredError( + 'Must be a full moderator to update subject state', + ) + } + + const { subject, takedown } = input.body + if (takedown) { + const modSrvc = ctx.services.moderation(ctx.db) + const authSrvc = ctx.services.auth(ctx.db) + if (isRepoRef(subject)) { + ensureValidAdminAud(auth, subject.did) + await Promise.all([ + modSrvc.updateRepoTakedownState(subject.did, takedown), + authSrvc.revokeRefreshTokensByDid(subject.did), + ]) + } else if (isStrongRef(subject)) { + const uri = new AtUri(subject.uri) + ensureValidAdminAud(auth, uri.hostname) + await modSrvc.updateRecordTakedownState(uri, takedown) + } else if (isRepoBlobRef(subject)) { + ensureValidAdminAud(auth, subject.did) + await modSrvc.updateBlobTakedownState( + subject.did, + CID.parse(subject.cid), + takedown, + ) + } else { + throw new InvalidRequestError('Invalid subject') + } + } + + return { + encoding: 'application/json', + body: { + subject, + takedown, + }, + } + }, + }) +} diff --git a/packages/pds/src/api/com/atproto/moderation/createReport.ts b/packages/pds/src/api/com/atproto/moderation/createReport.ts index ecdcc6e87cf..315b72c080a 100644 --- a/packages/pds/src/api/com/atproto/moderation/createReport.ts +++ b/packages/pds/src/api/com/atproto/moderation/createReport.ts @@ -1,43 +1,19 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { getReasonType, getSubject } from './util' export default function (server: Server, ctx: AppContext) { server.com.atproto.moderation.createReport({ auth: ctx.authVerifier.accessCheckTakedown, handler: async ({ input, auth }) => { const requester = auth.credentials.did - - if (ctx.cfg.bskyAppView.proxyModeration) { - const { data: result } = - await ctx.appViewAgent.com.atproto.moderation.createReport( - input.body, - { - ...(await ctx.serviceAuthHeaders(requester)), - encoding: 'application/json', - }, - ) - return { + const { data: result } = + await ctx.appViewAgent.com.atproto.moderation.createReport(input.body, { + ...(await ctx.serviceAuthHeaders(requester)), encoding: 'application/json', - body: result, - } - } - - const { db, services } = ctx - const { reasonType, reason, subject } = input.body - - const moderationService = services.moderation(db) - - const report = await moderationService.report({ - reasonType: getReasonType(reasonType), - reason, - subject: getSubject(subject), - reportedBy: requester, - }) - + }) return { encoding: 'application/json', - body: moderationService.views.reportPublic(report), + body: result, } }, }) diff --git a/packages/pds/src/api/com/atproto/repo/getRecord.ts b/packages/pds/src/api/com/atproto/repo/getRecord.ts index 5c99a7226c1..4a333cf0648 100644 --- a/packages/pds/src/api/com/atproto/repo/getRecord.ts +++ b/packages/pds/src/api/com/atproto/repo/getRecord.ts @@ -14,7 +14,7 @@ export default function (server: Server, ctx: AppContext) { const record = await ctx.services .record(ctx.db) .getRecord(uri, cid || null) - if (!record || record.takedownId !== null) { + if (!record || record.takedownRef !== null) { throw new InvalidRequestError(`Could not locate record: ${uri}`) } return { diff --git a/packages/pds/src/api/com/atproto/server/createAccount.ts b/packages/pds/src/api/com/atproto/server/createAccount.ts index 36bdc7b6b86..f1343b3c4e2 100644 --- a/packages/pds/src/api/com/atproto/server/createAccount.ts +++ b/packages/pds/src/api/com/atproto/server/createAccount.ts @@ -151,7 +151,7 @@ export const ensureCodeIsAvailable = async ( qb .selectFrom('repo_root') .selectAll() - .where('takedownId', 'is not', null) + .where('takedownRef', 'is not', null) .whereRef('did', '=', ref('invite_code.forUser')), ) .where('code', '=', inviteCode) diff --git a/packages/pds/src/api/com/atproto/server/deleteAccount.ts b/packages/pds/src/api/com/atproto/server/deleteAccount.ts index 4d12edb1b32..2088c387339 100644 --- a/packages/pds/src/api/com/atproto/server/deleteAccount.ts +++ b/packages/pds/src/api/com/atproto/server/deleteAccount.ts @@ -1,10 +1,9 @@ import { AuthRequiredError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' -import { TAKEDOWN } from '../../../../lexicon/types/com/atproto/admin/defs' import AppContext from '../../../../context' import { MINUTE } from '@atproto/common' -const REASON_ACCT_DELETION = 'ACCOUNT DELETION' +const REASON_ACCT_DELETION = 'account_deletion' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.deleteAccount({ @@ -25,32 +24,17 @@ export default function (server: Server, ctx: AppContext) { .account(ctx.db) .assertValidToken(did, 'delete_account', token) - const now = new Date() await ctx.db.transaction(async (dbTxn) => { const accountService = ctx.services.account(dbTxn) const moderationTxn = ctx.services.moderation(dbTxn) - const [currentAction] = await moderationTxn.getCurrentActions({ did }) - if (currentAction?.action === TAKEDOWN) { - // Do not disturb an existing takedown, continue with account deletion - return await accountService.deleteEmailToken(did, 'delete_account') - } - if (currentAction) { - // Reverse existing action to replace it with a self-takedown - await moderationTxn.logReverseAction({ - id: currentAction.id, - reason: REASON_ACCT_DELETION, - createdBy: did, - createdAt: now, + const currState = await moderationTxn.getRepoTakedownState(did) + // Do not disturb an existing takedown, continue with account deletion + if (currState?.takedown.applied !== true) { + await moderationTxn.updateRepoTakedownState(did, { + applied: true, + ref: REASON_ACCT_DELETION, }) } - const takedown = await moderationTxn.logAction({ - action: TAKEDOWN, - subject: { did }, - reason: REASON_ACCT_DELETION, - createdBy: did, - createdAt: now, - }) - await moderationTxn.takedownRepo({ did, takedownId: takedown.id }) await accountService.deleteEmailToken(did, 'delete_account') }) diff --git a/packages/pds/src/auth-verifier.ts b/packages/pds/src/auth-verifier.ts index 380f94b90b6..dba1550ba0b 100644 --- a/packages/pds/src/auth-verifier.ts +++ b/packages/pds/src/auth-verifier.ts @@ -1,4 +1,9 @@ -import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' +import { + AuthRequiredError, + InvalidRequestError, + verifyJwt as verifyServiceJwt, +} from '@atproto/xrpc-server' +import { IdResolver } from '@atproto/identity' import * as ui8 from 'uint8arrays' import express from 'express' import * as jwt from 'jsonwebtoken' @@ -34,6 +39,14 @@ type RoleOutput = { } } +type AdminServiceOutput = { + credentials: { + type: 'service' + aud: string + iss: string + } +} + type AccessOutput = { credentials: { type: 'access' @@ -65,6 +78,7 @@ export type AuthVerifierOpts = { adminPass: string moderatorPass: string triagePass: string + adminServiceDid: string } export class AuthVerifier { @@ -72,12 +86,18 @@ export class AuthVerifier { private _adminPass: string private _moderatorPass: string private _triagePass: string + public adminServiceDid: string - constructor(public db: Database, opts: AuthVerifierOpts) { + constructor( + public db: Database, + public idResolver: IdResolver, + opts: AuthVerifierOpts, + ) { this._secret = opts.jwtSecret this._adminPass = opts.adminPass this._moderatorPass = opts.moderatorPass this._triagePass = opts.triagePass + this.adminServiceDid = opts.adminServiceDid } // verifiers (arrow fns to preserve scope) @@ -98,7 +118,7 @@ export class AuthVerifier { .selectFrom('user_account') .innerJoin('repo_root', 'repo_root.did', 'user_account.did') .where('user_account.did', '=', result.credentials.did) - .where('repo_root.takedownId', 'is', null) + .where('repo_root.takedownRef', 'is', null) .select('user_account.did') .executeTakeFirst() if (!found) { @@ -178,6 +198,43 @@ export class AuthVerifier { } } + adminService = async (reqCtx: ReqCtx): Promise => { + const jwtStr = bearerTokenFromReq(reqCtx.req) + if (!jwtStr) { + throw new AuthRequiredError('missing jwt', 'MissingJwt') + } + const payload = await verifyServiceJwt( + jwtStr, + null, + async (did: string) => { + if (did !== this.adminServiceDid) { + throw new AuthRequiredError( + 'Untrusted issuer for admin actions', + 'UntrustedIss', + ) + } + return this.idResolver.did.resolveAtprotoKey(did) + }, + ) + return { + credentials: { + type: 'service', + aud: payload.aud, + iss: payload.iss, + }, + } + } + + roleOrAdminService = async ( + reqCtx: ReqCtx, + ): Promise => { + if (isBearerToken(reqCtx.req)) { + return this.adminService(reqCtx) + } else { + return this.role(reqCtx) + } + } + validateBearerToken( req: express.Request, scopes: AuthScope[], @@ -300,3 +357,18 @@ export const parseBasicAuth = ( if (!username || !password) return null return { username, password } } + +export const ensureValidAdminAud = ( + auth: RoleOutput | AdminServiceOutput, + subjectDid: string, +) => { + if ( + auth.credentials.type === 'service' && + auth.credentials.aud !== subjectDid + ) { + throw new AuthRequiredError( + 'jwt audience does not match account did', + 'BadJwtAudience', + ) + } +} diff --git a/packages/pds/src/context.ts b/packages/pds/src/context.ts index 42277c12bbf..b181ea8b3ad 100644 --- a/packages/pds/src/context.ts +++ b/packages/pds/src/context.ts @@ -151,11 +151,12 @@ export class AppContext { const appViewAgent = new AtpAgent({ service: cfg.bskyAppView.url }) - const authVerifier = new AuthVerifier(db, { + const authVerifier = new AuthVerifier(db, idResolver, { jwtSecret: secrets.jwtSecret, adminPass: secrets.adminPassword, moderatorPass: secrets.moderatorPassword, triagePass: secrets.triagePassword, + adminServiceDid: cfg.bskyAppView.did, }) const repoSigningKey = diff --git a/packages/pds/src/db/migrations/20231011T155513453Z-takedown-ref.ts b/packages/pds/src/db/migrations/20231011T155513453Z-takedown-ref.ts new file mode 100644 index 00000000000..e0d4d16e1f1 --- /dev/null +++ b/packages/pds/src/db/migrations/20231011T155513453Z-takedown-ref.ts @@ -0,0 +1,41 @@ +import { Kysely } from 'kysely' + +export async function up(db: Kysely): Promise { + await db.schema + .alterTable('repo_root') + .addColumn('takedownRef', 'varchar') + .execute() + await db.schema.alterTable('repo_root').dropColumn('takedownId').execute() + + await db.schema + .alterTable('repo_blob') + .addColumn('takedownRef', 'varchar') + .execute() + await db.schema.alterTable('repo_blob').dropColumn('takedownId').execute() + + await db.schema + .alterTable('record') + .addColumn('takedownRef', 'varchar') + .execute() + await db.schema.alterTable('record').dropColumn('takedownId').execute() +} + +export async function down(db: Kysely): Promise { + await db.schema + .alterTable('repo_root') + .addColumn('takedownId', 'integer') + .execute() + await db.schema.alterTable('repo_root').dropColumn('takedownRef').execute() + + await db.schema + .alterTable('repo_blob') + .addColumn('takedownId', 'integer') + .execute() + await db.schema.alterTable('repo_blob').dropColumn('takedownRef').execute() + + await db.schema + .alterTable('record') + .addColumn('takedownId', 'integer') + .execute() + await db.schema.alterTable('record').dropColumn('takedownRef').execute() +} diff --git a/packages/pds/src/db/migrations/index.ts b/packages/pds/src/db/migrations/index.ts index 9aead0d7012..51979099feb 100644 --- a/packages/pds/src/db/migrations/index.ts +++ b/packages/pds/src/db/migrations/index.ts @@ -6,3 +6,4 @@ export * as _20230613T164932261Z from './20230613T164932261Z-init' export * as _20230914T014727199Z from './20230914T014727199Z-repo-v3' export * as _20230926T195532354Z from './20230926T195532354Z-email-tokens' export * as _20230929T213219699Z from './20230929T213219699Z-takedown-id-as-int' +export * as _20231011T155513453Z from './20231011T155513453Z-takedown-ref' diff --git a/packages/pds/src/db/periodic-moderation-action-reversal.ts b/packages/pds/src/db/periodic-moderation-action-reversal.ts deleted file mode 100644 index b3b631de71d..00000000000 --- a/packages/pds/src/db/periodic-moderation-action-reversal.ts +++ /dev/null @@ -1,88 +0,0 @@ -import assert from 'assert' -import { wait } from '@atproto/common' -import { Leader } from './leader' -import { dbLogger } from '../logger' -import AppContext from '../context' -import { ModerationActionRow } from '../services/moderation' - -export const MODERATION_ACTION_REVERSAL_ID = 1011 - -export class PeriodicModerationActionReversal { - leader = new Leader(MODERATION_ACTION_REVERSAL_ID, this.appContext.db) - destroyed = false - - constructor(private appContext: AppContext) {} - - async revertAction(actionRow: ModerationActionRow) { - return this.appContext.db.transaction(async (dbTxn) => { - const moderationTxn = this.appContext.services.moderation(dbTxn) - await moderationTxn.revertAction({ - id: actionRow.id, - createdBy: actionRow.createdBy, - createdAt: new Date(), - reason: `[SCHEDULED_REVERSAL] Reverting action as originally scheduled`, - }) - }) - } - - async findAndRevertDueActions() { - const moderationService = this.appContext.services.moderation( - this.appContext.db, - ) - const actionsDueForReversal = - await moderationService.getActionsDueForReversal() - - // We shouldn't have too many actions due for reversal at any given time, so running in parallel is probably fine - // Internally, each reversal runs within its own transaction - await Promise.allSettled( - actionsDueForReversal.map(this.revertAction.bind(this)), - ) - } - - async run() { - assert( - this.appContext.db.dialect === 'pg', - 'Moderation action reversal can only be run by postgres', - ) - - while (!this.destroyed) { - try { - const { ran } = await this.leader.run(async ({ signal }) => { - while (!signal.aborted) { - // super basic synchronization by agreeing when the intervals land relative to unix timestamp - const now = Date.now() - const intervalMs = 1000 * 60 - const nextIteration = Math.ceil(now / intervalMs) - const nextInMs = nextIteration * intervalMs - now - await wait(nextInMs) - if (signal.aborted) break - await this.findAndRevertDueActions() - } - }) - if (ran && !this.destroyed) { - throw new Error('View maintainer completed, but should be persistent') - } - } catch (err) { - dbLogger.error( - { - err, - lockId: MODERATION_ACTION_REVERSAL_ID, - }, - 'moderation action reversal errored', - ) - } - if (!this.destroyed) { - await wait(10000 + jitter(2000)) - } - } - } - - destroy() { - this.destroyed = true - this.leader.destroy() - } -} - -function jitter(maxMs) { - return Math.round((Math.random() - 0.5) * maxMs * 2) -} diff --git a/packages/pds/src/db/tables/record.ts b/packages/pds/src/db/tables/record.ts index 03f1008ef0f..04d4dd8524f 100644 --- a/packages/pds/src/db/tables/record.ts +++ b/packages/pds/src/db/tables/record.ts @@ -7,7 +7,7 @@ export interface Record { rkey: string repoRev: string | null indexedAt: string - takedownId: number | null + takedownRef: string | null } export const tableName = 'record' diff --git a/packages/pds/src/db/tables/repo-blob.ts b/packages/pds/src/db/tables/repo-blob.ts index a1fed0877e5..ddeb6c59158 100644 --- a/packages/pds/src/db/tables/repo-blob.ts +++ b/packages/pds/src/db/tables/repo-blob.ts @@ -3,7 +3,7 @@ export interface RepoBlob { recordUri: string repoRev: string | null did: string - takedownId: number | null + takedownRef: string | null } export const tableName = 'repo_blob' diff --git a/packages/pds/src/db/tables/repo-root.ts b/packages/pds/src/db/tables/repo-root.ts index 6b6c921f380..74ca31d3f80 100644 --- a/packages/pds/src/db/tables/repo-root.ts +++ b/packages/pds/src/db/tables/repo-root.ts @@ -4,7 +4,7 @@ export interface RepoRoot { root: string rev: string | null indexedAt: string - takedownId: number | null + takedownRef: string | null } export const tableName = 'repo_root' diff --git a/packages/pds/src/db/util.ts b/packages/pds/src/db/util.ts index 696ac7dee8b..6dd31c67898 100644 --- a/packages/pds/src/db/util.ts +++ b/packages/pds/src/db/util.ts @@ -20,11 +20,11 @@ export const actorWhereClause = (actor: string) => { // Applies to repo_root or record table export const notSoftDeletedClause = (alias: DbRef) => { - return sql`${alias}."takedownId" is null` + return sql`${alias}."takedownRef" is null` } -export const softDeleted = (repoOrRecord: { takedownId: number | null }) => { - return repoOrRecord.takedownId !== null +export const softDeleted = (repoOrRecord: { takedownRef: string | null }) => { + return repoOrRecord.takedownRef !== null } export const countAll = sql`count(*)` diff --git a/packages/pds/src/index.ts b/packages/pds/src/index.ts index cc9e1555895..42544eba492 100644 --- a/packages/pds/src/index.ts +++ b/packages/pds/src/index.ts @@ -28,7 +28,6 @@ import compression from './util/compression' export * from './config' export { Database } from './db' -export { PeriodicModerationActionReversal } from './db/periodic-moderation-action-reversal' export { DiskBlobStore, MemoryBlobStore } from './storage' export { AppContext } from './context' export { httpLogger } from './logger' diff --git a/packages/pds/src/lexicon/index.ts b/packages/pds/src/lexicon/index.ts index 1fd8a1f127c..bf69ebafa68 100644 --- a/packages/pds/src/lexicon/index.ts +++ b/packages/pds/src/lexicon/index.ts @@ -12,6 +12,7 @@ import { schemas } from './lexicons' import * as ComAtprotoAdminDisableAccountInvites from './types/com/atproto/admin/disableAccountInvites' import * as ComAtprotoAdminDisableInviteCodes from './types/com/atproto/admin/disableInviteCodes' import * as ComAtprotoAdminEnableAccountInvites from './types/com/atproto/admin/enableAccountInvites' +import * as ComAtprotoAdminGetAccountInfo from './types/com/atproto/admin/getAccountInfo' import * as ComAtprotoAdminGetInviteCodes from './types/com/atproto/admin/getInviteCodes' import * as ComAtprotoAdminGetModerationAction from './types/com/atproto/admin/getModerationAction' import * as ComAtprotoAdminGetModerationActions from './types/com/atproto/admin/getModerationActions' @@ -19,6 +20,7 @@ import * as ComAtprotoAdminGetModerationReport from './types/com/atproto/admin/g import * as ComAtprotoAdminGetModerationReports from './types/com/atproto/admin/getModerationReports' import * as ComAtprotoAdminGetRecord from './types/com/atproto/admin/getRecord' import * as ComAtprotoAdminGetRepo from './types/com/atproto/admin/getRepo' +import * as ComAtprotoAdminGetSubjectStatus from './types/com/atproto/admin/getSubjectStatus' import * as ComAtprotoAdminResolveModerationReports from './types/com/atproto/admin/resolveModerationReports' import * as ComAtprotoAdminReverseModerationAction from './types/com/atproto/admin/reverseModerationAction' import * as ComAtprotoAdminSearchRepos from './types/com/atproto/admin/searchRepos' @@ -26,6 +28,7 @@ import * as ComAtprotoAdminSendEmail from './types/com/atproto/admin/sendEmail' import * as ComAtprotoAdminTakeModerationAction from './types/com/atproto/admin/takeModerationAction' import * as ComAtprotoAdminUpdateAccountEmail from './types/com/atproto/admin/updateAccountEmail' import * as ComAtprotoAdminUpdateAccountHandle from './types/com/atproto/admin/updateAccountHandle' +import * as ComAtprotoAdminUpdateSubjectStatus from './types/com/atproto/admin/updateSubjectStatus' import * as ComAtprotoIdentityResolveHandle from './types/com/atproto/identity/resolveHandle' import * as ComAtprotoIdentityUpdateHandle from './types/com/atproto/identity/updateHandle' import * as ComAtprotoLabelQueryLabels from './types/com/atproto/label/queryLabels' @@ -225,6 +228,17 @@ export class AdminNS { return this._server.xrpc.method(nsid, cfg) } + getAccountInfo( + cfg: ConfigOf< + AV, + ComAtprotoAdminGetAccountInfo.Handler>, + ComAtprotoAdminGetAccountInfo.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.admin.getAccountInfo' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } + getInviteCodes( cfg: ConfigOf< AV, @@ -302,6 +316,17 @@ export class AdminNS { return this._server.xrpc.method(nsid, cfg) } + getSubjectStatus( + cfg: ConfigOf< + AV, + ComAtprotoAdminGetSubjectStatus.Handler>, + ComAtprotoAdminGetSubjectStatus.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.admin.getSubjectStatus' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } + resolveModerationReports( cfg: ConfigOf< AV, @@ -378,6 +403,17 @@ export class AdminNS { const nsid = 'com.atproto.admin.updateAccountHandle' // @ts-ignore return this._server.xrpc.method(nsid, cfg) } + + updateSubjectStatus( + cfg: ConfigOf< + AV, + ComAtprotoAdminUpdateSubjectStatus.Handler>, + ComAtprotoAdminUpdateSubjectStatus.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.admin.updateSubjectStatus' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } } export class IdentityNS { diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index 537350b5f13..38af4475fb8 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -8,6 +8,18 @@ export const schemaDict = { lexicon: 1, id: 'com.atproto.admin.defs', defs: { + statusAttr: { + type: 'object', + required: ['applied'], + properties: { + applied: { + type: 'boolean', + }, + ref: { + type: 'string', + }, + }, + }, actionView: { type: 'object', required: [ @@ -424,6 +436,44 @@ export const schemaDict = { }, }, }, + accountView: { + type: 'object', + required: ['did', 'handle', 'indexedAt'], + properties: { + did: { + type: 'string', + format: 'did', + }, + handle: { + type: 'string', + format: 'handle', + }, + email: { + type: 'string', + }, + indexedAt: { + type: 'string', + format: 'datetime', + }, + invitedBy: { + type: 'ref', + ref: 'lex:com.atproto.server.defs#inviteCode', + }, + invites: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.server.defs#inviteCode', + }, + }, + invitesDisabled: { + type: 'boolean', + }, + inviteNote: { + type: 'string', + }, + }, + }, repoViewNotFound: { type: 'object', required: ['did'], @@ -444,6 +494,24 @@ export const schemaDict = { }, }, }, + repoBlobRef: { + type: 'object', + required: ['did', 'cid'], + properties: { + did: { + type: 'string', + format: 'did', + }, + cid: { + type: 'string', + format: 'cid', + }, + recordUri: { + type: 'string', + format: 'at-uri', + }, + }, + }, recordView: { type: 'object', required: [ @@ -730,6 +798,33 @@ export const schemaDict = { }, }, }, + ComAtprotoAdminGetAccountInfo: { + lexicon: 1, + id: 'com.atproto.admin.getAccountInfo', + defs: { + main: { + type: 'query', + description: 'View details about an account.', + parameters: { + type: 'params', + required: ['did'], + properties: { + did: { + type: 'string', + format: 'did', + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#accountView', + }, + }, + }, + }, + }, ComAtprotoAdminGetInviteCodes: { lexicon: 1, id: 'com.atproto.admin.getInviteCodes', @@ -1026,6 +1121,55 @@ export const schemaDict = { }, }, }, + ComAtprotoAdminGetSubjectStatus: { + lexicon: 1, + id: 'com.atproto.admin.getSubjectStatus', + defs: { + main: { + type: 'query', + description: + 'Fetch the service-specific the admin status of a subject (account, record, or blob)', + parameters: { + type: 'params', + properties: { + did: { + type: 'string', + format: 'did', + }, + uri: { + type: 'string', + format: 'at-uri', + }, + blob: { + type: 'string', + format: 'cid', + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['subject'], + properties: { + subject: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#repoRef', + 'lex:com.atproto.repo.strongRef', + 'lex:com.atproto.admin.defs#repoBlobRef', + ], + }, + takedown: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#statusAttr', + }, + }, + }, + }, + }, + }, + }, ComAtprotoAdminResolveModerationReports: { lexicon: 1, id: 'com.atproto.admin.resolveModerationReports', @@ -1118,9 +1262,6 @@ export const schemaDict = { q: { type: 'string', }, - invitedBy: { - type: 'string', - }, limit: { type: 'integer', minimum: 1, @@ -1326,6 +1467,59 @@ export const schemaDict = { }, }, }, + ComAtprotoAdminUpdateSubjectStatus: { + lexicon: 1, + id: 'com.atproto.admin.updateSubjectStatus', + defs: { + main: { + type: 'procedure', + description: + 'Update the service-specific admin status of a subject (account, record, or blob)', + input: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['subject'], + properties: { + subject: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#repoRef', + 'lex:com.atproto.repo.strongRef', + 'lex:com.atproto.admin.defs#repoBlobRef', + ], + }, + takedown: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#statusAttr', + }, + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['subject'], + properties: { + subject: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#repoRef', + 'lex:com.atproto.repo.strongRef', + 'lex:com.atproto.admin.defs#repoBlobRef', + ], + }, + takedown: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#statusAttr', + }, + }, + }, + }, + }, + }, + }, ComAtprotoIdentityResolveHandle: { lexicon: 1, id: 'com.atproto.identity.resolveHandle', @@ -7367,6 +7561,7 @@ export const ids = { 'com.atproto.admin.disableAccountInvites', ComAtprotoAdminDisableInviteCodes: 'com.atproto.admin.disableInviteCodes', ComAtprotoAdminEnableAccountInvites: 'com.atproto.admin.enableAccountInvites', + ComAtprotoAdminGetAccountInfo: 'com.atproto.admin.getAccountInfo', ComAtprotoAdminGetInviteCodes: 'com.atproto.admin.getInviteCodes', ComAtprotoAdminGetModerationAction: 'com.atproto.admin.getModerationAction', ComAtprotoAdminGetModerationActions: 'com.atproto.admin.getModerationActions', @@ -7374,6 +7569,7 @@ export const ids = { ComAtprotoAdminGetModerationReports: 'com.atproto.admin.getModerationReports', ComAtprotoAdminGetRecord: 'com.atproto.admin.getRecord', ComAtprotoAdminGetRepo: 'com.atproto.admin.getRepo', + ComAtprotoAdminGetSubjectStatus: 'com.atproto.admin.getSubjectStatus', ComAtprotoAdminResolveModerationReports: 'com.atproto.admin.resolveModerationReports', ComAtprotoAdminReverseModerationAction: @@ -7383,6 +7579,7 @@ export const ids = { ComAtprotoAdminTakeModerationAction: 'com.atproto.admin.takeModerationAction', ComAtprotoAdminUpdateAccountEmail: 'com.atproto.admin.updateAccountEmail', ComAtprotoAdminUpdateAccountHandle: 'com.atproto.admin.updateAccountHandle', + ComAtprotoAdminUpdateSubjectStatus: 'com.atproto.admin.updateSubjectStatus', ComAtprotoIdentityResolveHandle: 'com.atproto.identity.resolveHandle', ComAtprotoIdentityUpdateHandle: 'com.atproto.identity.updateHandle', ComAtprotoLabelDefs: 'com.atproto.label.defs', diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts index 968252a4c2c..ea463368f8e 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts @@ -10,6 +10,24 @@ import * as ComAtprotoModerationDefs from '../moderation/defs' import * as ComAtprotoServerDefs from '../server/defs' import * as ComAtprotoLabelDefs from '../label/defs' +export interface StatusAttr { + applied: boolean + ref?: string + [k: string]: unknown +} + +export function isStatusAttr(v: unknown): v is StatusAttr { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#statusAttr' + ) +} + +export function validateStatusAttr(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#statusAttr', v) +} + export interface ActionView { id: number action: ActionType @@ -238,6 +256,30 @@ export function validateRepoViewDetail(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#repoViewDetail', v) } +export interface AccountView { + did: string + handle: string + email?: string + indexedAt: string + invitedBy?: ComAtprotoServerDefs.InviteCode + invites?: ComAtprotoServerDefs.InviteCode[] + invitesDisabled?: boolean + inviteNote?: string + [k: string]: unknown +} + +export function isAccountView(v: unknown): v is AccountView { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#accountView' + ) +} + +export function validateAccountView(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#accountView', v) +} + export interface RepoViewNotFound { did: string [k: string]: unknown @@ -272,6 +314,25 @@ export function validateRepoRef(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#repoRef', v) } +export interface RepoBlobRef { + did: string + cid: string + recordUri?: string + [k: string]: unknown +} + +export function isRepoBlobRef(v: unknown): v is RepoBlobRef { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#repoBlobRef' + ) +} + +export function validateRepoBlobRef(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#repoBlobRef', v) +} + export interface RecordView { uri: string cid: string diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/getAccountInfo.ts b/packages/pds/src/lexicon/types/com/atproto/admin/getAccountInfo.ts new file mode 100644 index 00000000000..88a2b17a4b8 --- /dev/null +++ b/packages/pds/src/lexicon/types/com/atproto/admin/getAccountInfo.ts @@ -0,0 +1,41 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' +import * as ComAtprotoAdminDefs from './defs' + +export interface QueryParams { + did: string +} + +export type InputSchema = undefined +export type OutputSchema = ComAtprotoAdminDefs.AccountView +export type HandlerInput = undefined + +export interface HandlerSuccess { + encoding: 'application/json' + body: OutputSchema + headers?: { [key: string]: string } +} + +export interface HandlerError { + status: number + message?: string +} + +export type HandlerOutput = HandlerError | HandlerSuccess +export type HandlerReqCtx = { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +} +export type Handler = ( + ctx: HandlerReqCtx, +) => Promise | HandlerOutput diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/getSubjectStatus.ts b/packages/pds/src/lexicon/types/com/atproto/admin/getSubjectStatus.ts new file mode 100644 index 00000000000..7315e51e8c2 --- /dev/null +++ b/packages/pds/src/lexicon/types/com/atproto/admin/getSubjectStatus.ts @@ -0,0 +1,54 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' +import * as ComAtprotoAdminDefs from './defs' +import * as ComAtprotoRepoStrongRef from '../repo/strongRef' + +export interface QueryParams { + did?: string + uri?: string + blob?: string +} + +export type InputSchema = undefined + +export interface OutputSchema { + subject: + | ComAtprotoAdminDefs.RepoRef + | ComAtprotoRepoStrongRef.Main + | ComAtprotoAdminDefs.RepoBlobRef + | { $type: string; [k: string]: unknown } + takedown?: ComAtprotoAdminDefs.StatusAttr + [k: string]: unknown +} + +export type HandlerInput = undefined + +export interface HandlerSuccess { + encoding: 'application/json' + body: OutputSchema + headers?: { [key: string]: string } +} + +export interface HandlerError { + status: number + message?: string +} + +export type HandlerOutput = HandlerError | HandlerSuccess +export type HandlerReqCtx = { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +} +export type Handler = ( + ctx: HandlerReqCtx, +) => Promise | HandlerOutput diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/searchRepos.ts b/packages/pds/src/lexicon/types/com/atproto/admin/searchRepos.ts index 32266fd66fd..1e7e1a36bb6 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/searchRepos.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/searchRepos.ts @@ -13,7 +13,6 @@ export interface QueryParams { /** DEPRECATED: use 'q' instead */ term?: string q?: string - invitedBy?: string limit: number cursor?: string } diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/updateSubjectStatus.ts b/packages/pds/src/lexicon/types/com/atproto/admin/updateSubjectStatus.ts new file mode 100644 index 00000000000..559ee948380 --- /dev/null +++ b/packages/pds/src/lexicon/types/com/atproto/admin/updateSubjectStatus.ts @@ -0,0 +1,61 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' +import * as ComAtprotoAdminDefs from './defs' +import * as ComAtprotoRepoStrongRef from '../repo/strongRef' + +export interface QueryParams {} + +export interface InputSchema { + subject: + | ComAtprotoAdminDefs.RepoRef + | ComAtprotoRepoStrongRef.Main + | ComAtprotoAdminDefs.RepoBlobRef + | { $type: string; [k: string]: unknown } + takedown?: ComAtprotoAdminDefs.StatusAttr + [k: string]: unknown +} + +export interface OutputSchema { + subject: + | ComAtprotoAdminDefs.RepoRef + | ComAtprotoRepoStrongRef.Main + | ComAtprotoAdminDefs.RepoBlobRef + | { $type: string; [k: string]: unknown } + takedown?: ComAtprotoAdminDefs.StatusAttr + [k: string]: unknown +} + +export interface HandlerInput { + encoding: 'application/json' + body: InputSchema +} + +export interface HandlerSuccess { + encoding: 'application/json' + body: OutputSchema + headers?: { [key: string]: string } +} + +export interface HandlerError { + status: number + message?: string +} + +export type HandlerOutput = HandlerError | HandlerSuccess +export type HandlerReqCtx = { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +} +export type Handler = ( + ctx: HandlerReqCtx, +) => Promise | HandlerOutput diff --git a/packages/pds/src/services/account/index.ts b/packages/pds/src/services/account/index.ts index 9a6910d0e4f..73518199c32 100644 --- a/packages/pds/src/services/account/index.ts +++ b/packages/pds/src/services/account/index.ts @@ -14,6 +14,8 @@ import * as sequencer from '../../sequencer' import { AppPassword } from '../../lexicon/types/com/atproto/server/createAppPassword' import { EmailTokenPurpose } from '../../db/tables/email-token' import { getRandomToken } from '../../api/com/atproto/server/util' +import { AccountView } from '../../lexicon/types/com/atproto/admin/defs' +import { INVALID_HANDLE } from '@atproto/syntax' export class AccountService { constructor(public db: Database) {} @@ -59,7 +61,7 @@ export class AccountService { const found = await this.db.db .selectFrom('repo_root') .where('did', '=', did) - .where('takedownId', 'is', null) + .where('takedownRef', 'is', null) .select('did') .executeTakeFirst() return found !== undefined @@ -376,6 +378,38 @@ export class AccountService { }) } + async adminView(did: string): Promise { + const accountQb = this.db.db + .selectFrom('did_handle') + .innerJoin('user_account', 'user_account.did', 'did_handle.did') + .where('did_handle.did', '=', did) + .select([ + 'did_handle.did', + 'did_handle.handle', + 'user_account.email', + 'user_account.invitesDisabled', + 'user_account.inviteNote', + 'user_account.createdAt as indexedAt', + ]) + + const [account, invites, invitedBy] = await Promise.all([ + accountQb.executeTakeFirst(), + this.getAccountInviteCodes(did), + this.getInvitedByForAccounts([did]), + ]) + + if (!account) return null + + return { + ...account, + handle: account?.handle ?? INVALID_HANDLE, + invitesDisabled: account.invitesDisabled === 1, + inviteNote: account.inviteNote ?? undefined, + invites, + invitedBy: invitedBy[did], + } + } + selectInviteCodesQb() { const ref = this.db.db.dynamic.ref const builder = this.db.db diff --git a/packages/pds/src/services/moderation/index.ts b/packages/pds/src/services/moderation/index.ts index 9e46332cf33..240a95004d2 100644 --- a/packages/pds/src/services/moderation/index.ts +++ b/packages/pds/src/services/moderation/index.ts @@ -1,15 +1,13 @@ -import { Selectable, sql } from 'kysely' import { CID } from 'multiformats/cid' import { BlobStore } from '@atproto/repo' import { AtUri } from '@atproto/syntax' -import { InvalidRequestError } from '@atproto/xrpc-server' import Database from '../../db' -import { ModerationAction, ModerationReport } from '../../db/tables/moderation' -import { RecordService } from '../record' -import { ModerationViews } from './views' -import SqlRepoStorage from '../../sql-repo-storage' -import { TAKEDOWN } from '../../lexicon/types/com/atproto/admin/defs' -import { addHoursToDate } from '../../util/date' +import { + RepoBlobRef, + RepoRef, + StatusAttr, +} from '../../lexicon/types/com/atproto/admin/defs' +import { Main as StrongRef } from '../../lexicon/types/com/atproto/repo/strongRef' export class ModerationService { constructor(public db: Database, public blobstore: BlobStore) {} @@ -18,614 +16,110 @@ export class ModerationService { return (db: Database) => new ModerationService(db, blobstore) } - views = new ModerationViews(this.db) - - services = { - record: RecordService.creator(), - } - - async getAction(id: number): Promise { - return await this.db.db - .selectFrom('moderation_action') - .selectAll() - .where('id', '=', id) + async getRepoTakedownState( + did: string, + ): Promise | null> { + const res = await this.db.db + .selectFrom('repo_root') + .select('takedownRef') + .where('did', '=', did) .executeTakeFirst() - } - - async getActionOrThrow(id: number): Promise { - const action = await this.getAction(id) - if (!action) throw new InvalidRequestError('Action not found') - return action - } - - async getActions(opts: { - subject?: string - limit: number - cursor?: string - }): Promise { - const { subject, limit, cursor } = opts - let builder = this.db.db.selectFrom('moderation_action') - if (subject) { - builder = builder.where((qb) => { - return qb - .where('subjectDid', '=', subject) - .orWhere('subjectUri', '=', subject) - }) - } - if (cursor) { - const cursorNumeric = parseInt(cursor, 10) - if (isNaN(cursorNumeric)) { - throw new InvalidRequestError('Malformed cursor') - } - builder = builder.where('id', '<', cursorNumeric) - } - return await builder - .selectAll() - .orderBy('id', 'desc') - .limit(limit) - .execute() - } - - async getReport(id: number): Promise { - return await this.db.db - .selectFrom('moderation_report') - .selectAll() - .where('id', '=', id) + if (!res) return null + const state = takedownRefToStatus(res.takedownRef ?? null) + return { + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: did, + }, + takedown: state, + } + } + + async getRecordTakedownState( + uri: AtUri, + ): Promise | null> { + const res = await this.db.db + .selectFrom('record') + .select(['takedownRef', 'cid']) + .where('uri', '=', uri.toString()) .executeTakeFirst() - } - - async getReports(opts: { - subject?: string - resolved?: boolean - actionType?: string - limit: number - cursor?: string - ignoreSubjects?: string[] - reverse?: boolean - reporters?: string[] - actionedBy?: string - }): Promise { - const { - subject, - resolved, - actionType, - limit, - cursor, - ignoreSubjects, - reverse = false, - reporters, - actionedBy, - } = opts - const { ref } = this.db.db.dynamic - let builder = this.db.db.selectFrom('moderation_report') - if (subject) { - builder = builder.where((qb) => { - return qb - .where('subjectDid', '=', subject) - .orWhere('subjectUri', '=', subject) - }) - } - - if (ignoreSubjects?.length) { - const ignoreUris: string[] = [] - const ignoreDids: string[] = [] - - ignoreSubjects.forEach((subject) => { - if (subject.startsWith('at://')) { - ignoreUris.push(subject) - } else if (subject.startsWith('did:')) { - ignoreDids.push(subject) - } - }) - - if (ignoreDids.length) { - builder = builder.where('subjectDid', 'not in', ignoreDids) - } - if (ignoreUris.length) { - builder = builder.where((qb) => { - // Without the null condition, postgres will ignore all reports where `subjectUri` is null - // which will make all the account reports be ignored as well - return qb - .where('subjectUri', 'not in', ignoreUris) - .orWhere('subjectUri', 'is', null) - }) - } - } - - if (reporters?.length) { - builder = builder.where('reportedByDid', 'in', reporters) - } - - if (resolved !== undefined) { - const resolutionsQuery = this.db.db - .selectFrom('moderation_report_resolution') - .selectAll() - .whereRef( - 'moderation_report_resolution.reportId', - '=', - ref('moderation_report.id'), - ) - builder = resolved - ? builder.whereExists(resolutionsQuery) - : builder.whereNotExists(resolutionsQuery) - } - if (actionType !== undefined || actionedBy !== undefined) { - let resolutionActionsQuery = this.db.db - .selectFrom('moderation_report_resolution') - .innerJoin( - 'moderation_action', - 'moderation_action.id', - 'moderation_report_resolution.actionId', - ) - .whereRef( - 'moderation_report_resolution.reportId', - '=', - ref('moderation_report.id'), - ) - - if (actionType) { - resolutionActionsQuery = resolutionActionsQuery - .where('moderation_action.action', '=', sql`${actionType}`) - .where('moderation_action.reversedAt', 'is', null) - } - - if (actionedBy) { - resolutionActionsQuery = resolutionActionsQuery.where( - 'moderation_action.createdBy', - '=', - actionedBy, - ) - } - - builder = builder.whereExists(resolutionActionsQuery.selectAll()) - } - - if (cursor) { - const cursorNumeric = parseInt(cursor, 10) - if (isNaN(cursorNumeric)) { - throw new InvalidRequestError('Malformed cursor') - } - builder = builder.where('id', reverse ? '>' : '<', cursorNumeric) - } - - return await builder - .leftJoin('did_handle', 'did_handle.did', 'moderation_report.subjectDid') - .selectAll(['moderation_report', 'did_handle']) - .orderBy('id', reverse ? 'asc' : 'desc') - .limit(limit) - .execute() - } - - async getReportOrThrow(id: number): Promise { - const report = await this.getReport(id) - if (!report) throw new InvalidRequestError('Report not found') - return report - } - - async getCurrentActions( - subject: { did: string } | { uri: AtUri } | { cids: CID[] }, - ) { - const { ref } = this.db.db.dynamic - let builder = this.db.db - .selectFrom('moderation_action') - .selectAll() - .where('reversedAt', 'is', null) - if ('did' in subject) { - builder = builder - .where('subjectType', '=', 'com.atproto.admin.defs#repoRef') - .where('subjectDid', '=', subject.did) - } else if ('uri' in subject) { - builder = builder - .where('subjectType', '=', 'com.atproto.repo.strongRef') - .where('subjectUri', '=', subject.uri.toString()) - } else { - const blobsForAction = this.db.db - .selectFrom('moderation_action_subject_blob') - .selectAll() - .whereRef('actionId', '=', ref('moderation_action.id')) - .where( - 'cid', - 'in', - subject.cids.map((cid) => cid.toString()), - ) - builder = builder.whereExists(blobsForAction) - } - return await builder.execute() - } - - async logAction(info: { - action: ModerationActionRow['action'] - subject: { did: string } | { uri: AtUri; cid: CID } - subjectBlobCids?: CID[] - reason: string - createLabelVals?: string[] - negateLabelVals?: string[] - createdBy: string - createdAt?: Date - durationInHours?: number - }): Promise { - this.db.assertTransaction() - const { - action, - createdBy, - reason, - subject, - subjectBlobCids, - durationInHours, - createdAt = new Date(), - } = info - const createLabelVals = - info.createLabelVals && info.createLabelVals.length > 0 - ? info.createLabelVals.join(' ') - : undefined - const negateLabelVals = - info.negateLabelVals && info.negateLabelVals.length > 0 - ? info.negateLabelVals.join(' ') - : undefined - - // Resolve subject info - let subjectInfo: SubjectInfo - if ('did' in subject) { - // Allowing dids that may not exist: may have been deleted but needs to remain actionable. - subjectInfo = { - subjectType: 'com.atproto.admin.defs#repoRef', - subjectDid: subject.did, - subjectUri: null, - subjectCid: null, - } - if (subjectBlobCids?.length) { - throw new InvalidRequestError('Blobs do not apply to repo subjects') - } - } else { - // Allowing records/blobs that may not exist: may have been deleted but needs to remain actionable. - subjectInfo = { - subjectType: 'com.atproto.repo.strongRef', - subjectDid: subject.uri.host, - subjectUri: subject.uri.toString(), - subjectCid: subject.cid.toString(), - } - if (subjectBlobCids?.length) { - const cidsFromSubject = await this.db.db - .selectFrom('repo_blob') - .where('recordUri', '=', subject.uri.toString()) - .where( - 'cid', - 'in', - subjectBlobCids.map((c) => c.toString()), - ) - .select('cid') - .execute() - if (cidsFromSubject.length !== subjectBlobCids.length) { - throw new InvalidRequestError('Blobs do not match record subject') - } - } - } - - const subjectActions = await this.getCurrentActions(subject) - if (subjectActions.length) { - throw new InvalidRequestError( - `Subject already has an active action: #${subjectActions[0].id}`, - 'SubjectHasAction', - ) - } - - const actionResult = await this.db.db - .insertInto('moderation_action') - .values({ - action, - reason, - createdAt: createdAt.toISOString(), - createdBy, - createLabelVals, - negateLabelVals, - durationInHours, - expiresAt: - durationInHours !== undefined - ? addHoursToDate(durationInHours, createdAt).toISOString() - : undefined, - ...subjectInfo, - }) - .returningAll() - .executeTakeFirstOrThrow() - - if (subjectBlobCids?.length && !('did' in subject)) { - const blobActions = await this.getCurrentActions({ - cids: subjectBlobCids, - }) - if (blobActions.length) { - throw new InvalidRequestError( - `Blob already has an active action: #${blobActions[0].id}`, - 'SubjectHasAction', - ) - } - - await this.db.db - .insertInto('moderation_action_subject_blob') - .values( - subjectBlobCids.map((cid) => ({ - actionId: actionResult.id, - cid: cid.toString(), - recordUri: subject.uri.toString(), - })), - ) - .execute() - } - - return actionResult - } - - async getActionsDueForReversal(): Promise> { - const actionsDueForReversal = await this.db.db - .selectFrom('moderation_action') - // Get entries that have an durationInHours that has passed and have not been reversed - .where('durationInHours', 'is not', null) - .where('expiresAt', '<', new Date().toISOString()) - .where('reversedAt', 'is', null) - .selectAll() - .execute() - - return actionsDueForReversal - } - - async revertAction({ - id, - createdAt, - createdBy, - reason, - }: { - id: number - createdAt: Date - createdBy: string - reason: string - }) { - const result = await this.logReverseAction({ - id, - createdAt, - createdBy, - reason, - }) - - if ( - result.action === TAKEDOWN && - result.subjectType === 'com.atproto.admin.defs#repoRef' && - result.subjectDid - ) { - await this.reverseTakedownRepo({ - did: result.subjectDid, - }) - } - - if ( - result.action === TAKEDOWN && - result.subjectType === 'com.atproto.repo.strongRef' && - result.subjectUri - ) { - await this.reverseTakedownRecord({ - uri: new AtUri(result.subjectUri), - }) - } - - return result - } - - async logReverseAction(info: { - id: number - reason: string - createdBy: string - createdAt?: Date - }): Promise { - const { id, createdBy, reason, createdAt = new Date() } = info - - const result = await this.db.db - .updateTable('moderation_action') - .where('id', '=', id) - .set({ - reversedAt: createdAt.toISOString(), - reversedBy: createdBy, - reversedReason: reason, - }) - .returningAll() + if (!res) return null + const state = takedownRefToStatus(res.takedownRef ?? null) + return { + subject: { + $type: 'com.atproto.repo.strongRef', + uri: uri.toString(), + cid: res.cid, + }, + takedown: state, + } + } + + async getBlobTakedownState( + did: string, + cid: CID, + ): Promise | null> { + const res = await this.db.db + .selectFrom('repo_blob') + .select('takedownRef') + .where('did', '=', did) + .where('cid', '=', cid.toString()) .executeTakeFirst() - - if (!result) { - throw new InvalidRequestError('Moderation action not found') + if (!res) return null + const state = takedownRefToStatus(res.takedownRef ?? null) + return { + subject: { + $type: 'com.atproto.admin.defs#repoBlobRef', + did: did, + cid: cid.toString(), + }, + takedown: state, } - - return result - } - - async takedownRepo(info: { takedownId: number; did: string }) { - await this.db.db - .updateTable('repo_root') - .set({ takedownId: info.takedownId }) - .where('did', '=', info.did) - .where('takedownId', 'is', null) - .executeTakeFirst() } - async reverseTakedownRepo(info: { did: string }) { + async updateRepoTakedownState(did: string, takedown: StatusAttr) { + const takedownRef = statusTotakedownRef(takedown) await this.db.db .updateTable('repo_root') - .set({ takedownId: null }) - .where('did', '=', info.did) + .set({ takedownRef }) + .where('did', '=', did) .execute() } - async takedownRecord(info: { - takedownId: number - uri: AtUri - blobCids?: CID[] - }) { - this.db.assertTransaction() - await this.db.db - .updateTable('record') - .set({ takedownId: info.takedownId }) - .where('uri', '=', info.uri.toString()) - .where('takedownId', 'is', null) - .executeTakeFirst() - if (info.blobCids?.length) { - await this.db.db - .updateTable('repo_blob') - .set({ takedownId: info.takedownId }) - .where('recordUri', '=', info.uri.toString()) - .where( - 'cid', - 'in', - info.blobCids.map((c) => c.toString()), - ) - .where('takedownId', 'is', null) - .executeTakeFirst() - await Promise.all( - info.blobCids.map((cid) => this.blobstore.quarantine(cid)), - ) - } - } - - async reverseTakedownRecord(info: { uri: AtUri }) { - this.db.assertTransaction() + async updateRecordTakedownState(uri: AtUri, takedown: StatusAttr) { + const takedownRef = statusTotakedownRef(takedown) await this.db.db .updateTable('record') - .set({ takedownId: null }) - .where('uri', '=', info.uri.toString()) - .execute() - const blobs = await this.db.db - .updateTable('repo_blob') - .set({ takedownId: null }) - .where('takedownId', 'is not', null) - .where('recordUri', '=', info.uri.toString()) - .returning('cid') + .set({ takedownRef }) + .where('uri', '=', uri.toString()) .execute() - await Promise.all( - blobs.map(async (blob) => { - const cid = CID.parse(blob.cid) - await this.blobstore.unquarantine(cid) - }), - ) } - async resolveReports(info: { - reportIds: number[] - actionId: number - createdBy: string - createdAt?: Date - }): Promise { - const { reportIds, actionId, createdBy, createdAt = new Date() } = info - const action = await this.getActionOrThrow(actionId) - - if (!reportIds.length) return - const reports = await this.db.db - .selectFrom('moderation_report') - .where('id', 'in', reportIds) - .select(['id', 'subjectType', 'subjectDid', 'subjectUri']) - .execute() - - reportIds.forEach((reportId) => { - const report = reports.find((r) => r.id === reportId) - if (!report) throw new InvalidRequestError('Report not found') - if (action.subjectDid !== report.subjectDid) { - // Report and action always must target repo or record from the same did - throw new InvalidRequestError( - `Report ${report.id} cannot be resolved by action`, - ) - } - if ( - action.subjectType === 'com.atproto.repo.strongRef' && - report.subjectType === 'com.atproto.repo.strongRef' && - report.subjectUri !== action.subjectUri - ) { - // If report and action are both for a record, they must be for the same record - throw new InvalidRequestError( - `Report ${report.id} cannot be resolved by action`, - ) - } - }) - + async updateBlobTakedownState(did: string, blob: CID, takedown: StatusAttr) { + const takedownRef = statusTotakedownRef(takedown) await this.db.db - .insertInto('moderation_report_resolution') - .values( - reportIds.map((reportId) => ({ - reportId, - actionId, - createdAt: createdAt.toISOString(), - createdBy, - })), - ) - .onConflict((oc) => oc.doNothing()) + .updateTable('repo_blob') + .set({ takedownRef }) + .where('did', '=', did) + .where('cid', '=', blob.toString()) .execute() - } - - async report(info: { - reasonType: ModerationReportRow['reasonType'] - reason?: string - subject: { did: string } | { uri: AtUri; cid?: CID } - reportedBy: string - createdAt?: Date - }): Promise { - const { - reasonType, - reason, - reportedBy, - createdAt = new Date(), - subject, - } = info - - // Resolve subject info - let subjectInfo: SubjectInfo - if ('did' in subject) { - const repo = await new SqlRepoStorage(this.db, subject.did).getRoot() - if (!repo) throw new InvalidRequestError('Repo not found') - subjectInfo = { - subjectType: 'com.atproto.admin.defs#repoRef', - subjectDid: subject.did, - subjectUri: null, - subjectCid: null, - } + if (takedown.applied) { + await this.blobstore.quarantine(blob) } else { - const record = await this.services - .record(this.db) - .getRecord(subject.uri, subject.cid?.toString() ?? null, true) - if (!record) throw new InvalidRequestError('Record not found') - subjectInfo = { - subjectType: 'com.atproto.repo.strongRef', - subjectDid: subject.uri.host, - subjectUri: subject.uri.toString(), - subjectCid: record.cid, - } + await this.blobstore.unquarantine(blob) } - - const report = await this.db.db - .insertInto('moderation_report') - .values({ - reasonType, - reason: reason || null, - createdAt: createdAt.toISOString(), - reportedByDid: reportedBy, - ...subjectInfo, - }) - .returningAll() - .executeTakeFirstOrThrow() - - return report } } -export type ModerationActionRow = Selectable +type StatusResponse = { + subject: T + takedown: StatusAttr +} -export type ModerationReportRow = Selectable -export type ModerationReportRowWithHandle = ModerationReportRow & { - handle?: string | null +const takedownRefToStatus = (id: string | null): StatusAttr => { + return id === null ? { applied: false } : { applied: true, ref: id } } -export type SubjectInfo = - | { - subjectType: 'com.atproto.admin.defs#repoRef' - subjectDid: string - subjectUri: null - subjectCid: null - } - | { - subjectType: 'com.atproto.repo.strongRef' - subjectDid: string - subjectUri: string - subjectCid: string - } +const statusTotakedownRef = (state: StatusAttr): string | null => { + return state.applied ? state.ref ?? new Date().toISOString() : null +} diff --git a/packages/pds/src/services/moderation/views.ts b/packages/pds/src/services/moderation/views.ts deleted file mode 100644 index e8d89620d73..00000000000 --- a/packages/pds/src/services/moderation/views.ts +++ /dev/null @@ -1,633 +0,0 @@ -import { Selectable } from 'kysely' -import { ArrayEl, cborBytesToRecord } from '@atproto/common' -import { AtUri } from '@atproto/syntax' -import Database from '../../db' -import { DidHandle } from '../../db/tables/did-handle' -import { RepoRoot } from '../../db/tables/repo-root' -import { - RepoView, - RepoViewDetail, - RecordView, - RecordViewDetail, - ActionView, - ActionViewDetail, - ReportView, - ReportViewDetail, - BlobView, -} from '../../lexicon/types/com/atproto/admin/defs' -import { OutputSchema as ReportOutput } from '../../lexicon/types/com/atproto/moderation/createReport' -import { ModerationAction } from '../../db/tables/moderation' -import { AccountService } from '../account' -import { RecordService } from '../record' -import { ModerationReportRowWithHandle } from '.' -import { ids } from '../../lexicon/lexicons' - -export class ModerationViews { - constructor(private db: Database) {} - - services = { - account: AccountService.creator(), - record: RecordService.creator(), - } - - repo(result: RepoResult, opts: ModViewOptions): Promise - repo(result: RepoResult[], opts: ModViewOptions): Promise - async repo( - result: RepoResult | RepoResult[], - opts: ModViewOptions, - ): Promise { - const results = Array.isArray(result) ? result : [result] - if (results.length === 0) return [] - - const [info, actionResults, invitedBy] = await Promise.all([ - await this.db.db - .selectFrom('did_handle') - .leftJoin('user_account', 'user_account.did', 'did_handle.did') - .leftJoin('record as profile_record', (join) => - join - .onRef('profile_record.did', '=', 'did_handle.did') - .on('profile_record.collection', '=', ids.AppBskyActorProfile) - .on('profile_record.rkey', '=', 'self'), - ) - .leftJoin('ipld_block as profile_block', (join) => - join - .onRef('profile_block.cid', '=', 'profile_record.cid') - .onRef('profile_block.creator', '=', 'did_handle.did'), - ) - .where( - 'did_handle.did', - 'in', - results.map((r) => r.did), - ) - .select([ - 'did_handle.did as did', - 'user_account.email as email', - 'user_account.invitesDisabled as invitesDisabled', - 'user_account.inviteNote as inviteNote', - 'profile_block.content as profileBytes', - ]) - .execute(), - this.db.db - .selectFrom('moderation_action') - .where('reversedAt', 'is', null) - .where('subjectType', '=', 'com.atproto.admin.defs#repoRef') - .where( - 'subjectDid', - 'in', - results.map((r) => r.did), - ) - .select(['id', 'action', 'durationInHours', 'subjectDid']) - .execute(), - this.services - .account(this.db) - .getInvitedByForAccounts(results.map((r) => r.did)), - ]) - - const infoByDid = info.reduce( - (acc, cur) => Object.assign(acc, { [cur.did]: cur }), - {} as Record>, - ) - const actionByDid = actionResults.reduce( - (acc, cur) => Object.assign(acc, { [cur.subjectDid ?? '']: cur }), - {} as Record>, - ) - - const views = results.map((r) => { - const { email, invitesDisabled, profileBytes, inviteNote } = - infoByDid[r.did] ?? {} - const action = actionByDid[r.did] - const relatedRecords: object[] = [] - if (profileBytes) { - relatedRecords.push(cborBytesToRecord(profileBytes)) - } - return { - did: r.did, - handle: r.handle, - email: opts.includeEmails && email ? email : undefined, - relatedRecords, - indexedAt: r.indexedAt, - moderation: { - currentAction: action - ? { - id: action.id, - action: action.action, - durationInHours: action.durationInHours ?? undefined, - } - : undefined, - }, - invitedBy: invitedBy[r.did], - invitesDisabled: invitesDisabled === 1, - inviteNote: inviteNote ?? undefined, - } - }) - - return Array.isArray(result) ? views : views[0] - } - - async repoDetail( - result: RepoResult, - opts: ModViewOptions, - ): Promise { - const repo = await this.repo(result, opts) - const [reportResults, actionResults, inviteCodes] = await Promise.all([ - this.db.db - .selectFrom('moderation_report') - .where('subjectType', '=', 'com.atproto.admin.defs#repoRef') - .where('subjectDid', '=', repo.did) - .orderBy('id', 'desc') - .selectAll() - .execute(), - this.db.db - .selectFrom('moderation_action') - .where('subjectType', '=', 'com.atproto.admin.defs#repoRef') - .where('subjectDid', '=', repo.did) - .orderBy('id', 'desc') - .selectAll() - .execute(), - this.services.account(this.db).getAccountInviteCodes(repo.did), - ]) - const [reports, actions] = await Promise.all([ - this.report(reportResults), - this.action(actionResults), - ]) - return { - ...repo, - moderation: { - ...repo.moderation, - reports, - actions, - }, - invites: inviteCodes, - } - } - - record(result: RecordResult, opts: ModViewOptions): Promise - record(result: RecordResult[], opts: ModViewOptions): Promise - async record( - result: RecordResult | RecordResult[], - opts: ModViewOptions, - ): Promise { - const results = Array.isArray(result) ? result : [result] - if (results.length === 0) return [] - - const [repoResults, blobResults, actionResults] = await Promise.all([ - this.db.db - .selectFrom('repo_root') - .innerJoin('did_handle', 'did_handle.did', 'repo_root.did') - .where( - 'repo_root.did', - 'in', - results.map((r) => didFromUri(r.uri)), - ) - .selectAll('repo_root') - .selectAll('did_handle') - .execute(), - this.db.db - .selectFrom('repo_blob') - .where( - 'recordUri', - 'in', - results.map((r) => r.uri), - ) - .select(['cid', 'recordUri']) - .execute(), - this.db.db - .selectFrom('moderation_action') - .where('reversedAt', 'is', null) - .where('subjectType', '=', 'com.atproto.repo.strongRef') - .where( - 'subjectUri', - 'in', - results.map((r) => r.uri), - ) - .select(['id', 'action', 'durationInHours', 'subjectUri']) - .execute(), - ]) - const repos = await this.repo(repoResults, opts) - - const reposByDid = repos.reduce( - (acc, cur) => Object.assign(acc, { [cur.did]: cur }), - {} as Record>, - ) - const blobCidsByUri = blobResults.reduce((acc, cur) => { - acc[cur.recordUri] ??= [] - acc[cur.recordUri].push(cur.cid) - return acc - }, {} as Record) - const actionByUri = actionResults.reduce( - (acc, cur) => Object.assign(acc, { [cur.subjectUri ?? '']: cur }), - {} as Record>, - ) - - const views = results.map((res) => { - const repo = reposByDid[didFromUri(res.uri)] - const action = actionByUri[res.uri] - if (!repo) throw new Error(`Record repo is missing: ${res.uri}`) - return { - uri: res.uri, - cid: res.cid, - value: res.value, - blobCids: blobCidsByUri[res.uri] ?? [], - indexedAt: res.indexedAt, - repo, - moderation: { - currentAction: action - ? { - id: action.id, - action: action.action, - durationInHours: action.durationInHours ?? undefined, - } - : undefined, - }, - } - }) - - return Array.isArray(result) ? views : views[0] - } - - async recordDetail( - result: RecordResult, - opts: ModViewOptions, - ): Promise { - const [record, reportResults, actionResults] = await Promise.all([ - this.record(result, opts), - this.db.db - .selectFrom('moderation_report') - .where('subjectType', '=', 'com.atproto.repo.strongRef') - .where('subjectUri', '=', result.uri) - .leftJoin( - 'did_handle', - 'did_handle.did', - 'moderation_report.subjectDid', - ) - .orderBy('id', 'desc') - .selectAll() - .execute(), - this.db.db - .selectFrom('moderation_action') - .where('subjectType', '=', 'com.atproto.repo.strongRef') - .where('subjectUri', '=', result.uri) - .orderBy('id', 'desc') - .selectAll() - .execute(), - ]) - const [reports, actions, blobs] = await Promise.all([ - this.report(reportResults), - this.action(actionResults), - this.blob(record.blobCids), - ]) - return { - ...record, - blobs, - moderation: { - ...record.moderation, - reports, - actions, - }, - } - } - - action(result: ActionResult): Promise - action(result: ActionResult[]): Promise - async action( - result: ActionResult | ActionResult[], - ): Promise { - const results = Array.isArray(result) ? result : [result] - if (results.length === 0) return [] - - const [resolutions, subjectBlobResults] = await Promise.all([ - this.db.db - .selectFrom('moderation_report_resolution') - .select(['reportId as id', 'actionId']) - .where( - 'actionId', - 'in', - results.map((r) => r.id), - ) - .orderBy('id', 'desc') - .execute(), - await this.db.db - .selectFrom('moderation_action_subject_blob') - .selectAll() - .where( - 'actionId', - 'in', - results.map((r) => r.id), - ) - .execute(), - ]) - - const reportIdsByActionId = resolutions.reduce((acc, cur) => { - acc[cur.actionId] ??= [] - acc[cur.actionId].push(cur.id) - return acc - }, {} as Record) - const subjectBlobCidsByActionId = subjectBlobResults.reduce((acc, cur) => { - acc[cur.actionId] ??= [] - acc[cur.actionId].push(cur.cid) - return acc - }, {} as Record) - - const views = results.map((res) => ({ - id: res.id, - action: res.action, - durationInHours: res.durationInHours ?? undefined, - subject: - res.subjectType === 'com.atproto.admin.defs#repoRef' - ? { - $type: 'com.atproto.admin.defs#repoRef', - did: res.subjectDid, - } - : { - $type: 'com.atproto.repo.strongRef', - uri: res.subjectUri, - cid: res.subjectCid, - }, - subjectBlobCids: subjectBlobCidsByActionId[res.id] ?? [], - reason: res.reason, - createdAt: res.createdAt, - createdBy: res.createdBy, - createLabelVals: - res.createLabelVals && res.createLabelVals.length > 0 - ? res.createLabelVals.split(' ') - : undefined, - negateLabelVals: - res.negateLabelVals && res.negateLabelVals.length > 0 - ? res.negateLabelVals.split(' ') - : undefined, - reversal: - res.reversedAt !== null && - res.reversedBy !== null && - res.reversedReason !== null - ? { - createdAt: res.reversedAt, - createdBy: res.reversedBy, - reason: res.reversedReason, - } - : undefined, - resolvedReportIds: reportIdsByActionId[res.id] ?? [], - })) - - return Array.isArray(result) ? views : views[0] - } - - async actionDetail( - result: ActionResult, - opts: ModViewOptions, - ): Promise { - const action = await this.action(result) - const reportResults = action.resolvedReportIds.length - ? await this.db.db - .selectFrom('moderation_report') - .where('id', 'in', action.resolvedReportIds) - .orderBy('id', 'desc') - .selectAll() - .execute() - : [] - const [subject, resolvedReports, subjectBlobs] = await Promise.all([ - this.subject(result, opts), - this.report(reportResults), - this.blob(action.subjectBlobCids), - ]) - return { - id: action.id, - action: action.action, - durationInHours: action.durationInHours, - subject, - subjectBlobs, - createLabelVals: action.createLabelVals, - negateLabelVals: action.negateLabelVals, - reason: action.reason, - createdAt: action.createdAt, - createdBy: action.createdBy, - reversal: action.reversal, - resolvedReports, - } - } - - report(result: ReportResult): Promise - report(result: ReportResult[]): Promise - async report( - result: ReportResult | ReportResult[], - ): Promise { - const results = Array.isArray(result) ? result : [result] - if (results.length === 0) return [] - - const resolutions = await this.db.db - .selectFrom('moderation_report_resolution') - .select(['actionId as id', 'reportId']) - .where( - 'reportId', - 'in', - results.map((r) => r.id), - ) - .orderBy('id', 'desc') - .execute() - - const actionIdsByReportId = resolutions.reduce((acc, cur) => { - acc[cur.reportId] ??= [] - acc[cur.reportId].push(cur.id) - return acc - }, {} as Record) - - const views: ReportView[] = results.map((res) => { - const decoratedView: ReportView = { - id: res.id, - createdAt: res.createdAt, - reasonType: res.reasonType, - reason: res.reason ?? undefined, - reportedBy: res.reportedByDid, - subject: - res.subjectType === 'com.atproto.admin.defs#repoRef' - ? { - $type: 'com.atproto.admin.defs#repoRef', - did: res.subjectDid, - } - : { - $type: 'com.atproto.repo.strongRef', - uri: res.subjectUri, - cid: res.subjectCid, - }, - resolvedByActionIds: actionIdsByReportId[res.id] ?? [], - } - - if (res.handle) { - decoratedView.subjectRepoHandle = res.handle - } - - return decoratedView - }) - - return Array.isArray(result) ? views : views[0] - } - - reportPublic(report: ReportResult): ReportOutput { - return { - id: report.id, - createdAt: report.createdAt, - reasonType: report.reasonType, - reason: report.reason ?? undefined, - reportedBy: report.reportedByDid, - subject: - report.subjectType === 'com.atproto.admin.defs#repoRef' - ? { - $type: 'com.atproto.admin.defs#repoRef', - did: report.subjectDid, - } - : { - $type: 'com.atproto.repo.strongRef', - uri: report.subjectUri, - cid: report.subjectCid, - }, - } - } - - async reportDetail( - result: ReportResult, - opts: ModViewOptions, - ): Promise { - const report = await this.report(result) - const actionResults = report.resolvedByActionIds.length - ? await this.db.db - .selectFrom('moderation_action') - .where('id', 'in', report.resolvedByActionIds) - .orderBy('id', 'desc') - .selectAll() - .execute() - : [] - const [subject, resolvedByActions] = await Promise.all([ - this.subject(result, opts), - this.action(actionResults), - ]) - return { - id: report.id, - createdAt: report.createdAt, - reasonType: report.reasonType, - reason: report.reason ?? undefined, - reportedBy: report.reportedBy, - subject, - resolvedByActions, - } - } - - // Partial view for subjects - - async subject( - result: SubjectResult, - opts: ModViewOptions, - ): Promise { - let subject: SubjectView - if (result.subjectType === 'com.atproto.admin.defs#repoRef') { - const repoResult = await this.services - .account(this.db) - .getAccount(result.subjectDid, true) - if (repoResult) { - subject = await this.repo(repoResult, opts) - subject.$type = 'com.atproto.admin.defs#repoView' - } else { - subject = { did: result.subjectDid } - subject.$type = 'com.atproto.admin.defs#repoViewNotFound' - } - } else if ( - result.subjectType === 'com.atproto.repo.strongRef' && - result.subjectUri !== null - ) { - const recordResult = await this.services - .record(this.db) - .getRecord(new AtUri(result.subjectUri), null, true) - if (recordResult) { - subject = await this.record(recordResult, opts) - subject.$type = 'com.atproto.admin.defs#recordView' - } else { - subject = { uri: result.subjectUri } - subject.$type = 'com.atproto.admin.defs#recordViewNotFound' - } - } else { - throw new Error(`Bad subject data: (${result.id}) ${result.subjectType}`) - } - return subject - } - - // Partial view for blobs - - async blob(cids: string[]): Promise { - if (!cids.length) return [] - const [blobResults, actionResults] = await Promise.all([ - this.db.db - .selectFrom('blob') - .where('cid', 'in', cids) - .selectAll() - .execute(), - this.db.db - .selectFrom('moderation_action') - .where('reversedAt', 'is', null) - .innerJoin( - 'moderation_action_subject_blob as subject_blob', - 'subject_blob.actionId', - 'moderation_action.id', - ) - .select(['id', 'action', 'durationInHours', 'cid']) - .execute(), - ]) - const actionByCid = actionResults.reduce( - (acc, cur) => Object.assign(acc, { [cur.cid]: cur }), - {} as Record>, - ) - return blobResults.map((result) => { - const action = actionByCid[result.cid] - return { - cid: result.cid, - mimeType: result.mimeType, - size: result.size, - createdAt: result.createdAt, - // @TODO support #videoDetails here when we start tracking video length - details: - result.mimeType.startsWith('image/') && - result.height !== null && - result.width !== null - ? { - $type: 'com.atproto.admin.blob#imageDetails', - height: result.height, - width: result.width, - } - : undefined, - moderation: { - currentAction: action - ? { - id: action.id, - action: action.action, - durationInHours: action.durationInHours ?? undefined, - } - : undefined, - }, - } - }) - } -} - -type RepoResult = DidHandle & RepoRoot - -type ActionResult = Selectable - -type ReportResult = ModerationReportRowWithHandle - -type RecordResult = { - uri: string - cid: string - value: object - indexedAt: string - takedownId: number | null -} - -type SubjectResult = Pick< - ActionResult & ReportResult, - 'id' | 'subjectType' | 'subjectDid' | 'subjectUri' | 'subjectCid' -> - -type SubjectView = ActionViewDetail['subject'] & ReportViewDetail['subject'] - -function didFromUri(uri: string) { - return new AtUri(uri).host -} - -export type ModViewOptions = { includeEmails: boolean } diff --git a/packages/pds/src/services/record/index.ts b/packages/pds/src/services/record/index.ts index 1914d1b8c61..de2491c97f3 100644 --- a/packages/pds/src/services/record/index.ts +++ b/packages/pds/src/services/record/index.ts @@ -164,7 +164,7 @@ export class RecordService { cid: string value: object indexedAt: string - takedownId: number | null + takedownRef: string | null } | null> { const { ref } = this.db.db.dynamic let builder = this.db.db @@ -189,7 +189,7 @@ export class RecordService { cid: record.cid, value: cborToLexRecord(record.content), indexedAt: record.indexedAt, - takedownId: record.takedownId, + takedownRef: record.takedownRef ? record.takedownRef.toString() : null, } } diff --git a/packages/pds/src/services/repo/blobs.ts b/packages/pds/src/services/repo/blobs.ts index 2bedb88ecfd..318a5a26c4f 100644 --- a/packages/pds/src/services/repo/blobs.ts +++ b/packages/pds/src/services/repo/blobs.ts @@ -162,7 +162,7 @@ export class RepoBlobs { this.db.db .selectFrom('repo_blob') .selectAll() - .where('takedownId', 'is not', null) + .where('takedownRef', 'is not', null) .whereRef('cid', '=', ref('blob.cid')), ) .executeTakeFirst() diff --git a/packages/pds/tests/account-deletion.test.ts b/packages/pds/tests/account-deletion.test.ts index 12bdad8875a..c9adbaf0c5e 100644 --- a/packages/pds/tests/account-deletion.test.ts +++ b/packages/pds/tests/account-deletion.test.ts @@ -14,7 +14,6 @@ import { RepoBlob } from '../src/db/tables/repo-blob' import { Blob } from '../src/db/tables/blob' import { Record } from '../src/db/tables/record' import { RepoSeq } from '../src/db/tables/repo-seq' -import { ACKNOWLEDGE } from '../src/lexicon/types/com/atproto/admin/defs' describe('account deletion', () => { let network: TestNetworkNoAppView @@ -105,16 +104,14 @@ describe('account deletion', () => { }) it('deletes account with a valid token & password', async () => { - // Perform account deletion, including when there's an existing mod action on the account - await agent.api.com.atproto.admin.takeModerationAction( + // Perform account deletion, including when there's an existing takedown on the account + await agent.api.com.atproto.admin.updateSubjectStatus( { - action: ACKNOWLEDGE, subject: { $type: 'com.atproto.admin.defs#repoRef', did: carol.did, }, - createdBy: 'did:example:admin', - reason: 'X', + takedown: { applied: true }, }, { encoding: 'application/json', diff --git a/packages/pds/tests/admin/__snapshots__/moderation.test.ts.snap b/packages/pds/tests/admin/__snapshots__/moderation.test.ts.snap deleted file mode 100644 index 97c189a5ba3..00000000000 --- a/packages/pds/tests/admin/__snapshots__/moderation.test.ts.snap +++ /dev/null @@ -1,193 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`moderation actioning resolves reports on missing repos and records. 1`] = ` -Object { - "recordActionDetail": Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 4, - "reason": "Y", - "resolvedReports": Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 8, - "reason": "defamation", - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(1)", - "resolvedByActionIds": Array [ - 4, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 7, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(2)", - "resolvedByActionIds": Array [ - 4, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - }, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#recordViewNotFound", - "uri": "record(0)", - }, - "subjectBlobs": Array [], - }, - "repoDeletionActionDetail": Object { - "action": "com.atproto.admin.defs#takedown", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "user(0)", - "id": 3, - "reason": "ACCOUNT DELETION", - "resolvedReports": Array [], - "subject": Object { - "$type": "com.atproto.admin.defs#repoViewNotFound", - "did": "user(0)", - }, - "subjectBlobs": Array [], - }, - "reportADetail": Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 7, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(2)", - "resolvedByActions": Array [ - Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 4, - "reason": "Y", - "resolvedReportIds": Array [ - 8, - 7, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectBlobCids": Array [], - }, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#repoViewNotFound", - "did": "user(0)", - }, - }, - "reportBDetail": Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 8, - "reason": "defamation", - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(1)", - "resolvedByActions": Array [ - Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 4, - "reason": "Y", - "resolvedReportIds": Array [ - 8, - 7, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectBlobCids": Array [], - }, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#recordViewNotFound", - "uri": "record(0)", - }, - }, -} -`; - -exports[`moderation actioning resolves reports on repos and records. 1`] = ` -Object { - "action": "com.atproto.admin.defs#takedown", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 2, - "reason": "Y", - "resolvedReportIds": Array [ - 6, - 5, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - "subjectBlobCids": Array [], -} -`; - -exports[`moderation reporting creates reports of a record. 1`] = ` -Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 3, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(0)", - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 4, - "reason": "defamation", - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(1)", - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(1)", - "uri": "record(1)", - }, - }, -] -`; - -exports[`moderation reporting creates reports of a repo. 1`] = ` -Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(0)", - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(1)", - }, - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 2, - "reason": "impersonation", - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(2)", - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(1)", - }, - }, -] -`; diff --git a/packages/pds/tests/admin/moderation.test.ts b/packages/pds/tests/admin/moderation.test.ts deleted file mode 100644 index c65812adfed..00000000000 --- a/packages/pds/tests/admin/moderation.test.ts +++ /dev/null @@ -1,999 +0,0 @@ -import { - TestNetworkNoAppView, - ImageRef, - RecordRef, - SeedClient, -} from '@atproto/dev-env' -import AtpAgent from '@atproto/api' -import { AtUri } from '@atproto/syntax' -import { BlobNotFoundError } from '@atproto/repo' -import { forSnapshot } from '../_util' -import { PeriodicModerationActionReversal } from '../../src/db/periodic-moderation-action-reversal' -import basicSeed from '../seeds/basic' -import { - ACKNOWLEDGE, - ESCALATE, - FLAG, - TAKEDOWN, -} from '../../src/lexicon/types/com/atproto/admin/defs' -import { - REASONOTHER, - REASONSPAM, -} from '../../src/lexicon/types/com/atproto/moderation/defs' - -describe('moderation', () => { - let network: TestNetworkNoAppView - let agent: AtpAgent - let sc: SeedClient - - beforeAll(async () => { - network = await TestNetworkNoAppView.create({ - dbPostgresSchema: 'moderation', - }) - agent = network.pds.getClient() - sc = network.getSeedClient() - await basicSeed(sc) - }) - - afterAll(async () => { - await network.close() - }) - - describe('reporting', () => { - it('creates reports of a repo.', async () => { - const { data: reportA } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONSPAM, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, - }, - { - headers: sc.getHeaders(sc.dids.alice), - encoding: 'application/json', - }, - ) - const { data: reportB } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONOTHER, - reason: 'impersonation', - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, - }, - { - headers: sc.getHeaders(sc.dids.carol), - encoding: 'application/json', - }, - ) - expect(forSnapshot([reportA, reportB])).toMatchSnapshot() - }) - - it("fails reporting a repo that doesn't exist.", async () => { - const promise = agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONSPAM, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: 'did:plc:unknown', - }, - }, - { headers: sc.getHeaders(sc.dids.alice), encoding: 'application/json' }, - ) - await expect(promise).rejects.toThrow('Repo not found') - }) - - it('creates reports of a record.', async () => { - const postA = sc.posts[sc.dids.bob][0].ref - const postB = sc.posts[sc.dids.bob][1].ref - const { data: reportA } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONSPAM, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postA.uriStr, - cid: postA.cidStr, - }, - }, - { - headers: sc.getHeaders(sc.dids.alice), - encoding: 'application/json', - }, - ) - const { data: reportB } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONOTHER, - reason: 'defamation', - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postB.uriStr, - cid: postB.cidStr, - }, - }, - { - headers: sc.getHeaders(sc.dids.carol), - encoding: 'application/json', - }, - ) - expect(forSnapshot([reportA, reportB])).toMatchSnapshot() - }) - - it("fails reporting a record that doesn't exist.", async () => { - const postA = sc.posts[sc.dids.bob][0].ref - const postB = sc.posts[sc.dids.bob][1].ref - const postUriBad = new AtUri(postA.uriStr) - postUriBad.rkey = 'badrkey' - - const promiseA = agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONSPAM, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postUriBad.toString(), - cid: postA.cidStr, - }, - }, - { headers: sc.getHeaders(sc.dids.alice), encoding: 'application/json' }, - ) - await expect(promiseA).rejects.toThrow('Record not found') - - const promiseB = agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONOTHER, - reason: 'defamation', - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postB.uri.toString(), - cid: postA.cidStr, // bad cid - }, - }, - { headers: sc.getHeaders(sc.dids.carol), encoding: 'application/json' }, - ) - await expect(promiseB).rejects.toThrow('Record not found') - }) - }) - - describe('actioning', () => { - it('resolves reports on repos and records.', async () => { - const { data: reportA } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONSPAM, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, - }, - { - headers: sc.getHeaders(sc.dids.alice), - encoding: 'application/json', - }, - ) - const post = sc.posts[sc.dids.bob][1].ref - const { data: reportB } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONOTHER, - reason: 'defamation', - subject: { - $type: 'com.atproto.repo.strongRef', - uri: post.uri.toString(), - cid: post.cid.toString(), - }, - }, - { - headers: sc.getHeaders(sc.dids.carol), - encoding: 'application/json', - }, - ) - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - const { data: actionResolvedReports } = - await agent.api.com.atproto.admin.resolveModerationReports( - { - actionId: action.id, - reportIds: [reportB.id, reportA.id], - createdBy: 'did:example:admin', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - - expect(forSnapshot(actionResolvedReports)).toMatchSnapshot() - - // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: action.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - }) - - it('resolves reports on missing repos and records.', async () => { - // Create fresh user - const deleteme = await sc.createAccount('deleteme', { - email: 'deleteme.test@bsky.app', - handle: 'deleteme.test', - password: 'password', - }) - const post = await sc.post(deleteme.did, 'delete this post') - // Report user and post - const { data: reportA } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONSPAM, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: deleteme.did, - }, - }, - { - headers: sc.getHeaders(sc.dids.alice), - encoding: 'application/json', - }, - ) - const { data: reportB } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONOTHER, - reason: 'defamation', - subject: { - $type: 'com.atproto.repo.strongRef', - uri: post.ref.uriStr, - cid: post.ref.cidStr, - }, - }, - { - headers: sc.getHeaders(sc.dids.carol), - encoding: 'application/json', - }, - ) - // Delete full user account - await agent.api.com.atproto.server.requestAccountDelete(undefined, { - headers: sc.getHeaders(deleteme.did), - }) - const { token: deletionToken } = await network.pds.ctx.db.db - .selectFrom('email_token') - .where('purpose', '=', 'delete_account') - .where('did', '=', deleteme.did) - .selectAll() - .executeTakeFirstOrThrow() - await agent.api.com.atproto.server.deleteAccount({ - did: deleteme.did, - password: 'password', - token: deletionToken, - }) - await network.processAll() - // Take action on deleted content - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: FLAG, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: post.ref.uriStr, - cid: post.ref.cidStr, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - await agent.api.com.atproto.admin.resolveModerationReports( - { - actionId: action.id, - reportIds: [reportB.id, reportA.id], - createdBy: 'did:example:admin', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - // Check report and action details - const { data: repoDeletionActionDetail } = - await agent.api.com.atproto.admin.getModerationAction( - { id: action.id - 1 }, - { headers: network.pds.adminAuthHeaders() }, - ) - const { data: recordActionDetail } = - await agent.api.com.atproto.admin.getModerationAction( - { id: action.id }, - { headers: network.pds.adminAuthHeaders() }, - ) - const { data: reportADetail } = - await agent.api.com.atproto.admin.getModerationReport( - { id: reportA.id }, - { headers: network.pds.adminAuthHeaders() }, - ) - const { data: reportBDetail } = - await agent.api.com.atproto.admin.getModerationReport( - { id: reportB.id }, - { headers: network.pds.adminAuthHeaders() }, - ) - expect( - forSnapshot({ - repoDeletionActionDetail, - recordActionDetail, - reportADetail, - reportBDetail, - }), - ).toMatchSnapshot() - // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: action.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - }) - - it('does not resolve report for mismatching repo.', async () => { - const { data: report } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONSPAM, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, - }, - { - headers: sc.getHeaders(sc.dids.alice), - encoding: 'application/json', - }, - ) - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.carol, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - - const promise = agent.api.com.atproto.admin.resolveModerationReports( - { - actionId: action.id, - reportIds: [report.id], - createdBy: 'did:example:admin', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - - await expect(promise).rejects.toThrow( - 'Report 9 cannot be resolved by action', - ) - - // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: action.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - }) - - it('does not resolve report for mismatching record.', async () => { - const postRef1 = sc.posts[sc.dids.alice][0].ref - const postRef2 = sc.posts[sc.dids.bob][0].ref - const { data: report } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONSPAM, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postRef1.uriStr, - cid: postRef1.cidStr, - }, - }, - { - headers: sc.getHeaders(sc.dids.alice), - encoding: 'application/json', - }, - ) - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postRef2.uriStr, - cid: postRef2.cidStr, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - - const promise = agent.api.com.atproto.admin.resolveModerationReports( - { - actionId: action.id, - reportIds: [report.id], - createdBy: 'did:example:admin', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - - await expect(promise).rejects.toThrow( - 'Report 10 cannot be resolved by action', - ) - - // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: action.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - }) - - it('supports escalating and acknowledging for triage.', async () => { - const postRef1 = sc.posts[sc.dids.alice][0].ref - const postRef2 = sc.posts[sc.dids.bob][0].ref - const { data: action1 } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: ESCALATE, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postRef1.uri.toString(), - cid: postRef1.cid.toString(), - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders('triage'), - }, - ) - expect(action1).toEqual( - expect.objectContaining({ - action: ESCALATE, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postRef1.uriStr, - cid: postRef1.cidStr, - }, - }), - ) - const { data: action2 } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: ACKNOWLEDGE, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postRef2.uri.toString(), - cid: postRef2.cid.toString(), - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders('triage'), - }, - ) - expect(action2).toEqual( - expect.objectContaining({ - action: ACKNOWLEDGE, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postRef2.uriStr, - cid: postRef2.cidStr, - }, - }), - ) - // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: action1.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders('triage'), - }, - ) - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: action2.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders('triage'), - }, - ) - }) - - it('only allows record to have one current action.', async () => { - const postRef = sc.posts[sc.dids.alice][0].ref - const { data: acknowledge } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: ACKNOWLEDGE, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postRef.uriStr, - cid: postRef.cidStr, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - const flagPromise = agent.api.com.atproto.admin.takeModerationAction( - { - action: FLAG, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postRef.uriStr, - cid: postRef.cidStr, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - await expect(flagPromise).rejects.toThrow( - 'Subject already has an active action:', - ) - - // Reverse current then retry - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: acknowledge.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - const { data: flag } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: FLAG, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postRef.uriStr, - cid: postRef.cidStr, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - - // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: flag.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - }) - - it('only allows repo to have one current action.', async () => { - const { data: acknowledge } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: ACKNOWLEDGE, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.alice, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - const flagPromise = agent.api.com.atproto.admin.takeModerationAction( - { - action: FLAG, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.alice, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - await expect(flagPromise).rejects.toThrow( - 'Subject already has an active action:', - ) - - // Reverse current then retry - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: acknowledge.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - const { data: flag } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: FLAG, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.alice, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - - // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: flag.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - }) - - it('only allows blob to have one current action.', async () => { - const img = sc.posts[sc.dids.carol][0].images[0] - const postA = await sc.post(sc.dids.carol, 'image A', undefined, [img]) - const postB = await sc.post(sc.dids.carol, 'image B', undefined, [img]) - const { data: acknowledge } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: ACKNOWLEDGE, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postA.ref.uriStr, - cid: postA.ref.cidStr, - }, - subjectBlobCids: [img.image.ref.toString()], - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - const flagPromise = agent.api.com.atproto.admin.takeModerationAction( - { - action: FLAG, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postB.ref.uriStr, - cid: postB.ref.cidStr, - }, - subjectBlobCids: [img.image.ref.toString()], - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - await expect(flagPromise).rejects.toThrow( - 'Blob already has an active action:', - ) - // Reverse current then retry - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: acknowledge.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - const { data: flag } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: FLAG, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postB.ref.uriStr, - cid: postB.ref.cidStr, - }, - subjectBlobCids: [img.image.ref.toString()], - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - - // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: flag.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - }) - - it('allows full moderators to takedown.', async () => { - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - createdBy: 'did:example:moderator', - reason: 'Y', - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders('moderator'), - }, - ) - // cleanup - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: action.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - }) - - it('automatically reverses actions marked with duration', async () => { - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - createdBy: 'did:example:moderator', - reason: 'Y', - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, - // Use negative value to set the expiry time in the past so that the action is automatically reversed - // right away without having to wait n number of hours for a successful assertion - durationInHours: -1, - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders('moderator'), - }, - ) - - // In the actual app, this will be instantiated and run on server startup - const periodicReversal = new PeriodicModerationActionReversal( - network.pds.ctx, - ) - await periodicReversal.findAndRevertDueActions() - - const { data: reversedAction } = - await agent.api.com.atproto.admin.getModerationAction( - { id: action.id }, - { headers: network.pds.adminAuthHeaders() }, - ) - - // Verify that the automatic reversal is attributed to the original moderator of the temporary action - // and that the reason is set to indicate that the action was automatically reversed. - expect(reversedAction.reversal).toMatchObject({ - createdBy: action.createdBy, - reason: '[SCHEDULED_REVERSAL] Reverting action as originally scheduled', - }) - }) - - it('does not allow non-full moderators to takedown.', async () => { - const attemptTakedownTriage = - agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - createdBy: 'did:example:moderator', - reason: 'Y', - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders('triage'), - }, - ) - await expect(attemptTakedownTriage).rejects.toThrow( - 'Must be a full moderator to perform an account takedown', - ) - }) - }) - - describe('blob takedown', () => { - let post: { ref: RecordRef; images: ImageRef[] } - let blob: ImageRef - let actionId: number - - beforeAll(async () => { - post = sc.posts[sc.dids.carol][0] - blob = post.images[1] - const takeAction = await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: post.ref.uriStr, - cid: post.ref.cidStr, - }, - subjectBlobCids: [blob.image.ref.toString()], - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - actionId = takeAction.data.id - }) - - it('removes blob from the store', async () => { - const tryGetBytes = network.pds.ctx.blobstore.getBytes(blob.image.ref) - await expect(tryGetBytes).rejects.toThrow(BlobNotFoundError) - }) - - it('prevents blob from being referenced again.', async () => { - const uploaded = await sc.uploadFile( - sc.dids.alice, - 'tests/sample-img/key-alt.jpg', - 'image/jpeg', - ) - expect(uploaded.image.ref.equals(blob.image.ref)).toBeTruthy() - const referenceBlob = sc.post(sc.dids.alice, 'pic', [], [blob]) - await expect(referenceBlob).rejects.toThrow('Could not find blob:') - }) - - it('prevents image blob from being served, even when cached.', async () => { - const attempt = agent.api.com.atproto.sync.getBlob({ - did: sc.dids.carol, - cid: blob.image.ref.toString(), - }) - await expect(attempt).rejects.toThrow('Blob not found') - }) - - it('restores blob when action is reversed.', async () => { - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: actionId, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) - - // Can post and reference blob - const post = await sc.post(sc.dids.alice, 'pic', [], [blob]) - expect(post.images[0].image.ref.equals(blob.image.ref)).toBeTruthy() - - // Can fetch through image server - const res = await agent.api.com.atproto.sync.getBlob({ - did: sc.dids.carol, - cid: blob.image.ref.toString(), - }) - - expect(res.data.byteLength).toBeGreaterThan(9000) - }) - }) -}) diff --git a/packages/pds/tests/auth.test.ts b/packages/pds/tests/auth.test.ts index 542001cddce..650b2d1e9a7 100644 --- a/packages/pds/tests/auth.test.ts +++ b/packages/pds/tests/auth.test.ts @@ -1,7 +1,6 @@ import * as jwt from 'jsonwebtoken' import AtpAgent from '@atproto/api' import { TestNetworkNoAppView, SeedClient } from '@atproto/dev-env' -import { TAKEDOWN } from '@atproto/api/src/client/types/com/atproto/admin/defs' import * as CreateSession from '@atproto/api/src/client/types/com/atproto/server/createSession' import * as RefreshSession from '@atproto/api/src/client/types/com/atproto/server/refreshSession' @@ -243,15 +242,13 @@ describe('auth', () => { email: 'iris@test.com', password: 'password', }) - await agent.api.com.atproto.admin.takeModerationAction( + await agent.api.com.atproto.admin.updateSubjectStatus( { - action: TAKEDOWN, subject: { $type: 'com.atproto.admin.defs#repoRef', did: account.did, }, - createdBy: 'did:example:admin', - reason: 'Y', + takedown: { applied: true }, }, { encoding: 'application/json', @@ -269,15 +266,13 @@ describe('auth', () => { email: 'jared@test.com', password: 'password', }) - await agent.api.com.atproto.admin.takeModerationAction( + await agent.api.com.atproto.admin.updateSubjectStatus( { - action: TAKEDOWN, subject: { $type: 'com.atproto.admin.defs#repoRef', did: account.did, }, - createdBy: 'did:example:admin', - reason: 'Y', + takedown: { applied: true }, }, { encoding: 'application/json', diff --git a/packages/pds/tests/crud.test.ts b/packages/pds/tests/crud.test.ts index c0902e2db29..65544677ff2 100644 --- a/packages/pds/tests/crud.test.ts +++ b/packages/pds/tests/crud.test.ts @@ -13,7 +13,6 @@ import { defaultFetchHandler } from '@atproto/xrpc' import * as Post from '../src/lexicon/types/app/bsky/feed/post' import { paginateAll } from './_util' import AppContext from '../src/context' -import { TAKEDOWN } from '../src/lexicon/types/com/atproto/admin/defs' import { ids } from '../src/lexicon/lexicons' const alice = { @@ -1154,23 +1153,21 @@ describe('crud operations', () => { const posts = await agent.api.app.bsky.feed.post.list({ repo: alice.did }) expect(posts.records.map((r) => r.uri)).toContain(post.uri) - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: created.uri, - cid: created.cid, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: { authorization: network.pds.adminAuth() }, - }, - ) + const subject = { + $type: 'com.atproto.repo.strongRef', + uri: created.uri, + cid: created.cid, + } + await agent.api.com.atproto.admin.updateSubjectStatus( + { + subject, + takedown: { applied: true }, + }, + { + encoding: 'application/json', + headers: { authorization: network.pds.adminAuth() }, + }, + ) const postTakedownPromise = agent.api.app.bsky.feed.post.get({ repo: alice.did, @@ -1183,11 +1180,10 @@ describe('crud operations', () => { expect(postsTakedown.records.map((r) => r.uri)).not.toContain(post.uri) // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( + await agent.api.com.atproto.admin.updateSubjectStatus( { - id: action.id, - createdBy: 'did:example:admin', - reason: 'Y', + subject, + takedown: { applied: false }, }, { encoding: 'application/json', @@ -1200,22 +1196,21 @@ describe('crud operations', () => { const posts = await agent.api.app.bsky.feed.post.list({ repo: alice.did }) expect(posts.records.length).toBeGreaterThan(0) - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: alice.did, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: { authorization: network.pds.adminAuth() }, - }, - ) + const subject = { + $type: 'com.atproto.admin.defs#repoRef', + did: alice.did, + } + + await agent.api.com.atproto.admin.updateSubjectStatus( + { + subject, + takedown: { applied: true }, + }, + { + encoding: 'application/json', + headers: { authorization: network.pds.adminAuth() }, + }, + ) const tryListPosts = agent.api.app.bsky.feed.post.list({ repo: alice.did, @@ -1223,11 +1218,10 @@ describe('crud operations', () => { await expect(tryListPosts).rejects.toThrow(/Could not find repo/) // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( + await agent.api.com.atproto.admin.updateSubjectStatus( { - id: action.id, - createdBy: 'did:example:admin', - reason: 'Y', + subject, + takedown: { applied: false }, }, { encoding: 'application/json', diff --git a/packages/pds/tests/db.test.ts b/packages/pds/tests/db.test.ts index 1a2a42f0930..d5a87595ac8 100644 --- a/packages/pds/tests/db.test.ts +++ b/packages/pds/tests/db.test.ts @@ -52,7 +52,7 @@ describe('db', () => { root: 'x', rev: 'x', indexedAt: 'bad-date', - takedownId: null, + takedownRef: null, }) }) diff --git a/packages/pds/tests/invite-codes.test.ts b/packages/pds/tests/invite-codes.test.ts index f406b77cc3b..e48e1b46fc7 100644 --- a/packages/pds/tests/invite-codes.test.ts +++ b/packages/pds/tests/invite-codes.test.ts @@ -4,7 +4,6 @@ import { TestNetworkNoAppView } from '@atproto/dev-env' import { AppContext } from '../src' import { DAY } from '@atproto/common' import { genInvCodes } from '../src/api/com/atproto/server/util' -import { TAKEDOWN } from '@atproto/api/src/client/types/com/atproto/admin/defs' describe('account', () => { let network: TestNetworkNoAppView @@ -50,22 +49,20 @@ describe('account', () => { // assign an invite code to the user const code = await createInviteCode(network, agent, 1, account.did) // takedown the user's account - const { data: takedownAction } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: account.did, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), - }, - ) + const subject = { + $type: 'com.atproto.admin.defs#repoRef', + did: account.did, + } + await agent.api.com.atproto.admin.updateSubjectStatus( + { + subject, + takedown: { applied: true }, + }, + { + encoding: 'application/json', + headers: network.pds.adminAuthHeaders(), + }, + ) // attempt to create account with the previously generated invite code const promise = createAccountWithInvite(agent, code) await expect(promise).rejects.toThrow( @@ -73,11 +70,10 @@ describe('account', () => { ) // double check that reversing the takedown action makes the invite code valid again - await agent.api.com.atproto.admin.reverseModerationAction( + await agent.api.com.atproto.admin.updateSubjectStatus( { - id: takedownAction.id, - createdBy: 'did:example:admin', - reason: 'Y', + subject, + takedown: { applied: false }, }, { encoding: 'application/json', diff --git a/packages/pds/tests/admin/invites.test.ts b/packages/pds/tests/invites-admin.test.ts similarity index 91% rename from packages/pds/tests/admin/invites.test.ts rename to packages/pds/tests/invites-admin.test.ts index 4f52400a314..d971b75285c 100644 --- a/packages/pds/tests/admin/invites.test.ts +++ b/packages/pds/tests/invites-admin.test.ts @@ -167,20 +167,8 @@ describe('pds admin invite views', () => { expect(combined).toEqual(full.data.codes) }) - it('filters admin.searchRepos by invitedBy', async () => { - const searchView = await agent.api.com.atproto.admin.searchRepos( - { invitedBy: alice }, - { headers: network.pds.adminAuthHeaders() }, - ) - expect(searchView.data.repos.length).toBe(2) - expect(searchView.data.repos[0].invitedBy?.available).toBe(1) - expect(searchView.data.repos[0].invitedBy?.uses.length).toBe(1) - expect(searchView.data.repos[1].invitedBy?.available).toBe(1) - expect(searchView.data.repos[1].invitedBy?.uses.length).toBe(1) - }) - - it('hydrates invites into admin.getRepo', async () => { - const aliceView = await agent.api.com.atproto.admin.getRepo( + it('hydrates invites into admin.getAccountInfo', async () => { + const aliceView = await agent.api.com.atproto.admin.getAccountInfo( { did: alice }, { headers: network.pds.adminAuthHeaders() }, ) @@ -221,7 +209,7 @@ describe('pds admin invite views', () => { { encoding: 'application/json', headers: network.pds.adminAuthHeaders() }, ) - const repoRes = await agent.api.com.atproto.admin.getRepo( + const repoRes = await agent.api.com.atproto.admin.getAccountInfo( { did: carol }, { headers: network.pds.adminAuthHeaders() }, ) @@ -243,7 +231,7 @@ describe('pds admin invite views', () => { { encoding: 'application/json', headers: network.pds.adminAuthHeaders() }, ) - const afterEnable = await agent.api.com.atproto.admin.getRepo( + const afterEnable = await agent.api.com.atproto.admin.getAccountInfo( { did: carol }, { headers: network.pds.adminAuthHeaders() }, ) @@ -255,7 +243,7 @@ describe('pds admin invite views', () => { { encoding: 'application/json', headers: network.pds.adminAuthHeaders() }, ) - const afterDisable = await agent.api.com.atproto.admin.getRepo( + const afterDisable = await agent.api.com.atproto.admin.getAccountInfo( { did: carol }, { headers: network.pds.adminAuthHeaders() }, ) @@ -290,7 +278,7 @@ describe('pds admin invite views', () => { { encoding: 'application/json', headers: network.pds.adminAuthHeaders() }, ) - const repoRes = await agent.api.com.atproto.admin.getRepo( + const repoRes = await agent.api.com.atproto.admin.getAccountInfo( { did: carol }, { headers: network.pds.adminAuthHeaders() }, ) diff --git a/packages/pds/tests/moderation.test.ts b/packages/pds/tests/moderation.test.ts new file mode 100644 index 00000000000..ee68bb7aab5 --- /dev/null +++ b/packages/pds/tests/moderation.test.ts @@ -0,0 +1,357 @@ +import { TestNetworkNoAppView, ImageRef, SeedClient } from '@atproto/dev-env' +import AtpAgent from '@atproto/api' +import { BlobNotFoundError } from '@atproto/repo' +import { Secp256k1Keypair } from '@atproto/crypto' +import { createServiceAuthHeaders } from '@atproto/xrpc-server' +import basicSeed from './seeds/basic' +import { + RepoBlobRef, + RepoRef, +} from '../src/lexicon/types/com/atproto/admin/defs' +import { Main as StrongRef } from '../src/lexicon/types/com/atproto/repo/strongRef' + +describe('moderation', () => { + let network: TestNetworkNoAppView + let agent: AtpAgent + let sc: SeedClient + + let repoSubject: RepoRef + let recordSubject: StrongRef + let blobSubject: RepoBlobRef + let blobRef: ImageRef + + const appviewDid = 'did:example:appview' + const altAppviewDid = 'did:example:alt' + let appviewKey: Secp256k1Keypair + + beforeAll(async () => { + network = await TestNetworkNoAppView.create({ + dbPostgresSchema: 'moderation', + pds: { + bskyAppViewDid: appviewDid, + }, + }) + + appviewKey = await Secp256k1Keypair.create() + const origResolve = network.pds.ctx.idResolver.did.resolveAtprotoKey + network.pds.ctx.idResolver.did.resolveAtprotoKey = async ( + did: string, + forceRefresh?: boolean, + ) => { + if (did === appviewDid || did === altAppviewDid) { + return appviewKey.did() + } + return origResolve(did, forceRefresh) + } + + agent = network.pds.getClient() + sc = network.getSeedClient() + await basicSeed(sc) + await network.processAll() + repoSubject = { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.bob, + } + const post = sc.posts[sc.dids.carol][0] + recordSubject = { + $type: 'com.atproto.repo.strongRef', + uri: post.ref.uriStr, + cid: post.ref.cidStr, + } + blobRef = post.images[1] + blobSubject = { + $type: 'com.atproto.admin.defs#repoBlobRef', + did: sc.dids.carol, + cid: blobRef.image.ref.toString(), + } + }) + + afterAll(async () => { + await network.close() + }) + + it('takes down accounts', async () => { + await agent.api.com.atproto.admin.updateSubjectStatus( + { + subject: repoSubject, + takedown: { applied: true, ref: 'test-repo' }, + }, + { + encoding: 'application/json', + headers: network.pds.adminAuthHeaders('moderator'), + }, + ) + const res = await agent.api.com.atproto.admin.getSubjectStatus( + { + did: repoSubject.did, + }, + { headers: network.pds.adminAuthHeaders('moderator') }, + ) + expect(res.data.subject.did).toEqual(sc.dids.bob) + expect(res.data.takedown?.applied).toBe(true) + expect(res.data.takedown?.ref).toBe('test-repo') + }) + + it('restores takendown accounts', async () => { + await agent.api.com.atproto.admin.updateSubjectStatus( + { + subject: repoSubject, + takedown: { applied: false }, + }, + { + encoding: 'application/json', + headers: network.pds.adminAuthHeaders('moderator'), + }, + ) + const res = await agent.api.com.atproto.admin.getSubjectStatus( + { + did: repoSubject.did, + }, + { headers: network.pds.adminAuthHeaders('moderator') }, + ) + expect(res.data.subject.did).toEqual(sc.dids.bob) + expect(res.data.takedown?.applied).toBe(false) + expect(res.data.takedown?.ref).toBeUndefined() + }) + + it('takes down records', async () => { + await agent.api.com.atproto.admin.updateSubjectStatus( + { + subject: recordSubject, + takedown: { applied: true, ref: 'test-record' }, + }, + { + encoding: 'application/json', + headers: network.pds.adminAuthHeaders('moderator'), + }, + ) + const res = await agent.api.com.atproto.admin.getSubjectStatus( + { + uri: recordSubject.uri, + }, + { headers: network.pds.adminAuthHeaders('moderator') }, + ) + expect(res.data.subject.uri).toEqual(recordSubject.uri) + expect(res.data.takedown?.applied).toBe(true) + expect(res.data.takedown?.ref).toBe('test-record') + }) + + it('restores takendown records', async () => { + await agent.api.com.atproto.admin.updateSubjectStatus( + { + subject: recordSubject, + takedown: { applied: false }, + }, + { + encoding: 'application/json', + headers: network.pds.adminAuthHeaders('moderator'), + }, + ) + const res = await agent.api.com.atproto.admin.getSubjectStatus( + { + uri: recordSubject.uri, + }, + { headers: network.pds.adminAuthHeaders('moderator') }, + ) + expect(res.data.subject.uri).toEqual(recordSubject.uri) + expect(res.data.takedown?.applied).toBe(false) + expect(res.data.takedown?.ref).toBeUndefined() + }) + + it('does not allow non-full moderators to update subject state', async () => { + const subject = { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.bob, + } + const attemptTakedownTriage = + agent.api.com.atproto.admin.updateSubjectStatus( + { + subject, + takedown: { applied: true }, + }, + { + encoding: 'application/json', + headers: network.pds.adminAuthHeaders('triage'), + }, + ) + await expect(attemptTakedownTriage).rejects.toThrow( + 'Must be a full moderator to update subject state', + ) + const res = await agent.api.com.atproto.admin.getSubjectStatus( + { + did: subject.did, + }, + { headers: network.pds.adminAuthHeaders('moderator') }, + ) + expect(res.data.takedown?.applied).toBe(false) + }) + + describe('blob takedown', () => { + it('takes down blobs', async () => { + await agent.api.com.atproto.admin.updateSubjectStatus( + { + subject: blobSubject, + takedown: { applied: true, ref: 'test-blob' }, + }, + { + encoding: 'application/json', + headers: network.pds.adminAuthHeaders(), + }, + ) + const res = await agent.api.com.atproto.admin.getSubjectStatus( + { + did: blobSubject.did, + blob: blobSubject.cid, + }, + { headers: network.pds.adminAuthHeaders('moderator') }, + ) + expect(res.data.subject.did).toEqual(blobSubject.did) + expect(res.data.subject.cid).toEqual(blobSubject.cid) + expect(res.data.takedown?.applied).toBe(true) + expect(res.data.takedown?.ref).toBe('test-blob') + }) + + it('removes blob from the store', async () => { + const tryGetBytes = network.pds.ctx.blobstore.getBytes(blobRef.image.ref) + await expect(tryGetBytes).rejects.toThrow(BlobNotFoundError) + }) + + it('prevents blob from being referenced again.', async () => { + const uploaded = await sc.uploadFile( + sc.dids.alice, + 'tests/sample-img/key-alt.jpg', + 'image/jpeg', + ) + expect(uploaded.image.ref.equals(blobRef.image.ref)).toBeTruthy() + const referenceBlob = sc.post(sc.dids.alice, 'pic', [], [blobRef]) + await expect(referenceBlob).rejects.toThrow('Could not find blob:') + }) + + it('prevents image blob from being served, even when cached.', async () => { + const attempt = agent.api.com.atproto.sync.getBlob({ + did: sc.dids.carol, + cid: blobRef.image.ref.toString(), + }) + await expect(attempt).rejects.toThrow('Blob not found') + }) + + it('restores blob when takedown is removed', async () => { + await agent.api.com.atproto.admin.updateSubjectStatus( + { + subject: blobSubject, + takedown: { applied: false }, + }, + { + encoding: 'application/json', + headers: network.pds.adminAuthHeaders(), + }, + ) + + // Can post and reference blob + const post = await sc.post(sc.dids.alice, 'pic', [], [blobRef]) + expect(post.images[0].image.ref.equals(blobRef.image.ref)).toBeTruthy() + + // Can fetch through image server + const res = await agent.api.com.atproto.sync.getBlob({ + did: sc.dids.carol, + cid: blobRef.image.ref.toString(), + }) + + expect(res.data.byteLength).toBeGreaterThan(9000) + }) + }) + + describe('auth', () => { + it('allows service auth requests from the configured appview did', async () => { + const headers = await createServiceAuthHeaders({ + iss: appviewDid, + aud: repoSubject.did, + keypair: appviewKey, + }) + await agent.api.com.atproto.admin.updateSubjectStatus( + { + subject: repoSubject, + takedown: { applied: true, ref: 'test-repo' }, + }, + { + ...headers, + encoding: 'application/json', + }, + ) + + const res = await agent.api.com.atproto.admin.getSubjectStatus( + { + did: repoSubject.did, + }, + headers, + ) + expect(res.data.subject.did).toBe(repoSubject.did) + expect(res.data.takedown?.applied).toBe(true) + }) + + it('does not allow requests from another did', async () => { + const headers = await createServiceAuthHeaders({ + iss: altAppviewDid, + aud: repoSubject.did, + keypair: appviewKey, + }) + const attempt = agent.api.com.atproto.admin.updateSubjectStatus( + { + subject: repoSubject, + takedown: { applied: true, ref: 'test-repo' }, + }, + { + ...headers, + encoding: 'application/json', + }, + ) + await expect(attempt).rejects.toThrow( + 'Untrusted issuer for admin actions', + ) + }) + + it('does not allow requests with a bad signature', async () => { + const badKey = await Secp256k1Keypair.create() + const headers = await createServiceAuthHeaders({ + iss: appviewDid, + aud: repoSubject.did, + keypair: badKey, + }) + const attempt = agent.api.com.atproto.admin.updateSubjectStatus( + { + subject: repoSubject, + takedown: { applied: true, ref: 'test-repo' }, + }, + { + ...headers, + encoding: 'application/json', + }, + ) + await expect(attempt).rejects.toThrow( + 'jwt signature does not match jwt issuer', + ) + }) + + it('does not allow requests with a bad signature', async () => { + // repo subject is bob, so we set alice as the audience + const headers = await createServiceAuthHeaders({ + iss: appviewDid, + aud: sc.dids.alice, + keypair: appviewKey, + }) + const attempt = agent.api.com.atproto.admin.updateSubjectStatus( + { + subject: repoSubject, + takedown: { applied: true, ref: 'test-repo' }, + }, + { + ...headers, + encoding: 'application/json', + }, + ) + await expect(attempt).rejects.toThrow( + 'jwt audience does not match account did', + ) + }) + }) +}) diff --git a/packages/pds/tests/proxied/admin.test.ts b/packages/pds/tests/proxied/admin.test.ts index 23c801cd6b2..8b4fffae9e1 100644 --- a/packages/pds/tests/proxied/admin.test.ts +++ b/packages/pds/tests/proxied/admin.test.ts @@ -37,7 +37,10 @@ describe('proxies admin requests', () => { headers: network.pds.adminAuthHeaders(), }, ) - await basicSeed(sc, invite) + await basicSeed(sc, { + inviteCode: invite.code, + addModLabels: true, + }) await network.processAll() }) diff --git a/packages/pds/tests/proxied/feedgen.test.ts b/packages/pds/tests/proxied/feedgen.test.ts index 142d1235497..6f06ce0d020 100644 --- a/packages/pds/tests/proxied/feedgen.test.ts +++ b/packages/pds/tests/proxied/feedgen.test.ts @@ -23,7 +23,7 @@ describe('feedgen proxy view', () => { }) agent = network.pds.getClient() sc = network.getSeedClient() - await basicSeed(sc) + await basicSeed(sc, { addModLabels: true }) // publish feed const feed = await agent.api.app.bsky.feed.generator.create( { repo: sc.dids.alice, rkey: feedUri.rkey }, diff --git a/packages/pds/tests/proxied/notif.test.ts b/packages/pds/tests/proxied/notif.test.ts index fb3de2b8fe7..4fc559ee120 100644 --- a/packages/pds/tests/proxied/notif.test.ts +++ b/packages/pds/tests/proxied/notif.test.ts @@ -70,7 +70,7 @@ describe('notif service proxy', () => { notifDid, async () => network.pds.ctx.repoSigningKey.did(), ) - expect(auth).toEqual(sc.dids.bob) + expect(auth.iss).toEqual(sc.dids.bob) }) }) diff --git a/packages/pds/tests/proxied/procedures.test.ts b/packages/pds/tests/proxied/procedures.test.ts index 00dd02863ce..8c246e38da7 100644 --- a/packages/pds/tests/proxied/procedures.test.ts +++ b/packages/pds/tests/proxied/procedures.test.ts @@ -17,7 +17,7 @@ describe('proxies appview procedures', () => { }) agent = network.pds.getClient() sc = network.getSeedClient() - await basicSeed(sc) + await basicSeed(sc, { addModLabels: true }) await network.processAll() alice = sc.dids.alice bob = sc.dids.bob diff --git a/packages/pds/tests/proxied/read-after-write.test.ts b/packages/pds/tests/proxied/read-after-write.test.ts index 1d7d1412c11..1e9e4125084 100644 --- a/packages/pds/tests/proxied/read-after-write.test.ts +++ b/packages/pds/tests/proxied/read-after-write.test.ts @@ -21,7 +21,7 @@ describe('proxy read after write', () => { }) agent = network.pds.getClient() sc = network.getSeedClient() - await basicSeed(sc) + await basicSeed(sc, { addModLabels: true }) await network.processAll() alice = sc.dids.alice carol = sc.dids.carol diff --git a/packages/pds/tests/proxied/views.test.ts b/packages/pds/tests/proxied/views.test.ts index 13fa41174b4..94b76719d70 100644 --- a/packages/pds/tests/proxied/views.test.ts +++ b/packages/pds/tests/proxied/views.test.ts @@ -19,7 +19,7 @@ describe('proxies view requests', () => { }) agent = network.pds.getClient() sc = network.getSeedClient() - await basicSeed(sc) + await basicSeed(sc, { addModLabels: true }) alice = sc.dids.alice bob = sc.dids.bob carol = sc.dids.carol diff --git a/packages/pds/tests/seeds/basic.ts b/packages/pds/tests/seeds/basic.ts index 3d045fc9239..1f71b58ff63 100644 --- a/packages/pds/tests/seeds/basic.ts +++ b/packages/pds/tests/seeds/basic.ts @@ -3,8 +3,11 @@ import { ids } from '../../src/lexicon/lexicons' import { FLAG } from '../../src/lexicon/types/com/atproto/admin/defs' import usersSeed from './users' -export default async (sc: SeedClient, invite?: { code: string }) => { - await usersSeed(sc, invite) +export default async ( + sc: SeedClient, + opts?: { inviteCode?: string; addModLabels?: boolean }, +) => { + await usersSeed(sc, opts) const alice = sc.dids.alice const bob = sc.dids.bob @@ -128,22 +131,24 @@ export default async (sc: SeedClient, invite?: { code: string }) => { await sc.repost(dan, sc.posts[alice][1].ref) await sc.repost(dan, alicesReplyToBob.ref) - await sc.agent.com.atproto.admin.takeModerationAction( - { - action: FLAG, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: dan, + if (opts?.addModLabels) { + await sc.agent.com.atproto.admin.takeModerationAction( + { + action: FLAG, + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: dan, + }, + createdBy: 'did:example:admin', + reason: 'test', + createLabelVals: ['repo-action-label'], }, - createdBy: 'did:example:admin', - reason: 'test', - createLabelVals: ['repo-action-label'], - }, - { - encoding: 'application/json', - headers: sc.adminAuthHeaders(), - }, - ) + { + encoding: 'application/json', + headers: sc.adminAuthHeaders(), + }, + ) + } return sc } diff --git a/packages/pds/tests/seeds/users.ts b/packages/pds/tests/seeds/users.ts index bfe6f9abe1c..a142954ac68 100644 --- a/packages/pds/tests/seeds/users.ts +++ b/packages/pds/tests/seeds/users.ts @@ -1,10 +1,16 @@ import { SeedClient } from '@atproto/dev-env' -export default async (sc: SeedClient, invite?: { code: string }) => { - await sc.createAccount('alice', { ...users.alice, inviteCode: invite?.code }) - await sc.createAccount('bob', { ...users.bob, inviteCode: invite?.code }) - await sc.createAccount('carol', { ...users.carol, inviteCode: invite?.code }) - await sc.createAccount('dan', { ...users.dan, inviteCode: invite?.code }) +export default async (sc: SeedClient, opts?: { inviteCode?: string }) => { + await sc.createAccount('alice', { + ...users.alice, + inviteCode: opts?.inviteCode, + }) + await sc.createAccount('bob', { ...users.bob, inviteCode: opts?.inviteCode }) + await sc.createAccount('carol', { + ...users.carol, + inviteCode: opts?.inviteCode, + }) + await sc.createAccount('dan', { ...users.dan, inviteCode: opts?.inviteCode }) await sc.createProfile( sc.dids.alice, diff --git a/packages/pds/tests/sync/sync.test.ts b/packages/pds/tests/sync/sync.test.ts index 424ebc86337..4f99b3bb08c 100644 --- a/packages/pds/tests/sync/sync.test.ts +++ b/packages/pds/tests/sync/sync.test.ts @@ -5,7 +5,6 @@ import { randomStr } from '@atproto/crypto' import * as repo from '@atproto/repo' import { MemoryBlockstore } from '@atproto/repo' import { AtUri } from '@atproto/syntax' -import { TAKEDOWN } from '@atproto/api/src/client/types/com/atproto/admin/defs' import { CID } from 'multiformats/cid' import { AppContext } from '../../src' @@ -34,7 +33,6 @@ describe('repo sync', () => { password: 'alice-pass', }) did = sc.dids.alice - agent.api.setHeader('authorization', `Bearer ${sc.accounts[did].accessJwt}`) }) afterAll(async () => { @@ -83,11 +81,7 @@ describe('repo sync', () => { // delete two that are already sync & two that have not been for (let i = 0; i < DEL_COUNT; i++) { const uri = uris[i * 5] - await agent.api.app.bsky.feed.post.delete({ - repo: did, - collection: uri.collection, - rkey: uri.rkey, - }) + await sc.deletePost(did, uri) delete repoData[uri.collection][uri.rkey] } @@ -203,14 +197,19 @@ describe('repo sync', () => { describe('repo takedown', () => { beforeAll(async () => { - await sc.takeModerationAction({ - action: TAKEDOWN, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did, + await agent.api.com.atproto.admin.updateSubjectStatus( + { + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did, + }, + takedown: { applied: true }, }, - }) - agent.api.xrpc.unsetHeader('authorization') + { + encoding: 'application/json', + headers: network.pds.adminAuthHeaders(), + }, + ) }) it('does not sync repo unauthed', async () => { diff --git a/packages/xrpc-server/src/auth.ts b/packages/xrpc-server/src/auth.ts index eb4dfc537c8..9283b13815b 100644 --- a/packages/xrpc-server/src/auth.ts +++ b/packages/xrpc-server/src/auth.ts @@ -4,10 +4,13 @@ import * as crypto from '@atproto/crypto' import * as ui8 from 'uint8arrays' import { AuthRequiredError } from './types' -type ServiceJwtParams = { +type ServiceJwtPayload = { iss: string aud: string exp?: number +} + +type ServiceJwtParams = ServiceJwtPayload & { keypair: crypto.Keypair } @@ -46,7 +49,7 @@ export const verifyJwt = async ( jwtStr: string, ownDid: string | null, // null indicates to skip the audience check getSigningKey: (did: string) => Promise, -): Promise => { +): Promise => { const parts = jwtStr.split('.') if (parts.length !== 3) { throw new AuthRequiredError('poorly formatted jwt', 'BadJwt') @@ -85,7 +88,7 @@ export const verifyJwt = async ( ) } - return payload.iss + return payload } const parseB64UrlToJson = (b64: string) => { diff --git a/services/bsky/api.js b/services/bsky/api.js index 4ae71b17760..fac5b0a7c8b 100644 --- a/services/bsky/api.js +++ b/services/bsky/api.js @@ -18,6 +18,7 @@ const { CloudfrontInvalidator, MultiImageInvalidator, } = require('@atproto/aws') +const { Secp256k1Keypair } = require('@atproto/crypto') const { DatabaseCoordinator, PrimaryDatabase, @@ -64,6 +65,8 @@ const main = async () => { blobCacheLocation: env.blobCacheLocation, }) + const signingKey = await Secp256k1Keypair.import(env.serviceSigningKey) + // configure zero, one, or both image invalidators let imgInvalidator const bunnyInvalidator = env.bunnyAccessKey @@ -93,6 +96,7 @@ const main = async () => { const algos = env.feedPublisherDid ? makeAlgos(env.feedPublisherDid) : {} const bsky = BskyAppView.create({ db, + signingKey, config: cfg, imgInvalidator, algos, @@ -146,6 +150,7 @@ const getEnv = () => ({ dbPoolSize: maybeParseInt(process.env.DB_POOL_SIZE), dbPoolMaxUses: maybeParseInt(process.env.DB_POOL_MAX_USES), dbPoolIdleTimeoutMs: maybeParseInt(process.env.DB_POOL_IDLE_TIMEOUT_MS), + serviceSigningKey: process.env.SERVICE_SIGNING_KEY, publicUrl: process.env.PUBLIC_URL, didPlcUrl: process.env.DID_PLC_URL, imgUriSalt: process.env.IMG_URI_SALT, From b28fdb2ca464ef83c976acb0de04a2e4905dc951 Mon Sep 17 00:00:00 2001 From: YOCKOW Date: Tue, 31 Oct 2023 08:27:36 +0900 Subject: [PATCH 11/59] PDS: Allow configuring non-AWS S3 blob storage. (#1729) * PDS: Allow configuring non-AWS S3 blob storage. See https://github.com/bluesky-social/atproto/issues/1583 * tidy --------- Co-authored-by: devin ivy --- packages/pds/src/config/config.ts | 12 ++++++++++++ packages/pds/src/config/env.ts | 8 ++++++++ packages/pds/src/context.ts | 7 ++++++- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/packages/pds/src/config/config.ts b/packages/pds/src/config/config.ts index 58040abd781..ab43773dd8e 100644 --- a/packages/pds/src/config/config.ts +++ b/packages/pds/src/config/config.ts @@ -55,6 +55,15 @@ export const envToCfg = (env: ServerEnvironment): ServerConfig => { } if (env.blobstoreS3Bucket) { blobstoreCfg = { provider: 's3', bucket: env.blobstoreS3Bucket } + if (env.blobstoreS3Region) { + blobstoreCfg.region = env.blobstoreS3Region + } + if (env.blobstoreS3Endpoint) { + blobstoreCfg.endpoint = env.blobstoreS3Endpoint + } + if (env.blobstoreS3ForcePathStyle !== undefined) { + blobstoreCfg.forcePathStyle = env.blobstoreS3ForcePathStyle + } } else if (env.blobstoreDiskLocation) { blobstoreCfg = { provider: 'disk', @@ -238,6 +247,9 @@ export type PostgresConfig = { export type S3BlobstoreConfig = { provider: 's3' bucket: string + region?: string + endpoint?: string + forcePathStyle?: boolean } export type DiskBlobstoreConfig = { diff --git a/packages/pds/src/config/env.ts b/packages/pds/src/config/env.ts index 2c13124b4c9..3948d633883 100644 --- a/packages/pds/src/config/env.ts +++ b/packages/pds/src/config/env.ts @@ -24,6 +24,9 @@ export const readEnv = (): ServerEnvironment => { // blobstore: one required // s3 blobstoreS3Bucket: envStr('PDS_BLOBSTORE_S3_BUCKET'), + blobstoreS3Region: envStr('PDS_BLOBSTORE_S3_REGION'), + blobstoreS3Endpoint: envStr('PDS_BLOBSTORE_S3_ENDPOINT'), + blobstoreS3ForcePathStyle: envBool('PDS_BLOBSTORE_S3_FORCE_PATH_STYLE'), // disk blobstoreDiskLocation: envStr('PDS_BLOBSTORE_DISK_LOCATION'), blobstoreDiskTmpLocation: envStr('PDS_BLOBSTORE_DISK_TMP_LOCATION'), @@ -118,6 +121,11 @@ export type ServerEnvironment = { blobstoreDiskLocation?: string blobstoreDiskTmpLocation?: string + // -- optional s3 parameters + blobstoreS3Region?: string + blobstoreS3Endpoint?: string + blobstoreS3ForcePathStyle?: boolean + // identity didPlcUrl?: string didCacheStaleTTL?: number diff --git a/packages/pds/src/context.ts b/packages/pds/src/context.ts index b181ea8b3ad..dfa969e84f7 100644 --- a/packages/pds/src/context.ts +++ b/packages/pds/src/context.ts @@ -103,7 +103,12 @@ export class AppContext { }) const blobstore = cfg.blobstore.provider === 's3' - ? new S3BlobStore({ bucket: cfg.blobstore.bucket }) + ? new S3BlobStore({ + bucket: cfg.blobstore.bucket, + region: cfg.blobstore.region, + endpoint: cfg.blobstore.endpoint, + forcePathStyle: cfg.blobstore.forcePathStyle, + }) : await DiskBlobStore.create( cfg.blobstore.location, cfg.blobstore.tempLocation, From 2df6f2b836509aed83b7559c644f63797f617861 Mon Sep 17 00:00:00 2001 From: devin ivy Date: Tue, 31 Oct 2023 00:00:00 -0400 Subject: [PATCH 12/59] Support S3-compatible credentials for pds blobstore (#1787) * support s3 credentials for pds blobstore * tidy --- packages/pds/src/config/config.ts | 28 ++++++++++++++++++++-------- packages/pds/src/config/env.ts | 4 ++++ packages/pds/src/context.ts | 1 + 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/packages/pds/src/config/config.ts b/packages/pds/src/config/config.ts index ab43773dd8e..75227099835 100644 --- a/packages/pds/src/config/config.ts +++ b/packages/pds/src/config/config.ts @@ -54,15 +54,23 @@ export const envToCfg = (env: ServerEnvironment): ServerConfig => { throw new Error('Cannot set both S3 and disk blobstore env vars') } if (env.blobstoreS3Bucket) { - blobstoreCfg = { provider: 's3', bucket: env.blobstoreS3Bucket } - if (env.blobstoreS3Region) { - blobstoreCfg.region = env.blobstoreS3Region - } - if (env.blobstoreS3Endpoint) { - blobstoreCfg.endpoint = env.blobstoreS3Endpoint + blobstoreCfg = { + provider: 's3', + bucket: env.blobstoreS3Bucket, + region: env.blobstoreS3Region, + endpoint: env.blobstoreS3Endpoint, + forcePathStyle: env.blobstoreS3ForcePathStyle, } - if (env.blobstoreS3ForcePathStyle !== undefined) { - blobstoreCfg.forcePathStyle = env.blobstoreS3ForcePathStyle + if (env.blobstoreS3AccessKeyId || env.blobstoreS3SecretAccessKey) { + if (!env.blobstoreS3AccessKeyId || !env.blobstoreS3SecretAccessKey) { + throw new Error( + 'Must specify both S3 access key id and secret access key blobstore env vars', + ) + } + blobstoreCfg.credentials = { + accessKeyId: env.blobstoreS3AccessKeyId, + secretAccessKey: env.blobstoreS3SecretAccessKey, + } } } else if (env.blobstoreDiskLocation) { blobstoreCfg = { @@ -250,6 +258,10 @@ export type S3BlobstoreConfig = { region?: string endpoint?: string forcePathStyle?: boolean + credentials?: { + accessKeyId: string + secretAccessKey: string + } } export type DiskBlobstoreConfig = { diff --git a/packages/pds/src/config/env.ts b/packages/pds/src/config/env.ts index 3948d633883..a7f1c2636af 100644 --- a/packages/pds/src/config/env.ts +++ b/packages/pds/src/config/env.ts @@ -27,6 +27,8 @@ export const readEnv = (): ServerEnvironment => { blobstoreS3Region: envStr('PDS_BLOBSTORE_S3_REGION'), blobstoreS3Endpoint: envStr('PDS_BLOBSTORE_S3_ENDPOINT'), blobstoreS3ForcePathStyle: envBool('PDS_BLOBSTORE_S3_FORCE_PATH_STYLE'), + blobstoreS3AccessKeyId: envStr('PDS_BLOBSTORE_S3_ACCESS_KEY_ID'), + blobstoreS3SecretAccessKey: envStr('PDS_BLOBSTORE_S3_SECRET_ACCESS_KEY'), // disk blobstoreDiskLocation: envStr('PDS_BLOBSTORE_DISK_LOCATION'), blobstoreDiskTmpLocation: envStr('PDS_BLOBSTORE_DISK_TMP_LOCATION'), @@ -125,6 +127,8 @@ export type ServerEnvironment = { blobstoreS3Region?: string blobstoreS3Endpoint?: string blobstoreS3ForcePathStyle?: boolean + blobstoreS3AccessKeyId?: string + blobstoreS3SecretAccessKey?: string // identity didPlcUrl?: string diff --git a/packages/pds/src/context.ts b/packages/pds/src/context.ts index dfa969e84f7..56eead1dace 100644 --- a/packages/pds/src/context.ts +++ b/packages/pds/src/context.ts @@ -108,6 +108,7 @@ export class AppContext { region: cfg.blobstore.region, endpoint: cfg.blobstore.endpoint, forcePathStyle: cfg.blobstore.forcePathStyle, + credentials: cfg.blobstore.credentials, }) : await DiskBlobStore.create( cfg.blobstore.location, From 8637c367fe2c0e7444ed67868c075ab9b3b65809 Mon Sep 17 00:00:00 2001 From: devin ivy Date: Tue, 31 Oct 2023 18:09:02 -0400 Subject: [PATCH 13/59] Respect updated service auth keys (#1765) * bust key cache when verifying service auth * unit tests for xrpc auth * fix * support option for verifying non-low-s signatures * fix verifyJwt tests --- .../crypto/signature-fixtures.json | 12 +- packages/bsky/src/auth.ts | 15 ++- packages/bsky/tests/auth.test.ts | 64 +++++++++++ packages/crypto/src/p256/operations.ts | 9 +- packages/crypto/src/secp256k1/operations.ts | 9 +- packages/crypto/src/types.ts | 5 + packages/crypto/src/verify.ts | 9 +- packages/crypto/tests/signatures.test.ts | 44 +++++++ packages/xrpc-server/package.json | 2 + packages/xrpc-server/src/auth.ts | 28 ++++- packages/xrpc-server/tests/auth.test.ts | 108 +++++++++++++++++- pnpm-lock.yaml | 21 ++-- 12 files changed, 296 insertions(+), 30 deletions(-) create mode 100644 packages/bsky/tests/auth.test.ts diff --git a/interop-test-files/crypto/signature-fixtures.json b/interop-test-files/crypto/signature-fixtures.json index 917c6d02455..7cdeb55ea75 100644 --- a/interop-test-files/crypto/signature-fixtures.json +++ b/interop-test-files/crypto/signature-fixtures.json @@ -7,7 +7,8 @@ "publicKeyDid": "did:key:zDnaembgSGUhZULN2Caob4HLJPaxBh92N7rtH21TErzqf8HQo", "publicKeyMultibase": "zxdM8dSstjrpZaRUwBmDvjGXweKuEMVN95A9oJBFjkWMh", "signatureBase64": "2vZNsG3UKvvO/CDlrdvyZRISOFylinBh0Jupc6KcWoJWExHptCfduPleDbG3rko3YZnn9Lw0IjpixVmexJDegg", - "validSignature": true + "validSignature": true, + "tags": [] }, { "comment": "valid K-256 key and signature, with low-S signature", @@ -17,7 +18,8 @@ "publicKeyDid": "did:key:zQ3shqwJEJyMBsBXCWyCBpUBMqxcon9oHB7mCvx4sSpMdLJwc", "publicKeyMultibase": "z25z9DTpsiYYJKGsWmSPJK2NFN8PcJtZig12K59UgW7q5t", "signatureBase64": "5WpdIuEUUfVUYaozsi8G0B3cWO09cgZbIIwg1t2YKdUn/FEznOndsz/qgiYb89zwxYCbB71f7yQK5Lr7NasfoA", - "validSignature": true + "validSignature": true, + "tags": [] }, { "comment": "P-256 key and signature, with non-low-S signature which is invalid in atproto", @@ -27,7 +29,8 @@ "publicKeyDid": "did:key:zDnaembgSGUhZULN2Caob4HLJPaxBh92N7rtH21TErzqf8HQo", "publicKeyMultibase": "zxdM8dSstjrpZaRUwBmDvjGXweKuEMVN95A9oJBFjkWMh", "signatureBase64": "2vZNsG3UKvvO/CDlrdvyZRISOFylinBh0Jupc6KcWoKp7O4VS9giSAah8k5IUbXIW00SuOrjfEqQ9HEkN9JGzw", - "validSignature": false + "validSignature": false, + "tags": ["high-s"] }, { "comment": "K-256 key and signature, with non-low-S signature which is invalid in atproto", @@ -37,6 +40,7 @@ "publicKeyDid": "did:key:zQ3shqwJEJyMBsBXCWyCBpUBMqxcon9oHB7mCvx4sSpMdLJwc", "publicKeyMultibase": "z25z9DTpsiYYJKGsWmSPJK2NFN8PcJtZig12K59UgW7q5t", "signatureBase64": "5WpdIuEUUfVUYaozsi8G0B3cWO09cgZbIIwg1t2YKdXYA67MYxYiTMAVfdnkDCMN9S5B3vHosRe07aORmoshoQ", - "validSignature": false + "validSignature": false, + "tags": ["high-s"] } ] diff --git a/packages/bsky/src/auth.ts b/packages/bsky/src/auth.ts index 290ef3c7a42..b19e6860e5c 100644 --- a/packages/bsky/src/auth.ts +++ b/packages/bsky/src/auth.ts @@ -14,10 +14,17 @@ export const authVerifier = if (!jwtStr) { throw new AuthRequiredError('missing jwt', 'MissingJwt') } - const payload = await verifyJwt(jwtStr, opts.aud, async (did: string) => { - const atprotoData = await idResolver.did.resolveAtprotoData(did) - return atprotoData.signingKey - }) + const payload = await verifyJwt( + jwtStr, + opts.aud, + async (did, forceRefresh) => { + const atprotoData = await idResolver.did.resolveAtprotoData( + did, + forceRefresh, + ) + return atprotoData.signingKey + }, + ) return { credentials: { did: payload.iss }, artifacts: { aud: opts.aud } } } diff --git a/packages/bsky/tests/auth.test.ts b/packages/bsky/tests/auth.test.ts new file mode 100644 index 00000000000..a3ac3d1d9f9 --- /dev/null +++ b/packages/bsky/tests/auth.test.ts @@ -0,0 +1,64 @@ +import AtpAgent from '@atproto/api' +import { SeedClient, TestNetwork } from '@atproto/dev-env' +import usersSeed from './seeds/users' +import { createServiceJwt } from '@atproto/xrpc-server' +import { Keypair, Secp256k1Keypair } from '@atproto/crypto' + +describe('auth', () => { + let network: TestNetwork + let agent: AtpAgent + let sc: SeedClient + + beforeAll(async () => { + network = await TestNetwork.create({ + dbPostgresSchema: 'bsky_auth', + }) + agent = network.bsky.getClient() + sc = network.getSeedClient() + await usersSeed(sc) + await network.processAll() + }) + + afterAll(async () => { + await network.close() + }) + + it('handles signing key change for service auth.', async () => { + const issuer = sc.dids.alice + const attemptWithKey = async (keypair: Keypair) => { + const jwt = await createServiceJwt({ + iss: issuer, + aud: network.bsky.ctx.cfg.serverDid, + keypair, + }) + return agent.api.app.bsky.actor.getProfile( + { actor: sc.dids.carol }, + { headers: { authorization: `Bearer ${jwt}` } }, + ) + } + const origSigningKey = network.pds.ctx.repoSigningKey + const newSigningKey = await Secp256k1Keypair.create({ exportable: true }) + // confirm original signing key works + await expect(attemptWithKey(origSigningKey)).resolves.toBeDefined() + // confirm next signing key doesn't work yet + await expect(attemptWithKey(newSigningKey)).rejects.toThrow( + 'jwt signature does not match jwt issuer', + ) + // update to new signing key + await network.plc + .getClient() + .updateAtprotoKey( + issuer, + network.pds.ctx.plcRotationKey, + newSigningKey.did(), + ) + // old signing key still works due to did doc cache + await expect(attemptWithKey(origSigningKey)).resolves.toBeDefined() + // new signing key works + await expect(attemptWithKey(newSigningKey)).resolves.toBeDefined() + // old signing key no longer works after cache is updated + await expect(attemptWithKey(origSigningKey)).rejects.toThrow( + 'jwt signature does not match jwt issuer', + ) + }) +}) diff --git a/packages/crypto/src/p256/operations.ts b/packages/crypto/src/p256/operations.ts index f5292f4bd80..6f81b0371a9 100644 --- a/packages/crypto/src/p256/operations.ts +++ b/packages/crypto/src/p256/operations.ts @@ -2,24 +2,29 @@ import { p256 } from '@noble/curves/p256' import { sha256 } from '@noble/hashes/sha256' import { P256_JWT_ALG } from '../const' import { parseDidKey } from '../did' +import { VerifyOptions } from '../types' export const verifyDidSig = async ( did: string, data: Uint8Array, sig: Uint8Array, + opts?: VerifyOptions, ): Promise => { const { jwtAlg, keyBytes } = parseDidKey(did) if (jwtAlg !== P256_JWT_ALG) { throw new Error(`Not a P-256 did:key: ${did}`) } - return verifySig(keyBytes, data, sig) + return verifySig(keyBytes, data, sig, opts) } export const verifySig = async ( publicKey: Uint8Array, data: Uint8Array, sig: Uint8Array, + opts?: VerifyOptions, ): Promise => { const msgHash = await sha256(data) - return p256.verify(sig, msgHash, publicKey, { lowS: true }) + return p256.verify(sig, msgHash, publicKey, { + lowS: opts?.lowS ?? true, + }) } diff --git a/packages/crypto/src/secp256k1/operations.ts b/packages/crypto/src/secp256k1/operations.ts index 5d31a812506..f470c2da54c 100644 --- a/packages/crypto/src/secp256k1/operations.ts +++ b/packages/crypto/src/secp256k1/operations.ts @@ -2,24 +2,29 @@ import { secp256k1 as k256 } from '@noble/curves/secp256k1' import { sha256 } from '@noble/hashes/sha256' import { SECP256K1_JWT_ALG } from '../const' import { parseDidKey } from '../did' +import { VerifyOptions } from '../types' export const verifyDidSig = async ( did: string, data: Uint8Array, sig: Uint8Array, + opts?: VerifyOptions, ): Promise => { const { jwtAlg, keyBytes } = parseDidKey(did) if (jwtAlg !== SECP256K1_JWT_ALG) { throw new Error(`Not a secp256k1 did:key: ${did}`) } - return verifySig(keyBytes, data, sig) + return verifySig(keyBytes, data, sig, opts) } export const verifySig = async ( publicKey: Uint8Array, data: Uint8Array, sig: Uint8Array, + opts?: VerifyOptions, ): Promise => { const msgHash = await sha256(data) - return k256.verify(sig, msgHash, publicKey, { lowS: true }) + return k256.verify(sig, msgHash, publicKey, { + lowS: opts?.lowS ?? true, + }) } diff --git a/packages/crypto/src/types.ts b/packages/crypto/src/types.ts index e8cbdc57b62..a1089134f0a 100644 --- a/packages/crypto/src/types.ts +++ b/packages/crypto/src/types.ts @@ -16,5 +16,10 @@ export type DidKeyPlugin = { did: string, msg: Uint8Array, data: Uint8Array, + opts?: VerifyOptions, ) => Promise } + +export type VerifyOptions = { + lowS?: boolean +} diff --git a/packages/crypto/src/verify.ts b/packages/crypto/src/verify.ts index 43b2670c7cd..50ba87aba2e 100644 --- a/packages/crypto/src/verify.ts +++ b/packages/crypto/src/verify.ts @@ -1,26 +1,29 @@ import * as uint8arrays from 'uint8arrays' import { parseDidKey } from './did' import plugins from './plugins' +import { VerifyOptions } from './types' export const verifySignature = ( didKey: string, data: Uint8Array, sig: Uint8Array, + opts?: VerifyOptions, ): Promise => { const parsed = parseDidKey(didKey) const plugin = plugins.find((p) => p.jwtAlg === parsed.jwtAlg) if (!plugin) { - throw new Error(`Unsupported signature alg: :${parsed.jwtAlg}`) + throw new Error(`Unsupported signature alg: ${parsed.jwtAlg}`) } - return plugin.verifySignature(didKey, data, sig) + return plugin.verifySignature(didKey, data, sig, opts) } export const verifySignatureUtf8 = async ( didKey: string, data: string, sig: string, + opts?: VerifyOptions, ): Promise => { const dataBytes = uint8arrays.fromString(data, 'utf8') const sigBytes = uint8arrays.fromString(sig, 'base64url') - return verifySignature(didKey, dataBytes, sigBytes) + return verifySignature(didKey, dataBytes, sigBytes, opts) } diff --git a/packages/crypto/tests/signatures.test.ts b/packages/crypto/tests/signatures.test.ts index cebc8126b3a..83d2b6b72f0 100644 --- a/packages/crypto/tests/signatures.test.ts +++ b/packages/crypto/tests/signatures.test.ts @@ -57,6 +57,45 @@ describe('signatures', () => { } } }) + + it('verifies high-s signatures with explicit option', async () => { + const highSVectors = vectors.filter((vec) => vec.tags.includes('high-s')) + expect(highSVectors.length).toBeGreaterThanOrEqual(2) + for (const vector of highSVectors) { + const messageBytes = uint8arrays.fromString( + vector.messageBase64, + 'base64', + ) + const signatureBytes = uint8arrays.fromString( + vector.signatureBase64, + 'base64', + ) + const keyBytes = multibaseToBytes(vector.publicKeyMultibase) + const didKey = parseDidKey(vector.publicKeyDid) + expect(uint8arrays.equals(keyBytes, didKey.keyBytes)) + if (vector.algorithm === P256_JWT_ALG) { + const verified = await p256.verifySig( + keyBytes, + messageBytes, + signatureBytes, + { lowS: false }, + ) + expect(verified).toEqual(true) + expect(vector.validSignature).toEqual(false) // otherwise would fail per low-s requirement + } else if (vector.algorithm === SECP256K1_JWT_ALG) { + const verified = await secp.verifySig( + keyBytes, + messageBytes, + signatureBytes, + { lowS: false }, + ) + expect(verified).toEqual(true) + expect(vector.validSignature).toEqual(false) // otherwise would fail per low-s requirement + } else { + throw new Error('Unsupported test vector') + } + } + }) }) // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -79,6 +118,7 @@ async function generateTestVectors(): Promise { 'base64', ), validSignature: true, + tags: [], }, { messageBase64, @@ -93,6 +133,7 @@ async function generateTestVectors(): Promise { 'base64', ), validSignature: true, + tags: [], }, // these vectors test to ensure we don't allow high-s signatures { @@ -109,6 +150,7 @@ async function generateTestVectors(): Promise { P256_JWT_ALG, ), validSignature: false, + tags: ['high-s'], }, { messageBase64, @@ -124,6 +166,7 @@ async function generateTestVectors(): Promise { SECP256K1_JWT_ALG, ), validSignature: false, + tags: ['high-s'], }, ] } @@ -159,4 +202,5 @@ type TestVector = { messageBase64: string signatureBase64: string validSignature: boolean + tags: string[] } diff --git a/packages/xrpc-server/package.json b/packages/xrpc-server/package.json index d32319f301b..21589c26f6b 100644 --- a/packages/xrpc-server/package.json +++ b/packages/xrpc-server/package.json @@ -45,6 +45,8 @@ "@types/http-errors": "^2.0.1", "@types/ws": "^8.5.4", "get-port": "^6.1.2", + "jose": "^4.15.4", + "key-encoder": "^2.0.3", "multiformats": "^9.9.0" } } diff --git a/packages/xrpc-server/src/auth.ts b/packages/xrpc-server/src/auth.ts index 9283b13815b..0b0cbe03127 100644 --- a/packages/xrpc-server/src/auth.ts +++ b/packages/xrpc-server/src/auth.ts @@ -48,7 +48,7 @@ const jsonToB64Url = (json: Record): string => { export const verifyJwt = async ( jwtStr: string, ownDid: string | null, // null indicates to skip the audience check - getSigningKey: (did: string) => Promise, + getSigningKey: (did: string, forceRefresh: boolean) => Promise, ): Promise => { const parts = jwtStr.split('.') if (parts.length !== 3) { @@ -69,18 +69,40 @@ export const verifyJwt = async ( const msgBytes = ui8.fromString(parts.slice(0, 2).join('.'), 'utf8') const sigBytes = ui8.fromString(sig, 'base64url') + const verifySignatureWithKey = (key: string) => { + return crypto.verifySignature(key, msgBytes, sigBytes, { + lowS: false, + }) + } - const signingKey = await getSigningKey(payload.iss) + const signingKey = await getSigningKey(payload.iss, false) let validSig: boolean try { - validSig = await crypto.verifySignature(signingKey, msgBytes, sigBytes) + validSig = await verifySignatureWithKey(signingKey) } catch (err) { throw new AuthRequiredError( 'could not verify jwt signature', 'BadJwtSignature', ) } + + if (!validSig) { + // get fresh signing key in case it failed due to a recent rotation + const freshSigningKey = await getSigningKey(payload.iss, true) + try { + validSig = + freshSigningKey !== signingKey + ? await verifySignatureWithKey(freshSigningKey) + : false + } catch (err) { + throw new AuthRequiredError( + 'could not verify jwt signature', + 'BadJwtSignature', + ) + } + } + if (!validSig) { throw new AuthRequiredError( 'jwt signature does not match jwt issuer', diff --git a/packages/xrpc-server/tests/auth.test.ts b/packages/xrpc-server/tests/auth.test.ts index d36c05b6c3a..53f3a6c6d24 100644 --- a/packages/xrpc-server/tests/auth.test.ts +++ b/packages/xrpc-server/tests/auth.test.ts @@ -1,5 +1,11 @@ -import * as http from 'http' +import * as http from 'node:http' +import { KeyObject, createPrivateKey } from 'node:crypto' import getPort from 'get-port' +import * as jose from 'jose' +import KeyEncoder from 'key-encoder' +import * as ui8 from 'uint8arrays' +import { MINUTE } from '@atproto/common' +import { Secp256k1Keypair } from '@atproto/crypto' import xrpc, { ServiceClient, XRPCError } from '@atproto/xrpc' import * as xrpcServer from '../src' import { @@ -131,4 +137,104 @@ describe('Auth', () => { original: 'YWRtaW46cGFzc3dvcmQ=', }) }) + + describe('verifyJwt()', () => { + it('fails on expired jwt.', async () => { + const keypair = await Secp256k1Keypair.create() + const jwt = await xrpcServer.createServiceJwt({ + aud: 'did:example:aud', + iss: 'did:example:iss', + keypair, + exp: Math.floor((Date.now() - MINUTE) / 1000), + }) + const tryVerify = xrpcServer.verifyJwt( + jwt, + 'did:example:aud', + async () => { + return keypair.did() + }, + ) + await expect(tryVerify).rejects.toThrow('jwt expired') + }) + + it('fails on bad audience.', async () => { + const keypair = await Secp256k1Keypair.create() + const jwt = await xrpcServer.createServiceJwt({ + aud: 'did:example:aud1', + iss: 'did:example:iss', + keypair, + }) + const tryVerify = xrpcServer.verifyJwt( + jwt, + 'did:example:aud2', + async () => { + return keypair.did() + }, + ) + await expect(tryVerify).rejects.toThrow( + 'jwt audience does not match service did', + ) + }) + + it('refreshes key on verification failure.', async () => { + const keypair1 = await Secp256k1Keypair.create() + const keypair2 = await Secp256k1Keypair.create() + const jwt = await xrpcServer.createServiceJwt({ + aud: 'did:example:aud', + iss: 'did:example:iss', + keypair: keypair2, + }) + let usedKeypair1 = false + let usedKeypair2 = false + const tryVerify = xrpcServer.verifyJwt( + jwt, + 'did:example:aud', + async (_did, forceRefresh) => { + if (forceRefresh) { + usedKeypair2 = true + return keypair2.did() + } else { + usedKeypair1 = true + return keypair1.did() + } + }, + ) + await expect(tryVerify).resolves.toMatchObject({ + aud: 'did:example:aud', + iss: 'did:example:iss', + }) + expect(usedKeypair1).toBe(true) + expect(usedKeypair2).toBe(true) + }) + + it('interoperates with jwts signed by other libraries.', async () => { + const keypair = await Secp256k1Keypair.create({ exportable: true }) + const signingKey = await createPrivateKeyObject(keypair) + const payload = { + aud: 'did:example:aud', + iss: 'did:example:iss', + exp: Math.floor((Date.now() + MINUTE) / 1000), + } + const jwt = await new jose.SignJWT(payload) + .setProtectedHeader({ typ: 'JWT', alg: keypair.jwtAlg }) + .sign(signingKey) + const tryVerify = xrpcServer.verifyJwt( + jwt, + 'did:example:aud', + async () => { + return keypair.did() + }, + ) + await expect(tryVerify).resolves.toEqual(payload) + }) + }) }) + +const createPrivateKeyObject = async ( + privateKey: Secp256k1Keypair, +): Promise => { + const raw = await privateKey.export() + const encoder = new KeyEncoder('secp256k1') + const key = encoder.encodePrivate(ui8.toString(raw, 'hex'), 'raw', 'pem') + return createPrivateKey({ format: 'pem', key }) +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1500b3ece5d..aac3ef2bcba 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -734,6 +734,12 @@ importers: get-port: specifier: ^6.1.2 version: 6.1.2 + jose: + specifier: ^4.15.4 + version: 4.15.4 + key-encoder: + specifier: ^2.0.3 + version: 2.0.3 multiformats: specifier: ^9.9.0 version: 9.9.0 @@ -5294,7 +5300,6 @@ packages: resolution: {integrity: sha512-qNrYbZqMx0uJAfKnKclPh+dTwK33KfLHYqtyODwd5HnXOjnkhc4qgn3BrK6RWyGZm5+sIFE7Q7Vz6QQtJB7w7g==} dependencies: '@types/node': 18.17.8 - dev: false /@types/body-parser@1.19.2: resolution: {integrity: sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==} @@ -5321,7 +5326,6 @@ packages: resolution: {integrity: sha512-z4OBcDAU0GVwDTuwJzQCiL6188QvZMkvoERgcVjq0/mPM8jCfdwZ3x5zQEVoL9WCAru3aG5wl3Z5Ww5wBWn7ZQ==} dependencies: '@types/bn.js': 5.1.1 - dev: false /@types/express-serve-static-core@4.17.36: resolution: {integrity: sha512-zbivROJ0ZqLAtMzgzIUC4oNqDG9iF0lSsAqpOD9kbs5xcIM3dTiyuHvBc7R8MtWBp3AAWGaovJa+wzWPjLYW7Q==} @@ -5833,7 +5837,6 @@ packages: inherits: 2.0.4 minimalistic-assert: 1.0.1 safer-buffer: 2.1.2 - dev: false /asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} @@ -6031,7 +6034,6 @@ packages: /bn.js@4.12.0: resolution: {integrity: sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==} - dev: false /body-parser@1.20.1: resolution: {integrity: sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==} @@ -6085,7 +6087,6 @@ packages: /brorand@1.1.0: resolution: {integrity: sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==} - dev: false /browserslist@4.21.10: resolution: {integrity: sha512-bipEBdZfVH5/pwrvqc+Ub0kUPVfGUhlKxbvfD+z1BDnPEO/X98ruXGA1WP5ASpAFKan7Qr6j736IacbZQuAlKQ==} @@ -6739,7 +6740,6 @@ packages: inherits: 2.0.4 minimalistic-assert: 1.0.1 minimalistic-crypto-utils: 1.0.1 - dev: false /emittery@0.10.2: resolution: {integrity: sha512-aITqOwnLanpHLNXZJENbOgjUBeHocD+xsSJmNrjovKBW5HbSpW3d1pEls7GFQPUWXiwG9+0P4GtHfEqC/4M0Iw==} @@ -7849,7 +7849,6 @@ packages: dependencies: inherits: 2.0.4 minimalistic-assert: 1.0.1 - dev: false /he@1.2.0: resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==} @@ -7869,7 +7868,6 @@ packages: hash.js: 1.1.7 minimalistic-assert: 1.0.1 minimalistic-crypto-utils: 1.0.1 - dev: false /hosted-git-info@2.8.9: resolution: {integrity: sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==} @@ -8735,6 +8733,10 @@ packages: - ts-node dev: true + /jose@4.15.4: + resolution: {integrity: sha512-W+oqK4H+r5sITxfxpSU+MMdr/YSWGvgZMQDIsNoBDGGy4i7GBPTtvFKibQzW06n3U3TqHjhvBJsirShsEJ6eeQ==} + dev: true + /joycon@3.1.1: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} @@ -8856,7 +8858,6 @@ packages: asn1.js: 5.4.1 bn.js: 4.12.0 elliptic: 6.5.4 - dev: false /kind-of@6.0.3: resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} @@ -9150,11 +9151,9 @@ packages: /minimalistic-assert@1.0.1: resolution: {integrity: sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==} - dev: false /minimalistic-crypto-utils@1.0.1: resolution: {integrity: sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==} - dev: false /minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} From cae59094c71233a7afd1cb763292ab850ee11aae Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 1 Nov 2023 15:42:37 +0000 Subject: [PATCH 14/59] Don't validate known lexicons at runtime (#1790) * Make lexicon validation DEV-only * Apply code review suggestions --- packages/lex-cli/src/util.ts | 5 ++- packages/lexicon/src/lexicons.ts | 31 ++++--------------- packages/lexicon/src/types.ts | 13 ++------ packages/lexicon/tests/_scaffolds/lexicons.ts | 6 +++- packages/lexicon/tests/general.test.ts | 12 +++---- packages/xrpc-server/src/server.ts | 9 +++--- packages/xrpc/src/client.ts | 6 ++-- 7 files changed, 30 insertions(+), 52 deletions(-) diff --git a/packages/lex-cli/src/util.ts b/packages/lex-cli/src/util.ts index 6c399ab9bd8..b3fc5668a52 100644 --- a/packages/lex-cli/src/util.ts +++ b/packages/lex-cli/src/util.ts @@ -1,6 +1,6 @@ import fs from 'fs' import { join } from 'path' -import { lexiconDoc, LexiconDoc } from '@atproto/lexicon' +import { parseLexiconDoc, LexiconDoc } from '@atproto/lexicon' import { ZodError, ZodFormattedError } from 'zod' import chalk from 'chalk' import { GeneratedAPI, FileDiff } from './types' @@ -41,8 +41,7 @@ export function readLexicon(path: string): LexiconDoc { typeof (obj as LexiconDoc).lexicon === 'number' ) { try { - lexiconDoc.parse(obj) - return obj as LexiconDoc + return parseLexiconDoc(obj) } catch (e) { console.error(`Invalid lexicon`, path) if (e instanceof ZodError) { diff --git a/packages/lexicon/src/lexicons.ts b/packages/lexicon/src/lexicons.ts index 72d4ef0d5b2..29998140816 100644 --- a/packages/lexicon/src/lexicons.ts +++ b/packages/lexicon/src/lexicons.ts @@ -1,12 +1,9 @@ -import { ZodError } from 'zod' import { LexiconDoc, - lexiconDoc, LexRecord, LexXrpcProcedure, LexXrpcQuery, LexUserType, - LexiconDocMalformedError, LexiconDefNotFoundError, InvalidLexiconError, ValidationResult, @@ -32,7 +29,7 @@ export class Lexicons { docs: Map = new Map() defs: Map = new Map() - constructor(docs?: unknown[]) { + constructor(docs?: LexiconDoc[]) { if (docs?.length) { for (const doc of docs) { this.add(doc) @@ -43,24 +40,8 @@ export class Lexicons { /** * Add a lexicon doc. */ - add(doc: unknown): void { - try { - lexiconDoc.parse(doc) - } catch (e) { - if (e instanceof ZodError) { - throw new LexiconDocMalformedError( - `Failed to parse schema definition ${ - (doc as Record).id - }`, - doc, - e.issues, - ) - } else { - throw e - } - } - const validatedDoc = doc as LexiconDoc - const uri = toLexUri(validatedDoc.id) + add(doc: LexiconDoc): void { + const uri = toLexUri(doc.id) if (this.docs.has(uri)) { throw new Error(`${uri} has already been registered`) } @@ -68,10 +49,10 @@ export class Lexicons { // WARNING // mutates the object // -prf - resolveRefUris(validatedDoc, uri) + resolveRefUris(doc, uri) - this.docs.set(uri, validatedDoc) - for (const [defUri, def] of iterDefs(validatedDoc)) { + this.docs.set(uri, doc) + for (const [defUri, def] of iterDefs(doc)) { this.defs.set(defUri, def) } } diff --git a/packages/lexicon/src/types.ts b/packages/lexicon/src/types.ts index 906cd353328..5d7ed1f4a2a 100644 --- a/packages/lexicon/src/types.ts +++ b/packages/lexicon/src/types.ts @@ -415,16 +415,9 @@ export function isDiscriminatedObject( return discriminatedObject.safeParse(value).success } -export class LexiconDocMalformedError extends Error { - constructor( - message: string, - public schemaDef: unknown, - public issues?: z.ZodIssue[], - ) { - super(message) - this.schemaDef = schemaDef - this.issues = issues - } +export function parseLexiconDoc(v: unknown): LexiconDoc { + lexiconDoc.parse(v) + return v as LexiconDoc } export type ValidationResult = diff --git a/packages/lexicon/tests/_scaffolds/lexicons.ts b/packages/lexicon/tests/_scaffolds/lexicons.ts index 4057c1efd50..d0cf414ccef 100644 --- a/packages/lexicon/tests/_scaffolds/lexicons.ts +++ b/packages/lexicon/tests/_scaffolds/lexicons.ts @@ -1,4 +1,6 @@ -export default [ +import { LexiconDoc } from '../../src/index' + +const lexicons: LexiconDoc[] = [ { lexicon: 1, id: 'com.example.kitchenSink', @@ -521,3 +523,5 @@ export default [ }, }, ] + +export default lexicons diff --git a/packages/lexicon/tests/general.test.ts b/packages/lexicon/tests/general.test.ts index 5217ad49c52..685c99f40e0 100644 --- a/packages/lexicon/tests/general.test.ts +++ b/packages/lexicon/tests/general.test.ts @@ -1,5 +1,5 @@ import { CID } from 'multiformats/cid' -import { lexiconDoc, Lexicons } from '../src/index' +import { LexiconDoc, Lexicons, parseLexiconDoc } from '../src/index' import LexiconDocs from './_scaffolds/lexicons' describe('Lexicons collection', () => { @@ -97,7 +97,7 @@ describe('General validation', () => { }, } expect(() => { - lexiconDoc.parse(schema) + parseLexiconDoc(schema) }).toThrow('Required field \\"foo\\" not defined') }) it('fails when unknown fields are present', () => { @@ -113,11 +113,11 @@ describe('General validation', () => { } expect(() => { - lexiconDoc.parse(schema) + parseLexiconDoc(schema) }).toThrow("Unrecognized key(s) in object: 'foo'") }) it('fails lexicon parsing when uri is invalid', () => { - const schema = { + const schema: LexiconDoc = { lexicon: 1, id: 'com.example.invalidUri', defs: { @@ -135,7 +135,7 @@ describe('General validation', () => { }).toThrow('Uri can only have one hash segment') }) it('fails validation when ref uri has multiple hash segments', () => { - const schema = { + const schema: LexiconDoc = { lexicon: 1, id: 'com.example.invalidUri', defs: { @@ -168,7 +168,7 @@ describe('General validation', () => { }).toThrow('Uri can only have one hash segment') }) it('union handles both implicit and explicit #main', () => { - const schemas = [ + const schemas: LexiconDoc[] = [ { lexicon: 1, id: 'com.example.implicitMain', diff --git a/packages/xrpc-server/src/server.ts b/packages/xrpc-server/src/server.ts index 4e0a84ce4b7..1a258097d66 100644 --- a/packages/xrpc-server/src/server.ts +++ b/packages/xrpc-server/src/server.ts @@ -5,6 +5,7 @@ import express, { RequestHandler, } from 'express' import { + LexiconDoc, Lexicons, lexToJson, LexXrpcProcedure, @@ -45,7 +46,7 @@ import { import log from './logger' import { consumeMany } from './rate-limiter' -export function createServer(lexicons?: unknown[], options?: Options) { +export function createServer(lexicons?: LexiconDoc[], options?: Options) { return new Server(lexicons, options) } @@ -60,7 +61,7 @@ export class Server { sharedRateLimiters: Record routeRateLimiterFns: Record - constructor(lexicons?: unknown[], opts?: Options) { + constructor(lexicons?: LexiconDoc[], opts?: Options) { if (lexicons) { this.addLexicons(lexicons) } @@ -140,11 +141,11 @@ export class Server { // schemas // = - addLexicon(doc: unknown) { + addLexicon(doc: LexiconDoc) { this.lex.add(doc) } - addLexicons(docs: unknown[]) { + addLexicons(docs: LexiconDoc[]) { for (const doc of docs) { this.addLexicon(doc) } diff --git a/packages/xrpc/src/client.ts b/packages/xrpc/src/client.ts index fbb5d33662c..6603345608a 100644 --- a/packages/xrpc/src/client.ts +++ b/packages/xrpc/src/client.ts @@ -1,4 +1,4 @@ -import { Lexicons, ValidationError } from '@atproto/lexicon' +import { LexiconDoc, Lexicons, ValidationError } from '@atproto/lexicon' import { getMethodSchemaHTTPMethod, constructMethodCallUri, @@ -46,11 +46,11 @@ export class Client { // schemas // = - addLexicon(doc: unknown) { + addLexicon(doc: LexiconDoc) { this.lex.add(doc) } - addLexicons(docs: unknown[]) { + addLexicons(docs: LexiconDoc[]) { for (const doc of docs) { this.addLexicon(doc) } From a161f815de83b9c2969c8df9d703654bb6e6cc17 Mon Sep 17 00:00:00 2001 From: devin ivy Date: Wed, 1 Nov 2023 11:54:00 -0400 Subject: [PATCH 15/59] Make plc op unknown object rather than bytes (#1792) make plc op unknown object rather than bytes --- lexicons/com/atproto/server/createAccount.json | 2 +- packages/api/src/client/lexicons.ts | 2 +- .../api/src/client/types/com/atproto/server/createAccount.ts | 2 +- packages/bsky/src/lexicon/lexicons.ts | 2 +- .../src/lexicon/types/com/atproto/server/createAccount.ts | 2 +- packages/pds/src/lexicon/lexicons.ts | 4 +++- .../pds/src/lexicon/types/com/atproto/server/createAccount.ts | 2 +- 7 files changed, 9 insertions(+), 7 deletions(-) diff --git a/lexicons/com/atproto/server/createAccount.json b/lexicons/com/atproto/server/createAccount.json index 4db1f31e040..d9549624741 100644 --- a/lexicons/com/atproto/server/createAccount.json +++ b/lexicons/com/atproto/server/createAccount.json @@ -17,7 +17,7 @@ "inviteCode": { "type": "string" }, "password": { "type": "string" }, "recoveryKey": { "type": "string" }, - "plcOp": { "type": "bytes" } + "plcOp": { "type": "unknown" } } } }, diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index df696e5d06b..4030fea961a 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -2512,7 +2512,7 @@ export const schemaDict = { type: 'string', }, plcOp: { - type: 'bytes', + type: 'unknown', }, }, }, diff --git a/packages/api/src/client/types/com/atproto/server/createAccount.ts b/packages/api/src/client/types/com/atproto/server/createAccount.ts index 4281128cae0..b5fb32fe90b 100644 --- a/packages/api/src/client/types/com/atproto/server/createAccount.ts +++ b/packages/api/src/client/types/com/atproto/server/createAccount.ts @@ -16,7 +16,7 @@ export interface InputSchema { inviteCode?: string password: string recoveryKey?: string - plcOp?: Uint8Array + plcOp?: {} [k: string]: unknown } diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index df696e5d06b..4030fea961a 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -2512,7 +2512,7 @@ export const schemaDict = { type: 'string', }, plcOp: { - type: 'bytes', + type: 'unknown', }, }, }, diff --git a/packages/bsky/src/lexicon/types/com/atproto/server/createAccount.ts b/packages/bsky/src/lexicon/types/com/atproto/server/createAccount.ts index bd138919101..f82fe9ed82a 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/server/createAccount.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/server/createAccount.ts @@ -17,7 +17,7 @@ export interface InputSchema { inviteCode?: string password: string recoveryKey?: string - plcOp?: Uint8Array + plcOp?: {} [k: string]: unknown } diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index 38af4475fb8..4030fea961a 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -1826,6 +1826,8 @@ export const schemaDict = { }, reason: { type: 'string', + maxGraphemes: 2000, + maxLength: 20000, }, subject: { type: 'union', @@ -2510,7 +2512,7 @@ export const schemaDict = { type: 'string', }, plcOp: { - type: 'bytes', + type: 'unknown', }, }, }, diff --git a/packages/pds/src/lexicon/types/com/atproto/server/createAccount.ts b/packages/pds/src/lexicon/types/com/atproto/server/createAccount.ts index bd138919101..f82fe9ed82a 100644 --- a/packages/pds/src/lexicon/types/com/atproto/server/createAccount.ts +++ b/packages/pds/src/lexicon/types/com/atproto/server/createAccount.ts @@ -17,7 +17,7 @@ export interface InputSchema { inviteCode?: string password: string recoveryKey?: string - plcOp?: Uint8Array + plcOp?: {} [k: string]: unknown } From cf848e87ab5166a73a4803d0f46e4d529cde34fd Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Wed, 1 Nov 2023 14:56:57 -0500 Subject: [PATCH 16/59] Tweak schemas for entryway createAccount (#1797) * tweak scheams * require email & password --- lexicons/com/atproto/server/createAccount.json | 2 +- lexicons/com/atproto/server/reserveSigningKey.json | 12 ++++++++++++ packages/api/src/client/lexicons.ts | 14 +++++++++++++- .../types/com/atproto/server/createAccount.ts | 4 ++-- .../types/com/atproto/server/reserveSigningKey.ts | 7 ++++++- packages/bsky/src/lexicon/lexicons.ts | 14 +++++++++++++- .../types/com/atproto/server/createAccount.ts | 4 ++-- .../types/com/atproto/server/reserveSigningKey.ts | 11 +++++++++-- .../src/api/com/atproto/server/createAccount.ts | 6 +++++- packages/pds/src/lexicon/lexicons.ts | 14 +++++++++++++- .../types/com/atproto/server/createAccount.ts | 4 ++-- .../types/com/atproto/server/reserveSigningKey.ts | 11 +++++++++-- 12 files changed, 87 insertions(+), 16 deletions(-) diff --git a/lexicons/com/atproto/server/createAccount.json b/lexicons/com/atproto/server/createAccount.json index d9549624741..8d927163951 100644 --- a/lexicons/com/atproto/server/createAccount.json +++ b/lexicons/com/atproto/server/createAccount.json @@ -9,7 +9,7 @@ "encoding": "application/json", "schema": { "type": "object", - "required": ["handle", "email", "password"], + "required": ["handle"], "properties": { "email": { "type": "string" }, "handle": { "type": "string", "format": "handle" }, diff --git a/lexicons/com/atproto/server/reserveSigningKey.json b/lexicons/com/atproto/server/reserveSigningKey.json index 27fb0597b0a..3a67ad0a3c8 100644 --- a/lexicons/com/atproto/server/reserveSigningKey.json +++ b/lexicons/com/atproto/server/reserveSigningKey.json @@ -5,6 +5,18 @@ "main": { "type": "procedure", "description": "Reserve a repo signing key for account creation.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "properties": { + "did": { + "type": "string", + "description": "The did to reserve a new did:key for" + } + } + } + }, "output": { "encoding": "application/json", "schema": { diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index 4030fea961a..d9de1551bcc 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -2489,7 +2489,7 @@ export const schemaDict = { encoding: 'application/json', schema: { type: 'object', - required: ['handle', 'email', 'password'], + required: ['handle'], properties: { email: { type: 'string', @@ -3170,6 +3170,18 @@ export const schemaDict = { main: { type: 'procedure', description: 'Reserve a repo signing key for account creation.', + input: { + encoding: 'application/json', + schema: { + type: 'object', + properties: { + did: { + type: 'string', + description: 'The did to reserve a new did:key for', + }, + }, + }, + }, output: { encoding: 'application/json', schema: { diff --git a/packages/api/src/client/types/com/atproto/server/createAccount.ts b/packages/api/src/client/types/com/atproto/server/createAccount.ts index b5fb32fe90b..7631727ef19 100644 --- a/packages/api/src/client/types/com/atproto/server/createAccount.ts +++ b/packages/api/src/client/types/com/atproto/server/createAccount.ts @@ -10,11 +10,11 @@ import { CID } from 'multiformats/cid' export interface QueryParams {} export interface InputSchema { - email: string + email?: string handle: string did?: string inviteCode?: string - password: string + password?: string recoveryKey?: string plcOp?: {} [k: string]: unknown diff --git a/packages/api/src/client/types/com/atproto/server/reserveSigningKey.ts b/packages/api/src/client/types/com/atproto/server/reserveSigningKey.ts index e6f4f7a618a..f5e515ff5cf 100644 --- a/packages/api/src/client/types/com/atproto/server/reserveSigningKey.ts +++ b/packages/api/src/client/types/com/atproto/server/reserveSigningKey.ts @@ -9,7 +9,11 @@ import { CID } from 'multiformats/cid' export interface QueryParams {} -export type InputSchema = undefined +export interface InputSchema { + /** The did to reserve a new did:key for */ + did?: string + [k: string]: unknown +} export interface OutputSchema { /** Public signing key in the form of a did:key. */ @@ -20,6 +24,7 @@ export interface OutputSchema { export interface CallOptions { headers?: Headers qp?: QueryParams + encoding: 'application/json' } export interface Response { diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index 4030fea961a..d9de1551bcc 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -2489,7 +2489,7 @@ export const schemaDict = { encoding: 'application/json', schema: { type: 'object', - required: ['handle', 'email', 'password'], + required: ['handle'], properties: { email: { type: 'string', @@ -3170,6 +3170,18 @@ export const schemaDict = { main: { type: 'procedure', description: 'Reserve a repo signing key for account creation.', + input: { + encoding: 'application/json', + schema: { + type: 'object', + properties: { + did: { + type: 'string', + description: 'The did to reserve a new did:key for', + }, + }, + }, + }, output: { encoding: 'application/json', schema: { diff --git a/packages/bsky/src/lexicon/types/com/atproto/server/createAccount.ts b/packages/bsky/src/lexicon/types/com/atproto/server/createAccount.ts index f82fe9ed82a..109d34cf202 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/server/createAccount.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/server/createAccount.ts @@ -11,11 +11,11 @@ import { HandlerAuth } from '@atproto/xrpc-server' export interface QueryParams {} export interface InputSchema { - email: string + email?: string handle: string did?: string inviteCode?: string - password: string + password?: string recoveryKey?: string plcOp?: {} [k: string]: unknown diff --git a/packages/bsky/src/lexicon/types/com/atproto/server/reserveSigningKey.ts b/packages/bsky/src/lexicon/types/com/atproto/server/reserveSigningKey.ts index 495b87dc03c..ad5a5a8758c 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/server/reserveSigningKey.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/server/reserveSigningKey.ts @@ -10,7 +10,11 @@ import { HandlerAuth } from '@atproto/xrpc-server' export interface QueryParams {} -export type InputSchema = undefined +export interface InputSchema { + /** The did to reserve a new did:key for */ + did?: string + [k: string]: unknown +} export interface OutputSchema { /** Public signing key in the form of a did:key. */ @@ -18,7 +22,10 @@ export interface OutputSchema { [k: string]: unknown } -export type HandlerInput = undefined +export interface HandlerInput { + encoding: 'application/json' + body: InputSchema +} export interface HandlerSuccess { encoding: 'application/json' diff --git a/packages/pds/src/api/com/atproto/server/createAccount.ts b/packages/pds/src/api/com/atproto/server/createAccount.ts index f1343b3c4e2..e747eb4e9cc 100644 --- a/packages/pds/src/api/com/atproto/server/createAccount.ts +++ b/packages/pds/src/api/com/atproto/server/createAccount.ts @@ -21,7 +21,11 @@ export default function (server: Server, ctx: AppContext) { }, handler: async ({ input, req }) => { const { email, password, inviteCode } = input.body - if (input.body.plcOp) { + if (!email) { + throw new InvalidRequestError('Missing input: "email"') + } else if (!password) { + throw new InvalidRequestError('Missing input: "password"') + } else if (input.body.plcOp) { throw new InvalidRequestError('Unsupported input: "plcOp"') } diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index 4030fea961a..d9de1551bcc 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -2489,7 +2489,7 @@ export const schemaDict = { encoding: 'application/json', schema: { type: 'object', - required: ['handle', 'email', 'password'], + required: ['handle'], properties: { email: { type: 'string', @@ -3170,6 +3170,18 @@ export const schemaDict = { main: { type: 'procedure', description: 'Reserve a repo signing key for account creation.', + input: { + encoding: 'application/json', + schema: { + type: 'object', + properties: { + did: { + type: 'string', + description: 'The did to reserve a new did:key for', + }, + }, + }, + }, output: { encoding: 'application/json', schema: { diff --git a/packages/pds/src/lexicon/types/com/atproto/server/createAccount.ts b/packages/pds/src/lexicon/types/com/atproto/server/createAccount.ts index f82fe9ed82a..109d34cf202 100644 --- a/packages/pds/src/lexicon/types/com/atproto/server/createAccount.ts +++ b/packages/pds/src/lexicon/types/com/atproto/server/createAccount.ts @@ -11,11 +11,11 @@ import { HandlerAuth } from '@atproto/xrpc-server' export interface QueryParams {} export interface InputSchema { - email: string + email?: string handle: string did?: string inviteCode?: string - password: string + password?: string recoveryKey?: string plcOp?: {} [k: string]: unknown diff --git a/packages/pds/src/lexicon/types/com/atproto/server/reserveSigningKey.ts b/packages/pds/src/lexicon/types/com/atproto/server/reserveSigningKey.ts index 495b87dc03c..ad5a5a8758c 100644 --- a/packages/pds/src/lexicon/types/com/atproto/server/reserveSigningKey.ts +++ b/packages/pds/src/lexicon/types/com/atproto/server/reserveSigningKey.ts @@ -10,7 +10,11 @@ import { HandlerAuth } from '@atproto/xrpc-server' export interface QueryParams {} -export type InputSchema = undefined +export interface InputSchema { + /** The did to reserve a new did:key for */ + did?: string + [k: string]: unknown +} export interface OutputSchema { /** Public signing key in the form of a did:key. */ @@ -18,7 +22,10 @@ export interface OutputSchema { [k: string]: unknown } -export type HandlerInput = undefined +export interface HandlerInput { + encoding: 'application/json' + body: InputSchema +} export interface HandlerSuccess { encoding: 'application/json' From 84e2d4d2b6694f344d80c18672c78b650189d423 Mon Sep 17 00:00:00 2001 From: bnewbold Date: Thu, 2 Nov 2023 00:45:13 -0700 Subject: [PATCH 17/59] Update atproto source license to MIT/Apache2 (#1788) * update LICENSE files to MIT/Apache2 * README: update top-level license * update package-level READMEs with license * changeset: license update --- .changeset/warm-lions-occur.md | 18 +++ LICENSE-APACHE.txt | 202 +++++++++++++++++++++++++++++++++ LICENSE => LICENSE-MIT.txt | 2 - LICENSE.txt | 7 ++ README.md | 7 +- packages/api/README.md | 7 +- packages/bsky/README.md | 7 +- packages/common-web/README.md | 7 +- packages/common/README.md | 7 +- packages/crypto/README.md | 7 +- packages/dev-env/README.md | 7 +- packages/identity/README.md | 7 +- packages/lex-cli/README.md | 7 +- packages/lexicon/README.md | 7 +- packages/pds/README.md | 7 +- packages/repo/README.md | 7 +- packages/syntax/README.md | 7 +- packages/xrpc-server/README.md | 7 +- packages/xrpc/README.md | 7 +- 19 files changed, 317 insertions(+), 17 deletions(-) create mode 100644 .changeset/warm-lions-occur.md create mode 100644 LICENSE-APACHE.txt rename LICENSE => LICENSE-MIT.txt (96%) create mode 100644 LICENSE.txt diff --git a/.changeset/warm-lions-occur.md b/.changeset/warm-lions-occur.md new file mode 100644 index 00000000000..3e86a4d3d2f --- /dev/null +++ b/.changeset/warm-lions-occur.md @@ -0,0 +1,18 @@ +--- +'@atproto/xrpc-server': patch +'@atproto/common-web': patch +'@atproto/identity': patch +'@atproto/dev-env': patch +'@atproto/lex-cli': patch +'@atproto/lexicon': patch +'@atproto/common': patch +'@atproto/crypto': patch +'@atproto/syntax': patch +'@atproto/bsky': patch +'@atproto/repo': patch +'@atproto/xrpc': patch +'@atproto/api': patch +'@atproto/pds': patch +--- + +update license to "MIT or Apache2" diff --git a/LICENSE-APACHE.txt b/LICENSE-APACHE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/LICENSE-APACHE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSE b/LICENSE-MIT.txt similarity index 96% rename from LICENSE rename to LICENSE-MIT.txt index 042cffe65ef..9cf106272ac 100644 --- a/LICENSE +++ b/LICENSE-MIT.txt @@ -1,7 +1,5 @@ MIT License -Copyright (c) 2022-2023 Bluesky PBC - Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 00000000000..36f42cf81d0 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,7 @@ +Dual MIT/Apache-2.0 License + +Copyright (c) 2022-2023 Bluesky PBC, and Contributors + +Except as otherwise noted in individual files, this software is licensed under the MIT license (), or the Apache License, Version 2.0 (). + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. diff --git a/README.md b/README.md index 0fe2bdade63..5a0a18ce853 100644 --- a/README.md +++ b/README.md @@ -108,4 +108,9 @@ If you discover any security issues, please send an email to security@bsky.app. ## License -MIT License. See [LICENSE](./LICENSE) for full text. +This project is dual-licensed under MIT and Apache 2.0 terms: + +- MIT license ([LICENSE-MIT.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-MIT.txt) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0, ([LICENSE-APACHE.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-APACHE.txt) or http://www.apache.org/licenses/LICENSE-2.0) + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. diff --git a/packages/api/README.md b/packages/api/README.md index 9f59f3fbc9b..4d1cfb5f35f 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -335,4 +335,9 @@ BskyAgent.configure({ ## License -MIT +This project is dual-licensed under MIT and Apache 2.0 terms: + +- MIT license ([LICENSE-MIT.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-MIT.txt) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0, ([LICENSE-APACHE.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-APACHE.txt) or http://www.apache.org/licenses/LICENSE-2.0) + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. diff --git a/packages/bsky/README.md b/packages/bsky/README.md index 8066ae30e1f..8849ae34de3 100644 --- a/packages/bsky/README.md +++ b/packages/bsky/README.md @@ -7,4 +7,9 @@ TypeScript implementation of the `app.bsky` Lexicons backing the https://bsky.ap ## License -MIT License +This project is dual-licensed under MIT and Apache 2.0 terms: + +- MIT license ([LICENSE-MIT.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-MIT.txt) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0, ([LICENSE-APACHE.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-APACHE.txt) or http://www.apache.org/licenses/LICENSE-2.0) + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. diff --git a/packages/common-web/README.md b/packages/common-web/README.md index 74426aebe37..9dec61b4373 100644 --- a/packages/common-web/README.md +++ b/packages/common-web/README.md @@ -7,4 +7,9 @@ Shared TypeScript code for other `@atproto/*` packages, which is web-friendly (r ## License -MIT License +This project is dual-licensed under MIT and Apache 2.0 terms: + +- MIT license ([LICENSE-MIT.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-MIT.txt) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0, ([LICENSE-APACHE.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-APACHE.txt) or http://www.apache.org/licenses/LICENSE-2.0) + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. diff --git a/packages/common/README.md b/packages/common/README.md index c08104cebe1..2cd3e74b23a 100644 --- a/packages/common/README.md +++ b/packages/common/README.md @@ -7,4 +7,9 @@ Shared TypeScript code for other `@atproto/*` packages. This package is oriented ## License -MIT License +This project is dual-licensed under MIT and Apache 2.0 terms: + +- MIT license ([LICENSE-MIT.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-MIT.txt) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0, ([LICENSE-APACHE.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-APACHE.txt) or http://www.apache.org/licenses/LICENSE-2.0) + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. diff --git a/packages/crypto/README.md b/packages/crypto/README.md index 0b610bf7f6f..f92c5cbfb1f 100644 --- a/packages/crypto/README.md +++ b/packages/crypto/README.md @@ -43,4 +43,9 @@ if (!ok) { ## License -MIT License +This project is dual-licensed under MIT and Apache 2.0 terms: + +- MIT license ([LICENSE-MIT.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-MIT.txt) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0, ([LICENSE-APACHE.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-APACHE.txt) or http://www.apache.org/licenses/LICENSE-2.0) + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. diff --git a/packages/dev-env/README.md b/packages/dev-env/README.md index 60befbe44ca..6c2fbae1f3a 100644 --- a/packages/dev-env/README.md +++ b/packages/dev-env/README.md @@ -31,4 +31,9 @@ Get the `ServiceClient` for the given user. ## License -MIT License +This project is dual-licensed under MIT and Apache 2.0 terms: + +- MIT license ([LICENSE-MIT.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-MIT.txt) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0, ([LICENSE-APACHE.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-APACHE.txt) or http://www.apache.org/licenses/LICENSE-2.0) + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. diff --git a/packages/identity/README.md b/packages/identity/README.md index 874fe23570e..9806a3c5f67 100644 --- a/packages/identity/README.md +++ b/packages/identity/README.md @@ -37,4 +37,9 @@ if (data.handle != handle) { ## License -MIT License +This project is dual-licensed under MIT and Apache 2.0 terms: + +- MIT license ([LICENSE-MIT.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-MIT.txt) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0, ([LICENSE-APACHE.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-APACHE.txt) or http://www.apache.org/licenses/LICENSE-2.0) + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. diff --git a/packages/lex-cli/README.md b/packages/lex-cli/README.md index 1eee8f0c5ac..a7fef9e3a2a 100644 --- a/packages/lex-cli/README.md +++ b/packages/lex-cli/README.md @@ -36,4 +36,9 @@ $ lex gen-server ./server/src/xrpc ./schemas/com/service/*.json ./schemas/com/an ## License -MIT +This project is dual-licensed under MIT and Apache 2.0 terms: + +- MIT license ([LICENSE-MIT.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-MIT.txt) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0, ([LICENSE-APACHE.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-APACHE.txt) or http://www.apache.org/licenses/LICENSE-2.0) + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. diff --git a/packages/lexicon/README.md b/packages/lexicon/README.md index 33fd777ce7f..697abd97d03 100644 --- a/packages/lexicon/README.md +++ b/packages/lexicon/README.md @@ -31,4 +31,9 @@ lex.assertValidXrpcOutput('com.example.query', {...}) ## License -MIT +This project is dual-licensed under MIT and Apache 2.0 terms: + +- MIT license ([LICENSE-MIT.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-MIT.txt) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0, ([LICENSE-APACHE.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-APACHE.txt) or http://www.apache.org/licenses/LICENSE-2.0) + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. diff --git a/packages/pds/README.md b/packages/pds/README.md index b70d99cb780..e99e2b42965 100644 --- a/packages/pds/README.md +++ b/packages/pds/README.md @@ -9,4 +9,9 @@ If you are interested in self-hosting a PDS, you probably want this repository i ## License -MIT License +This project is dual-licensed under MIT and Apache 2.0 terms: + +- MIT license ([LICENSE-MIT.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-MIT.txt) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0, ([LICENSE-APACHE.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-APACHE.txt) or http://www.apache.org/licenses/LICENSE-2.0) + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. diff --git a/packages/repo/README.md b/packages/repo/README.md index e018cf76ff8..a972e547124 100644 --- a/packages/repo/README.md +++ b/packages/repo/README.md @@ -9,4 +9,9 @@ Repositories in atproto are signed key/value stores containing CBOR-encoded data ## License -MIT License +This project is dual-licensed under MIT and Apache 2.0 terms: + +- MIT license ([LICENSE-MIT.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-MIT.txt) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0, ([LICENSE-APACHE.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-APACHE.txt) or http://www.apache.org/licenses/LICENSE-2.0) + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. diff --git a/packages/syntax/README.md b/packages/syntax/README.md index 0658b64d59c..c27205580b8 100644 --- a/packages/syntax/README.md +++ b/packages/syntax/README.md @@ -69,4 +69,9 @@ uri.rkey // => '1234' ## License -MIT +This project is dual-licensed under MIT and Apache 2.0 terms: + +- MIT license ([LICENSE-MIT.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-MIT.txt) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0, ([LICENSE-APACHE.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-APACHE.txt) or http://www.apache.org/licenses/LICENSE-2.0) + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. diff --git a/packages/xrpc-server/README.md b/packages/xrpc-server/README.md index 03314c342a6..7c53339bc12 100644 --- a/packages/xrpc-server/README.md +++ b/packages/xrpc-server/README.md @@ -51,4 +51,9 @@ app.listen(8080) ## License -MIT +This project is dual-licensed under MIT and Apache 2.0 terms: + +- MIT license ([LICENSE-MIT.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-MIT.txt) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0, ([LICENSE-APACHE.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-APACHE.txt) or http://www.apache.org/licenses/LICENSE-2.0) + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. diff --git a/packages/xrpc/README.md b/packages/xrpc/README.md index 5789302658e..31b66e7f32c 100644 --- a/packages/xrpc/README.md +++ b/packages/xrpc/README.md @@ -71,4 +71,9 @@ const res3 = await xrpc.service('https://example.com').call( ## License -MIT +This project is dual-licensed under MIT and Apache 2.0 terms: + +- MIT license ([LICENSE-MIT.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-MIT.txt) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0, ([LICENSE-APACHE.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-APACHE.txt) or http://www.apache.org/licenses/LICENSE-2.0) + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. From b5a77babdb6229a29488e839c762ce4e8b835d64 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 2 Nov 2023 19:51:40 +0000 Subject: [PATCH 18/59] Adjust types to use LexiconDoc (#1807) --- packages/xrpc-server/README.md | 9 ++++++--- packages/xrpc-server/tests/auth.test.ts | 3 ++- packages/xrpc-server/tests/bodies.test.ts | 3 ++- packages/xrpc-server/tests/errors.test.ts | 3 ++- packages/xrpc-server/tests/ipld.test.ts | 3 ++- packages/xrpc-server/tests/parameters.test.ts | 3 ++- packages/xrpc-server/tests/procedures.test.ts | 3 ++- packages/xrpc-server/tests/queries.test.ts | 3 ++- packages/xrpc-server/tests/rate-limiter.test.ts | 3 ++- packages/xrpc-server/tests/responses.test.ts | 3 ++- packages/xrpc-server/tests/subscriptions.test.ts | 3 ++- packages/xrpc/README.md | 11 +++++++---- 12 files changed, 33 insertions(+), 17 deletions(-) diff --git a/packages/xrpc-server/README.md b/packages/xrpc-server/README.md index 7c53339bc12..1228b425d2d 100644 --- a/packages/xrpc-server/README.md +++ b/packages/xrpc-server/README.md @@ -8,11 +8,11 @@ TypeScript library for implementing [atproto](https://atproto.com) HTTP API serv ## Usage ```typescript +import { LexiconDoc } from '@atproto/lexicon' import * as xrpc from '@atproto/xrpc-server' import express from 'express' -// create xrpc server -const server = xrpc.createServer([ +const lexicons: LexiconDoc[] = [ { lexicon: 1, id: 'io.example.ping', @@ -29,7 +29,10 @@ const server = xrpc.createServer([ }, }, }, -]) +] + +// create xrpc server +const server = xrpc.createServer(lexicons) function ping(ctx: { auth: xrpc.HandlerAuth | undefined diff --git a/packages/xrpc-server/tests/auth.test.ts b/packages/xrpc-server/tests/auth.test.ts index 53f3a6c6d24..bbd202d1024 100644 --- a/packages/xrpc-server/tests/auth.test.ts +++ b/packages/xrpc-server/tests/auth.test.ts @@ -6,6 +6,7 @@ import KeyEncoder from 'key-encoder' import * as ui8 from 'uint8arrays' import { MINUTE } from '@atproto/common' import { Secp256k1Keypair } from '@atproto/crypto' +import { LexiconDoc } from '@atproto/lexicon' import xrpc, { ServiceClient, XRPCError } from '@atproto/xrpc' import * as xrpcServer from '../src' import { @@ -15,7 +16,7 @@ import { basicAuthHeaders, } from './_util' -const LEXICONS = [ +const LEXICONS: LexiconDoc[] = [ { lexicon: 1, id: 'io.example.authTest', diff --git a/packages/xrpc-server/tests/bodies.test.ts b/packages/xrpc-server/tests/bodies.test.ts index 725146907e2..072299cdcd7 100644 --- a/packages/xrpc-server/tests/bodies.test.ts +++ b/packages/xrpc-server/tests/bodies.test.ts @@ -2,6 +2,7 @@ import * as http from 'http' import { Readable } from 'stream' import { gzipSync } from 'zlib' import getPort from 'get-port' +import { LexiconDoc } from '@atproto/lexicon' import xrpc, { ServiceClient } from '@atproto/xrpc' import { bytesToStream, cidForCbor } from '@atproto/common' import { randomBytes } from '@atproto/crypto' @@ -9,7 +10,7 @@ import { createServer, closeServer } from './_util' import * as xrpcServer from '../src' import logger from '../src/logger' -const LEXICONS = [ +const LEXICONS: LexiconDoc[] = [ { lexicon: 1, id: 'io.example.validationTest', diff --git a/packages/xrpc-server/tests/errors.test.ts b/packages/xrpc-server/tests/errors.test.ts index 4703c5231f0..c7781e71907 100644 --- a/packages/xrpc-server/tests/errors.test.ts +++ b/packages/xrpc-server/tests/errors.test.ts @@ -1,5 +1,6 @@ import * as http from 'http' import getPort from 'get-port' +import { LexiconDoc } from '@atproto/lexicon' import { createServer, closeServer } from './_util' import * as xrpcServer from '../src' import xrpc, { @@ -9,7 +10,7 @@ import xrpc, { XRPCInvalidResponseError, } from '@atproto/xrpc' -const LEXICONS = [ +const LEXICONS: LexiconDoc[] = [ { lexicon: 1, id: 'io.example.error', diff --git a/packages/xrpc-server/tests/ipld.test.ts b/packages/xrpc-server/tests/ipld.test.ts index 2c84e8ed5dc..4ba1fec867e 100644 --- a/packages/xrpc-server/tests/ipld.test.ts +++ b/packages/xrpc-server/tests/ipld.test.ts @@ -1,11 +1,12 @@ import * as http from 'http' +import { LexiconDoc } from '@atproto/lexicon' import xrpc, { ServiceClient } from '@atproto/xrpc' import { CID } from 'multiformats/cid' import getPort from 'get-port' import { createServer, closeServer } from './_util' import * as xrpcServer from '../src' -const LEXICONS = [ +const LEXICONS: LexiconDoc[] = [ { lexicon: 1, id: 'io.example.ipld', diff --git a/packages/xrpc-server/tests/parameters.test.ts b/packages/xrpc-server/tests/parameters.test.ts index a6575f121da..6ab2066ae8f 100644 --- a/packages/xrpc-server/tests/parameters.test.ts +++ b/packages/xrpc-server/tests/parameters.test.ts @@ -1,10 +1,11 @@ import * as http from 'http' import getPort from 'get-port' +import { LexiconDoc } from '@atproto/lexicon' import xrpc, { ServiceClient } from '@atproto/xrpc' import { createServer, closeServer } from './_util' import * as xrpcServer from '../src' -const LEXICONS = [ +const LEXICONS: LexiconDoc[] = [ { lexicon: 1, id: 'io.example.paramTest', diff --git a/packages/xrpc-server/tests/procedures.test.ts b/packages/xrpc-server/tests/procedures.test.ts index 6d4cef19d02..df38fc855f9 100644 --- a/packages/xrpc-server/tests/procedures.test.ts +++ b/packages/xrpc-server/tests/procedures.test.ts @@ -1,11 +1,12 @@ import * as http from 'http' import { Readable } from 'stream' +import { LexiconDoc } from '@atproto/lexicon' import xrpc, { ServiceClient } from '@atproto/xrpc' import getPort from 'get-port' import { createServer, closeServer } from './_util' import * as xrpcServer from '../src' -const LEXICONS = [ +const LEXICONS: LexiconDoc[] = [ { lexicon: 1, id: 'io.example.pingOne', diff --git a/packages/xrpc-server/tests/queries.test.ts b/packages/xrpc-server/tests/queries.test.ts index a80e813ec4e..fd45d812e00 100644 --- a/packages/xrpc-server/tests/queries.test.ts +++ b/packages/xrpc-server/tests/queries.test.ts @@ -1,10 +1,11 @@ import * as http from 'http' import getPort from 'get-port' +import { LexiconDoc } from '@atproto/lexicon' import xrpc, { ServiceClient } from '@atproto/xrpc' import { createServer, closeServer } from './_util' import * as xrpcServer from '../src' -const LEXICONS = [ +const LEXICONS: LexiconDoc[] = [ { lexicon: 1, id: 'io.example.pingOne', diff --git a/packages/xrpc-server/tests/rate-limiter.test.ts b/packages/xrpc-server/tests/rate-limiter.test.ts index 09a235a38b9..20610c95688 100644 --- a/packages/xrpc-server/tests/rate-limiter.test.ts +++ b/packages/xrpc-server/tests/rate-limiter.test.ts @@ -1,12 +1,13 @@ import * as http from 'http' import getPort from 'get-port' +import { LexiconDoc } from '@atproto/lexicon' import xrpc, { ServiceClient } from '@atproto/xrpc' import { createServer, closeServer } from './_util' import * as xrpcServer from '../src' import { RateLimiter } from '../src' import { MINUTE } from '@atproto/common' -const LEXICONS = [ +const LEXICONS: LexiconDoc[] = [ { lexicon: 1, id: 'io.example.routeLimit', diff --git a/packages/xrpc-server/tests/responses.test.ts b/packages/xrpc-server/tests/responses.test.ts index 0eaccba0633..b61467dca22 100644 --- a/packages/xrpc-server/tests/responses.test.ts +++ b/packages/xrpc-server/tests/responses.test.ts @@ -1,11 +1,12 @@ import * as http from 'http' import getPort from 'get-port' +import { LexiconDoc } from '@atproto/lexicon' import xrpc, { ServiceClient } from '@atproto/xrpc' import { byteIterableToStream } from '@atproto/common' import { createServer, closeServer } from './_util' import * as xrpcServer from '../src' -const LEXICONS = [ +const LEXICONS: LexiconDoc[] = [ { lexicon: 1, id: 'io.example.readableStream', diff --git a/packages/xrpc-server/tests/subscriptions.test.ts b/packages/xrpc-server/tests/subscriptions.test.ts index 13b0301ca87..e23cef5e3d4 100644 --- a/packages/xrpc-server/tests/subscriptions.test.ts +++ b/packages/xrpc-server/tests/subscriptions.test.ts @@ -2,6 +2,7 @@ import * as http from 'http' import { WebSocket, WebSocketServer, createWebSocketStream } from 'ws' import getPort from 'get-port' import { wait } from '@atproto/common' +import { LexiconDoc } from '@atproto/lexicon' import { byFrame, MessageFrame, ErrorFrame, Frame, Subscription } from '../src' import { createServer, @@ -11,7 +12,7 @@ import { } from './_util' import * as xrpcServer from '../src' -const LEXICONS = [ +const LEXICONS: LexiconDoc[] = [ { lexicon: 1, id: 'io.example.streamOne', diff --git a/packages/xrpc/README.md b/packages/xrpc/README.md index 31b66e7f32c..17c48c93735 100644 --- a/packages/xrpc/README.md +++ b/packages/xrpc/README.md @@ -8,9 +8,10 @@ TypeScript client library for talking to [atproto](https://atproto.com) services ## Usage ```typescript +import { LexiconDoc } from '@atproto/lexicon' import xrpc from '@atproto/xrpc' -xrpc.addLexicon({ +const pingLexicon: LexiconDoc = { lexicon: 1, id: 'io.example.ping', defs: { @@ -31,7 +32,8 @@ xrpc.addLexicon({ }, }, }, -}) +} +xrpc.addLexicon(pingLexicon) const res1 = await xrpc.call('https://example.com', 'io.example.ping', { message: 'hello world', @@ -44,7 +46,7 @@ const res2 = await xrpc res2.encoding // => 'application/json' res2.body // => {message: 'hello world'} -xrpc.addLexicon({ +const writeJsonLexicon: LexiconDoc = { lexicon: 1, id: 'io.example.writeJsonFile', defs: { @@ -60,7 +62,8 @@ xrpc.addLexicon({ }, }, }, -}) +} +xrpc.addLexicon(writeJsonLexicon) const res3 = await xrpc.service('https://example.com').call( 'io.example.writeJsonFile', From ce49743d7f8800d33116b88001d7b512553c2c89 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 2 Nov 2023 19:52:36 +0000 Subject: [PATCH 19/59] Add changeset for #1790 (#1801) --- .changeset/dry-cameras-hang.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/dry-cameras-hang.md diff --git a/.changeset/dry-cameras-hang.md b/.changeset/dry-cameras-hang.md new file mode 100644 index 00000000000..5c511c3766f --- /dev/null +++ b/.changeset/dry-cameras-hang.md @@ -0,0 +1,7 @@ +--- +'@atproto/lexicon': minor +'@atproto/xrpc': minor +'@atproto/xrpc-server': minor +--- + +Methods that accepts lexicons now take `LexiconDoc` type instead of `unknown` From 3598898c031c268cec2ce3bf5b67d1bbb73bf849 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 Nov 2023 19:58:57 +0000 Subject: [PATCH 20/59] Version packages (#1802) Co-authored-by: github-actions[bot] --- .changeset/dry-cameras-hang.md | 7 ------- .changeset/warm-lions-occur.md | 18 ------------------ packages/api/CHANGELOG.md | 12 ++++++++++++ packages/api/package.json | 2 +- packages/aws/CHANGELOG.md | 9 +++++++++ packages/aws/package.json | 2 +- packages/bsky/CHANGELOG.md | 16 ++++++++++++++++ packages/bsky/package.json | 2 +- packages/common-web/CHANGELOG.md | 6 ++++++ packages/common-web/package.json | 2 +- packages/common/CHANGELOG.md | 9 +++++++++ packages/common/package.json | 2 +- packages/crypto/CHANGELOG.md | 7 +++++++ packages/crypto/package.json | 2 +- packages/dev-env/CHANGELOG.md | 17 +++++++++++++++++ packages/dev-env/package.json | 2 +- packages/identity/CHANGELOG.md | 10 ++++++++++ packages/identity/package.json | 2 +- packages/lex-cli/CHANGELOG.md | 10 ++++++++++ packages/lex-cli/package.json | 2 +- packages/lexicon/CHANGELOG.md | 14 ++++++++++++++ packages/lexicon/package.json | 2 +- packages/pds/CHANGELOG.md | 18 ++++++++++++++++++ packages/pds/package.json | 2 +- packages/repo/CHANGELOG.md | 14 ++++++++++++++ packages/repo/package.json | 2 +- packages/syntax/CHANGELOG.md | 9 +++++++++ packages/syntax/package.json | 2 +- packages/xrpc-server/CHANGELOG.md | 15 +++++++++++++++ packages/xrpc-server/package.json | 2 +- packages/xrpc/CHANGELOG.md | 13 +++++++++++++ packages/xrpc/package.json | 2 +- 32 files changed, 194 insertions(+), 40 deletions(-) delete mode 100644 .changeset/dry-cameras-hang.md delete mode 100644 .changeset/warm-lions-occur.md create mode 100644 packages/crypto/CHANGELOG.md diff --git a/.changeset/dry-cameras-hang.md b/.changeset/dry-cameras-hang.md deleted file mode 100644 index 5c511c3766f..00000000000 --- a/.changeset/dry-cameras-hang.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@atproto/lexicon': minor -'@atproto/xrpc': minor -'@atproto/xrpc-server': minor ---- - -Methods that accepts lexicons now take `LexiconDoc` type instead of `unknown` diff --git a/.changeset/warm-lions-occur.md b/.changeset/warm-lions-occur.md deleted file mode 100644 index 3e86a4d3d2f..00000000000 --- a/.changeset/warm-lions-occur.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -'@atproto/xrpc-server': patch -'@atproto/common-web': patch -'@atproto/identity': patch -'@atproto/dev-env': patch -'@atproto/lex-cli': patch -'@atproto/lexicon': patch -'@atproto/common': patch -'@atproto/crypto': patch -'@atproto/syntax': patch -'@atproto/bsky': patch -'@atproto/repo': patch -'@atproto/xrpc': patch -'@atproto/api': patch -'@atproto/pds': patch ---- - -update license to "MIT or Apache2" diff --git a/packages/api/CHANGELOG.md b/packages/api/CHANGELOG.md index d3c3a00c131..2cbffc45b04 100644 --- a/packages/api/CHANGELOG.md +++ b/packages/api/CHANGELOG.md @@ -1,5 +1,17 @@ # @atproto/api +## 0.6.22 + +### Patch Changes + +- [#1788](https://github.com/bluesky-social/atproto/pull/1788) [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423) Thanks [@bnewbold](https://github.com/bnewbold)! - update license to "MIT or Apache2" + +- Updated dependencies [[`ce49743d`](https://github.com/bluesky-social/atproto/commit/ce49743d7f8800d33116b88001d7b512553c2c89), [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423)]: + - @atproto/lexicon@0.3.0 + - @atproto/xrpc@0.4.0 + - @atproto/common-web@0.2.3 + - @atproto/syntax@0.1.4 + ## 0.6.21 ### Patch Changes diff --git a/packages/api/package.json b/packages/api/package.json index ed373c472c4..7f32b02ce0d 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/api", - "version": "0.6.21", + "version": "0.6.22", "license": "MIT", "description": "Client library for atproto and Bluesky", "keywords": [ diff --git a/packages/aws/CHANGELOG.md b/packages/aws/CHANGELOG.md index a587c2e8e26..43ada7d9e0f 100644 --- a/packages/aws/CHANGELOG.md +++ b/packages/aws/CHANGELOG.md @@ -1,5 +1,14 @@ # @atproto/aws +## 0.1.4 + +### Patch Changes + +- Updated dependencies [[`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423)]: + - @atproto/common@0.3.3 + - @atproto/crypto@0.2.3 + - @atproto/repo@0.3.4 + ## 0.1.3 ### Patch Changes diff --git a/packages/aws/package.json b/packages/aws/package.json index b370a69f5f4..40eaba51d83 100644 --- a/packages/aws/package.json +++ b/packages/aws/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/aws", - "version": "0.1.3", + "version": "0.1.4", "license": "MIT", "description": "Shared AWS cloud API helpers for atproto services", "keywords": [ diff --git a/packages/bsky/CHANGELOG.md b/packages/bsky/CHANGELOG.md index 867c1b62660..7a0a3664202 100644 --- a/packages/bsky/CHANGELOG.md +++ b/packages/bsky/CHANGELOG.md @@ -1,5 +1,21 @@ # @atproto/bsky +## 0.0.13 + +### Patch Changes + +- [#1788](https://github.com/bluesky-social/atproto/pull/1788) [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423) Thanks [@bnewbold](https://github.com/bnewbold)! - update license to "MIT or Apache2" + +- Updated dependencies [[`ce49743d`](https://github.com/bluesky-social/atproto/commit/ce49743d7f8800d33116b88001d7b512553c2c89), [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423)]: + - @atproto/lexicon@0.3.0 + - @atproto/xrpc-server@0.4.0 + - @atproto/identity@0.3.1 + - @atproto/common@0.3.3 + - @atproto/crypto@0.2.3 + - @atproto/syntax@0.1.4 + - @atproto/repo@0.3.4 + - @atproto/api@0.6.22 + ## 0.0.12 ### Patch Changes diff --git a/packages/bsky/package.json b/packages/bsky/package.json index ab2b69cb5a6..1129c038bbd 100644 --- a/packages/bsky/package.json +++ b/packages/bsky/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/bsky", - "version": "0.0.12", + "version": "0.0.13", "license": "MIT", "description": "Reference implementation of app.bsky App View (Bluesky API)", "keywords": [ diff --git a/packages/common-web/CHANGELOG.md b/packages/common-web/CHANGELOG.md index 28bc7f04df7..57a602b2267 100644 --- a/packages/common-web/CHANGELOG.md +++ b/packages/common-web/CHANGELOG.md @@ -1,5 +1,11 @@ # @atproto/common-web +## 0.2.3 + +### Patch Changes + +- [#1788](https://github.com/bluesky-social/atproto/pull/1788) [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423) Thanks [@bnewbold](https://github.com/bnewbold)! - update license to "MIT or Apache2" + ## 0.2.2 ### Patch Changes diff --git a/packages/common-web/package.json b/packages/common-web/package.json index 01d5768f91e..1df7806ab51 100644 --- a/packages/common-web/package.json +++ b/packages/common-web/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/common-web", - "version": "0.2.2", + "version": "0.2.3", "license": "MIT", "description": "Shared web-platform-friendly code for atproto libraries", "keywords": [ diff --git a/packages/common/CHANGELOG.md b/packages/common/CHANGELOG.md index 8c9ff1b090a..4d70c2a117c 100644 --- a/packages/common/CHANGELOG.md +++ b/packages/common/CHANGELOG.md @@ -1,5 +1,14 @@ # @atproto/common +## 0.3.3 + +### Patch Changes + +- [#1788](https://github.com/bluesky-social/atproto/pull/1788) [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423) Thanks [@bnewbold](https://github.com/bnewbold)! - update license to "MIT or Apache2" + +- Updated dependencies [[`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423)]: + - @atproto/common-web@0.2.3 + ## 0.3.2 ### Patch Changes diff --git a/packages/common/package.json b/packages/common/package.json index beef7fe7fa4..c9aaa9293e2 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/common", - "version": "0.3.2", + "version": "0.3.3", "license": "MIT", "description": "Shared web-platform-friendly code for atproto libraries", "keywords": [ diff --git a/packages/crypto/CHANGELOG.md b/packages/crypto/CHANGELOG.md new file mode 100644 index 00000000000..c023f142e7a --- /dev/null +++ b/packages/crypto/CHANGELOG.md @@ -0,0 +1,7 @@ +# @atproto/crypto + +## 0.2.3 + +### Patch Changes + +- [#1788](https://github.com/bluesky-social/atproto/pull/1788) [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423) Thanks [@bnewbold](https://github.com/bnewbold)! - update license to "MIT or Apache2" diff --git a/packages/crypto/package.json b/packages/crypto/package.json index 7b552ca6b43..b5b9773c077 100644 --- a/packages/crypto/package.json +++ b/packages/crypto/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/crypto", - "version": "0.2.2", + "version": "0.2.3", "license": "MIT", "description": "Library for cryptographic keys and signing in atproto", "keywords": [ diff --git a/packages/dev-env/CHANGELOG.md b/packages/dev-env/CHANGELOG.md index 3f00886dded..01cbddbf2f4 100644 --- a/packages/dev-env/CHANGELOG.md +++ b/packages/dev-env/CHANGELOG.md @@ -1,5 +1,22 @@ # @atproto/dev-env +## 0.2.13 + +### Patch Changes + +- [#1788](https://github.com/bluesky-social/atproto/pull/1788) [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423) Thanks [@bnewbold](https://github.com/bnewbold)! - update license to "MIT or Apache2" + +- Updated dependencies [[`ce49743d`](https://github.com/bluesky-social/atproto/commit/ce49743d7f8800d33116b88001d7b512553c2c89), [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423)]: + - @atproto/lexicon@0.3.0 + - @atproto/xrpc-server@0.4.0 + - @atproto/common-web@0.2.3 + - @atproto/identity@0.3.1 + - @atproto/crypto@0.2.3 + - @atproto/syntax@0.1.4 + - @atproto/bsky@0.0.13 + - @atproto/api@0.6.22 + - @atproto/pds@0.3.1 + ## 0.2.12 ### Patch Changes diff --git a/packages/dev-env/package.json b/packages/dev-env/package.json index 7cf3dbb2ade..362cc960a37 100644 --- a/packages/dev-env/package.json +++ b/packages/dev-env/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/dev-env", - "version": "0.2.12", + "version": "0.2.13", "license": "MIT", "description": "Local development environment helper for atproto development", "keywords": [ diff --git a/packages/identity/CHANGELOG.md b/packages/identity/CHANGELOG.md index 321025853e2..ac49fc13f1a 100644 --- a/packages/identity/CHANGELOG.md +++ b/packages/identity/CHANGELOG.md @@ -1,5 +1,15 @@ # @atproto/identity +## 0.3.1 + +### Patch Changes + +- [#1788](https://github.com/bluesky-social/atproto/pull/1788) [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423) Thanks [@bnewbold](https://github.com/bnewbold)! - update license to "MIT or Apache2" + +- Updated dependencies [[`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423)]: + - @atproto/common-web@0.2.3 + - @atproto/crypto@0.2.3 + ## 0.3.0 ### Minor Changes diff --git a/packages/identity/package.json b/packages/identity/package.json index 119017e77d0..e4f39c24ecd 100644 --- a/packages/identity/package.json +++ b/packages/identity/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/identity", - "version": "0.3.0", + "version": "0.3.1", "license": "MIT", "description": "Library for decentralized identities in atproto using DIDs and handles", "keywords": [ diff --git a/packages/lex-cli/CHANGELOG.md b/packages/lex-cli/CHANGELOG.md index 02f8d6e9a8e..b9c617f9a9d 100644 --- a/packages/lex-cli/CHANGELOG.md +++ b/packages/lex-cli/CHANGELOG.md @@ -1,5 +1,15 @@ # @atproto/lex-cli +## 0.2.4 + +### Patch Changes + +- [#1788](https://github.com/bluesky-social/atproto/pull/1788) [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423) Thanks [@bnewbold](https://github.com/bnewbold)! - update license to "MIT or Apache2" + +- Updated dependencies [[`ce49743d`](https://github.com/bluesky-social/atproto/commit/ce49743d7f8800d33116b88001d7b512553c2c89), [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423)]: + - @atproto/lexicon@0.3.0 + - @atproto/syntax@0.1.4 + ## 0.2.3 ### Patch Changes diff --git a/packages/lex-cli/package.json b/packages/lex-cli/package.json index ee8d7b47184..2f57a55b3cb 100644 --- a/packages/lex-cli/package.json +++ b/packages/lex-cli/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/lex-cli", - "version": "0.2.3", + "version": "0.2.4", "license": "MIT", "description": "TypeScript codegen tool for atproto Lexicon schemas", "keywords": [ diff --git a/packages/lexicon/CHANGELOG.md b/packages/lexicon/CHANGELOG.md index a5efe153f5c..7194b0258ba 100644 --- a/packages/lexicon/CHANGELOG.md +++ b/packages/lexicon/CHANGELOG.md @@ -1,5 +1,19 @@ # @atproto/lexicon +## 0.3.0 + +### Minor Changes + +- [#1801](https://github.com/bluesky-social/atproto/pull/1801) [`ce49743d`](https://github.com/bluesky-social/atproto/commit/ce49743d7f8800d33116b88001d7b512553c2c89) Thanks [@gaearon](https://github.com/gaearon)! - Methods that accepts lexicons now take `LexiconDoc` type instead of `unknown` + +### Patch Changes + +- [#1788](https://github.com/bluesky-social/atproto/pull/1788) [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423) Thanks [@bnewbold](https://github.com/bnewbold)! - update license to "MIT or Apache2" + +- Updated dependencies [[`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423)]: + - @atproto/common-web@0.2.3 + - @atproto/syntax@0.1.4 + ## 0.2.3 ### Patch Changes diff --git a/packages/lexicon/package.json b/packages/lexicon/package.json index b4eaeeedf85..fc776e7c273 100644 --- a/packages/lexicon/package.json +++ b/packages/lexicon/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/lexicon", - "version": "0.2.3", + "version": "0.3.0", "license": "MIT", "description": "atproto Lexicon schema language library", "keywords": [ diff --git a/packages/pds/CHANGELOG.md b/packages/pds/CHANGELOG.md index bd333402677..74eebe5cd1e 100644 --- a/packages/pds/CHANGELOG.md +++ b/packages/pds/CHANGELOG.md @@ -1,5 +1,23 @@ # @atproto/pds +## 0.3.1 + +### Patch Changes + +- [#1788](https://github.com/bluesky-social/atproto/pull/1788) [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423) Thanks [@bnewbold](https://github.com/bnewbold)! - update license to "MIT or Apache2" + +- Updated dependencies [[`ce49743d`](https://github.com/bluesky-social/atproto/commit/ce49743d7f8800d33116b88001d7b512553c2c89), [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423)]: + - @atproto/lexicon@0.3.0 + - @atproto/xrpc@0.4.0 + - @atproto/xrpc-server@0.4.0 + - @atproto/identity@0.3.1 + - @atproto/common@0.3.3 + - @atproto/crypto@0.2.3 + - @atproto/syntax@0.1.4 + - @atproto/repo@0.3.4 + - @atproto/api@0.6.22 + - @atproto/aws@0.1.4 + ## 0.3.0 ### Patch Changes diff --git a/packages/pds/package.json b/packages/pds/package.json index a7a61e86ea2..a2d1d454156 100644 --- a/packages/pds/package.json +++ b/packages/pds/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/pds", - "version": "0.3.0", + "version": "0.3.1", "license": "MIT", "description": "Reference implementation of atproto Personal Data Server (PDS)", "keywords": [ diff --git a/packages/repo/CHANGELOG.md b/packages/repo/CHANGELOG.md index 05258506fe9..753b8f39811 100644 --- a/packages/repo/CHANGELOG.md +++ b/packages/repo/CHANGELOG.md @@ -1,5 +1,19 @@ # @atproto/repo +## 0.3.4 + +### Patch Changes + +- [#1788](https://github.com/bluesky-social/atproto/pull/1788) [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423) Thanks [@bnewbold](https://github.com/bnewbold)! - update license to "MIT or Apache2" + +- Updated dependencies [[`ce49743d`](https://github.com/bluesky-social/atproto/commit/ce49743d7f8800d33116b88001d7b512553c2c89), [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423)]: + - @atproto/lexicon@0.3.0 + - @atproto/common-web@0.2.3 + - @atproto/identity@0.3.1 + - @atproto/common@0.3.3 + - @atproto/crypto@0.2.3 + - @atproto/syntax@0.1.4 + ## 0.3.3 ### Patch Changes diff --git a/packages/repo/package.json b/packages/repo/package.json index b8839ed9f6a..23022149653 100644 --- a/packages/repo/package.json +++ b/packages/repo/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/repo", - "version": "0.3.3", + "version": "0.3.4", "license": "MIT", "description": "atproto repo and MST implementation", "keywords": [ diff --git a/packages/syntax/CHANGELOG.md b/packages/syntax/CHANGELOG.md index 4df89fb53a1..bc243559bbf 100644 --- a/packages/syntax/CHANGELOG.md +++ b/packages/syntax/CHANGELOG.md @@ -1,5 +1,14 @@ # @atproto/syntax +## 0.1.4 + +### Patch Changes + +- [#1788](https://github.com/bluesky-social/atproto/pull/1788) [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423) Thanks [@bnewbold](https://github.com/bnewbold)! - update license to "MIT or Apache2" + +- Updated dependencies [[`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423)]: + - @atproto/common-web@0.2.3 + ## 0.1.3 ### Patch Changes diff --git a/packages/syntax/package.json b/packages/syntax/package.json index ff59423f8eb..645926886f5 100644 --- a/packages/syntax/package.json +++ b/packages/syntax/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/syntax", - "version": "0.1.3", + "version": "0.1.4", "license": "MIT", "description": "Validation for atproto identifiers and formats: DID, handle, NSID, AT URI, etc", "keywords": [ diff --git a/packages/xrpc-server/CHANGELOG.md b/packages/xrpc-server/CHANGELOG.md index 4be9b022bea..1a0f2ab8279 100644 --- a/packages/xrpc-server/CHANGELOG.md +++ b/packages/xrpc-server/CHANGELOG.md @@ -1,5 +1,20 @@ # @atproto/xrpc-server +## 0.4.0 + +### Minor Changes + +- [#1801](https://github.com/bluesky-social/atproto/pull/1801) [`ce49743d`](https://github.com/bluesky-social/atproto/commit/ce49743d7f8800d33116b88001d7b512553c2c89) Thanks [@gaearon](https://github.com/gaearon)! - Methods that accepts lexicons now take `LexiconDoc` type instead of `unknown` + +### Patch Changes + +- [#1788](https://github.com/bluesky-social/atproto/pull/1788) [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423) Thanks [@bnewbold](https://github.com/bnewbold)! - update license to "MIT or Apache2" + +- Updated dependencies [[`ce49743d`](https://github.com/bluesky-social/atproto/commit/ce49743d7f8800d33116b88001d7b512553c2c89), [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423)]: + - @atproto/lexicon@0.3.0 + - @atproto/common@0.3.3 + - @atproto/crypto@0.2.3 + ## 0.3.3 ### Patch Changes diff --git a/packages/xrpc-server/package.json b/packages/xrpc-server/package.json index 21589c26f6b..fae70972446 100644 --- a/packages/xrpc-server/package.json +++ b/packages/xrpc-server/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/xrpc-server", - "version": "0.3.3", + "version": "0.4.0", "license": "MIT", "description": "atproto HTTP API (XRPC) server library", "keywords": [ diff --git a/packages/xrpc/CHANGELOG.md b/packages/xrpc/CHANGELOG.md index 2d37c765ead..76ffea62682 100644 --- a/packages/xrpc/CHANGELOG.md +++ b/packages/xrpc/CHANGELOG.md @@ -1,5 +1,18 @@ # @atproto/xrpc +## 0.4.0 + +### Minor Changes + +- [#1801](https://github.com/bluesky-social/atproto/pull/1801) [`ce49743d`](https://github.com/bluesky-social/atproto/commit/ce49743d7f8800d33116b88001d7b512553c2c89) Thanks [@gaearon](https://github.com/gaearon)! - Methods that accepts lexicons now take `LexiconDoc` type instead of `unknown` + +### Patch Changes + +- [#1788](https://github.com/bluesky-social/atproto/pull/1788) [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423) Thanks [@bnewbold](https://github.com/bnewbold)! - update license to "MIT or Apache2" + +- Updated dependencies [[`ce49743d`](https://github.com/bluesky-social/atproto/commit/ce49743d7f8800d33116b88001d7b512553c2c89), [`84e2d4d2`](https://github.com/bluesky-social/atproto/commit/84e2d4d2b6694f344d80c18672c78b650189d423)]: + - @atproto/lexicon@0.3.0 + ## 0.3.3 ### Patch Changes diff --git a/packages/xrpc/package.json b/packages/xrpc/package.json index f9845c7fcfb..49cf7a06aa3 100644 --- a/packages/xrpc/package.json +++ b/packages/xrpc/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/xrpc", - "version": "0.3.3", + "version": "0.4.0", "license": "MIT", "description": "atproto HTTP API (XRPC) client library", "keywords": [ From 772736a01081f39504e1b19a1b3687783bb78f07 Mon Sep 17 00:00:00 2001 From: devin ivy Date: Thu, 2 Nov 2023 16:16:26 -0400 Subject: [PATCH 21/59] Add did doc output to getSession for session resumption (#1806) * add optional did doc output to getSession lexicon * add did doc output to getSession on pds, update client to use it * api test fixes * api changeset * tidy --- .changeset/thick-apples-invite.md | 5 +++++ lexicons/com/atproto/server/getSession.json | 3 ++- packages/api/src/agent.ts | 1 + packages/api/src/client/lexicons.ts | 3 +++ .../types/com/atproto/server/getSession.ts | 1 + packages/api/src/moderation/accumulator.ts | 1 - packages/api/tests/agent.test.ts | 17 ++++++++++++++--- packages/bsky/src/lexicon/lexicons.ts | 3 +++ .../types/com/atproto/server/getSession.ts | 1 + .../src/api/com/atproto/server/getSession.ts | 7 ++++++- packages/pds/src/lexicon/lexicons.ts | 3 +++ .../types/com/atproto/server/getSession.ts | 1 + 12 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 .changeset/thick-apples-invite.md diff --git a/.changeset/thick-apples-invite.md b/.changeset/thick-apples-invite.md new file mode 100644 index 00000000000..6f30f88d83e --- /dev/null +++ b/.changeset/thick-apples-invite.md @@ -0,0 +1,5 @@ +--- +'@atproto/api': patch +--- + +respect pds endpoint during session resumption diff --git a/lexicons/com/atproto/server/getSession.json b/lexicons/com/atproto/server/getSession.json index 7ff5569eb1b..5f7700882da 100644 --- a/lexicons/com/atproto/server/getSession.json +++ b/lexicons/com/atproto/server/getSession.json @@ -14,7 +14,8 @@ "handle": { "type": "string", "format": "handle" }, "did": { "type": "string", "format": "did" }, "email": { "type": "string" }, - "emailConfirmed": { "type": "boolean" } + "emailConfirmed": { "type": "boolean" }, + "didDoc": { "type": "unknown" } } } } diff --git a/packages/api/src/agent.ts b/packages/api/src/agent.ts index ce34865c189..aea3cce9d4b 100644 --- a/packages/api/src/agent.ts +++ b/packages/api/src/agent.ts @@ -165,6 +165,7 @@ export class AtpAgent { this.session.email = res.data.email this.session.handle = res.data.handle this.session.emailConfirmed = res.data.emailConfirmed + this._updateApiEndpoint(res.data.didDoc) return res } catch (e) { this.session = undefined diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index d9de1551bcc..f17885819a0 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -3007,6 +3007,9 @@ export const schemaDict = { emailConfirmed: { type: 'boolean', }, + didDoc: { + type: 'unknown', + }, }, }, }, diff --git a/packages/api/src/client/types/com/atproto/server/getSession.ts b/packages/api/src/client/types/com/atproto/server/getSession.ts index 91d51860982..6b82781f081 100644 --- a/packages/api/src/client/types/com/atproto/server/getSession.ts +++ b/packages/api/src/client/types/com/atproto/server/getSession.ts @@ -16,6 +16,7 @@ export interface OutputSchema { did: string email?: string emailConfirmed?: boolean + didDoc?: {} [k: string]: unknown } diff --git a/packages/api/src/moderation/accumulator.ts b/packages/api/src/moderation/accumulator.ts index 53180023934..716647ee772 100644 --- a/packages/api/src/moderation/accumulator.ts +++ b/packages/api/src/moderation/accumulator.ts @@ -1,5 +1,4 @@ import { AppBskyGraphDefs } from '../client/index' -import { AtUri } from '@atproto/syntax' import { Label, LabelPreference, diff --git a/packages/api/tests/agent.test.ts b/packages/api/tests/agent.test.ts index 7f85f3079af..807046deb00 100644 --- a/packages/api/tests/agent.test.ts +++ b/packages/api/tests/agent.test.ts @@ -1,3 +1,4 @@ +import assert from 'assert' import { defaultFetchHandler } from '@atproto/xrpc' import { AtpAgent, @@ -6,6 +7,7 @@ import { AtpSessionData, } from '..' import { TestNetworkNoAppView } from '@atproto/dev-env' +import { getPdsEndpoint, isValidDidDoc } from '@atproto/common-web' describe('agent', () => { let network: TestNetworkNoAppView @@ -46,16 +48,19 @@ describe('agent', () => { expect(agent.session?.did).toEqual(res.data.did) expect(agent.session?.email).toEqual('user1@test.com') expect(agent.session?.emailConfirmed).toEqual(false) + assert(isValidDidDoc(res.data.didDoc)) + expect(agent.api.xrpc.uri.origin).toEqual(getPdsEndpoint(res.data.didDoc)) const { data: sessionInfo } = await agent.api.com.atproto.server.getSession( {}, ) - expect(sessionInfo).toEqual({ + expect(sessionInfo).toMatchObject({ did: res.data.did, handle: res.data.handle, email: 'user1@test.com', emailConfirmed: false, }) + expect(isValidDidDoc(sessionInfo.didDoc)).toBe(true) expect(events.length).toEqual(1) expect(events[0]).toEqual('create') @@ -93,15 +98,18 @@ describe('agent', () => { expect(agent2.session?.did).toEqual(res1.data.did) expect(agent2.session?.email).toEqual('user2@test.com') expect(agent2.session?.emailConfirmed).toEqual(false) + assert(isValidDidDoc(res1.data.didDoc)) + expect(agent2.api.xrpc.uri.origin).toEqual(getPdsEndpoint(res1.data.didDoc)) const { data: sessionInfo } = await agent2.api.com.atproto.server.getSession({}) - expect(sessionInfo).toEqual({ + expect(sessionInfo).toMatchObject({ did: res1.data.did, handle: res1.data.handle, email, emailConfirmed: false, }) + expect(isValidDidDoc(sessionInfo.didDoc)).toBe(true) expect(events.length).toEqual(2) expect(events[0]).toEqual('create') @@ -136,15 +144,18 @@ describe('agent', () => { expect(agent2.hasSession).toEqual(true) expect(agent2.session?.handle).toEqual(res1.data.handle) expect(agent2.session?.did).toEqual(res1.data.did) + assert(isValidDidDoc(res1.data.didDoc)) + expect(agent2.api.xrpc.uri.origin).toEqual(getPdsEndpoint(res1.data.didDoc)) const { data: sessionInfo } = await agent2.api.com.atproto.server.getSession({}) - expect(sessionInfo).toEqual({ + expect(sessionInfo).toMatchObject({ did: res1.data.did, handle: res1.data.handle, email: res1.data.email, emailConfirmed: false, }) + expect(isValidDidDoc(sessionInfo.didDoc)).toBe(true) expect(events.length).toEqual(2) expect(events[0]).toEqual('create') diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index d9de1551bcc..f17885819a0 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -3007,6 +3007,9 @@ export const schemaDict = { emailConfirmed: { type: 'boolean', }, + didDoc: { + type: 'unknown', + }, }, }, }, diff --git a/packages/bsky/src/lexicon/types/com/atproto/server/getSession.ts b/packages/bsky/src/lexicon/types/com/atproto/server/getSession.ts index 7f066a500bf..4f95acf523d 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/server/getSession.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/server/getSession.ts @@ -17,6 +17,7 @@ export interface OutputSchema { did: string email?: string emailConfirmed?: boolean + didDoc?: {} [k: string]: unknown } diff --git a/packages/pds/src/api/com/atproto/server/getSession.ts b/packages/pds/src/api/com/atproto/server/getSession.ts index fa192f0057f..bf271a9c02e 100644 --- a/packages/pds/src/api/com/atproto/server/getSession.ts +++ b/packages/pds/src/api/com/atproto/server/getSession.ts @@ -1,13 +1,17 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import AppContext from '../../../../context' import { Server } from '../../../../lexicon' +import { didDocForSession } from './util' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.getSession({ auth: ctx.authVerifier.access, handler: async ({ auth }) => { const did = auth.credentials.did - const user = await ctx.services.account(ctx.db).getAccount(did) + const [user, didDoc] = await Promise.all([ + ctx.services.account(ctx.db).getAccount(did), + didDocForSession(ctx, did), + ]) if (!user) { throw new InvalidRequestError( `Could not find user info for account: ${did}`, @@ -18,6 +22,7 @@ export default function (server: Server, ctx: AppContext) { body: { handle: user.handle, did: user.did, + didDoc, email: user.email, emailConfirmed: !!user.emailConfirmedAt, }, diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index d9de1551bcc..f17885819a0 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -3007,6 +3007,9 @@ export const schemaDict = { emailConfirmed: { type: 'boolean', }, + didDoc: { + type: 'unknown', + }, }, }, }, diff --git a/packages/pds/src/lexicon/types/com/atproto/server/getSession.ts b/packages/pds/src/lexicon/types/com/atproto/server/getSession.ts index 7f066a500bf..4f95acf523d 100644 --- a/packages/pds/src/lexicon/types/com/atproto/server/getSession.ts +++ b/packages/pds/src/lexicon/types/com/atproto/server/getSession.ts @@ -17,6 +17,7 @@ export interface OutputSchema { did: string email?: string emailConfirmed?: boolean + didDoc?: {} [k: string]: unknown } From b90f2c4ef82560d35e2f2dfae0c7ae6a7e2b893f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 Nov 2023 16:30:44 -0400 Subject: [PATCH 22/59] Version packages (#1808) Co-authored-by: github-actions[bot] --- .changeset/thick-apples-invite.md | 5 ----- packages/api/CHANGELOG.md | 6 ++++++ packages/api/package.json | 2 +- packages/bsky/CHANGELOG.md | 7 +++++++ packages/bsky/package.json | 2 +- packages/dev-env/CHANGELOG.md | 9 +++++++++ packages/dev-env/package.json | 2 +- packages/pds/CHANGELOG.md | 7 +++++++ packages/pds/package.json | 2 +- 9 files changed, 33 insertions(+), 9 deletions(-) delete mode 100644 .changeset/thick-apples-invite.md diff --git a/.changeset/thick-apples-invite.md b/.changeset/thick-apples-invite.md deleted file mode 100644 index 6f30f88d83e..00000000000 --- a/.changeset/thick-apples-invite.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@atproto/api': patch ---- - -respect pds endpoint during session resumption diff --git a/packages/api/CHANGELOG.md b/packages/api/CHANGELOG.md index 2cbffc45b04..ca4ed16617b 100644 --- a/packages/api/CHANGELOG.md +++ b/packages/api/CHANGELOG.md @@ -1,5 +1,11 @@ # @atproto/api +## 0.6.23 + +### Patch Changes + +- [#1806](https://github.com/bluesky-social/atproto/pull/1806) [`772736a0`](https://github.com/bluesky-social/atproto/commit/772736a01081f39504e1b19a1b3687783bb78f07) Thanks [@devinivy](https://github.com/devinivy)! - respect pds endpoint during session resumption + ## 0.6.22 ### Patch Changes diff --git a/packages/api/package.json b/packages/api/package.json index 7f32b02ce0d..4267152e641 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/api", - "version": "0.6.22", + "version": "0.6.23", "license": "MIT", "description": "Client library for atproto and Bluesky", "keywords": [ diff --git a/packages/bsky/CHANGELOG.md b/packages/bsky/CHANGELOG.md index 7a0a3664202..fb715a89591 100644 --- a/packages/bsky/CHANGELOG.md +++ b/packages/bsky/CHANGELOG.md @@ -1,5 +1,12 @@ # @atproto/bsky +## 0.0.14 + +### Patch Changes + +- Updated dependencies [[`772736a0`](https://github.com/bluesky-social/atproto/commit/772736a01081f39504e1b19a1b3687783bb78f07)]: + - @atproto/api@0.6.23 + ## 0.0.13 ### Patch Changes diff --git a/packages/bsky/package.json b/packages/bsky/package.json index 1129c038bbd..60991c5f791 100644 --- a/packages/bsky/package.json +++ b/packages/bsky/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/bsky", - "version": "0.0.13", + "version": "0.0.14", "license": "MIT", "description": "Reference implementation of app.bsky App View (Bluesky API)", "keywords": [ diff --git a/packages/dev-env/CHANGELOG.md b/packages/dev-env/CHANGELOG.md index 01cbddbf2f4..97b9eac682c 100644 --- a/packages/dev-env/CHANGELOG.md +++ b/packages/dev-env/CHANGELOG.md @@ -1,5 +1,14 @@ # @atproto/dev-env +## 0.2.14 + +### Patch Changes + +- Updated dependencies [[`772736a0`](https://github.com/bluesky-social/atproto/commit/772736a01081f39504e1b19a1b3687783bb78f07)]: + - @atproto/api@0.6.23 + - @atproto/bsky@0.0.14 + - @atproto/pds@0.3.2 + ## 0.2.13 ### Patch Changes diff --git a/packages/dev-env/package.json b/packages/dev-env/package.json index 362cc960a37..3140d8ff189 100644 --- a/packages/dev-env/package.json +++ b/packages/dev-env/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/dev-env", - "version": "0.2.13", + "version": "0.2.14", "license": "MIT", "description": "Local development environment helper for atproto development", "keywords": [ diff --git a/packages/pds/CHANGELOG.md b/packages/pds/CHANGELOG.md index 74eebe5cd1e..aca0e381ad4 100644 --- a/packages/pds/CHANGELOG.md +++ b/packages/pds/CHANGELOG.md @@ -1,5 +1,12 @@ # @atproto/pds +## 0.3.2 + +### Patch Changes + +- Updated dependencies [[`772736a0`](https://github.com/bluesky-social/atproto/commit/772736a01081f39504e1b19a1b3687783bb78f07)]: + - @atproto/api@0.6.23 + ## 0.3.1 ### Patch Changes diff --git a/packages/pds/package.json b/packages/pds/package.json index a2d1d454156..adc0ed9e508 100644 --- a/packages/pds/package.json +++ b/packages/pds/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/pds", - "version": "0.3.1", + "version": "0.3.2", "license": "MIT", "description": "Reference implementation of atproto Personal Data Server (PDS)", "keywords": [ From bba9388ea95dd17c3aeb92c3415063bc7fb25918 Mon Sep 17 00:00:00 2001 From: Eric Bailey Date: Thu, 2 Nov 2023 17:29:27 -0500 Subject: [PATCH 23/59] Add a test for links that end in .php (#1810) --- packages/api/tests/rich-text-detection.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/api/tests/rich-text-detection.test.ts b/packages/api/tests/rich-text-detection.test.ts index 64a53d1131c..88f2940c59e 100644 --- a/packages/api/tests/rich-text-detection.test.ts +++ b/packages/api/tests/rich-text-detection.test.ts @@ -40,6 +40,7 @@ describe('detectFacets', () => { 'start.com/foo/bar?baz=bux#hash middle end', 'start middle end.com/foo/bar?baz=bux#hash', 'newline1.com\nnewline2.com', + 'a example.com/index.php php link', 'not.. a..url ..here', 'e.g.', @@ -156,6 +157,11 @@ describe('detectFacets', () => { ['\n'], ['newline2.com', 'https://newline2.com'], ], + [ + ['a '], + ['example.com/index.php', 'https://example.com/index.php'], + [' php link'], + ], [['not.. a..url ..here']], [['e.g.']], From a3d81dd911b39877dc07d993912c14a2271646aa Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Fri, 3 Nov 2023 16:55:50 -0500 Subject: [PATCH 24/59] Prevent thread loops (#1813) * prevent thread loops * include original uri * tidy * tweak * last tweak * last tweak i swear * wording --- .../src/api/app/bsky/feed/getPostThread.ts | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/bsky/src/api/app/bsky/feed/getPostThread.ts b/packages/bsky/src/api/app/bsky/feed/getPostThread.ts index 1480e54ac1a..0e31107d052 100644 --- a/packages/bsky/src/api/app/bsky/feed/getPostThread.ts +++ b/packages/bsky/src/api/app/bsky/feed/getPostThread.ts @@ -255,11 +255,15 @@ const getThreadData = async ( .orderBy('sortAt', 'desc') .execute(), ]) - const parentsByUri = parents.reduce((acc, parent) => { - return Object.assign(acc, { [parent.postUri]: parent }) + // prevent self-referential loops + const includedPosts = new Set([uri]) + const parentsByUri = parents.reduce((acc, post) => { + return Object.assign(acc, { [post.uri]: post }) }, {} as Record) const childrenByParentUri = children.reduce((acc, child) => { if (!child.replyParent) return acc + if (includedPosts.has(child.uri)) return acc + includedPosts.add(child.uri) acc[child.replyParent] ??= [] acc[child.replyParent].push(child) return acc @@ -269,7 +273,12 @@ const getThreadData = async ( return { post, parent: post.replyParent - ? getParentData(parentsByUri, post.replyParent, parentHeight) + ? getParentData( + parentsByUri, + includedPosts, + post.replyParent, + parentHeight, + ) : undefined, replies: getChildrenData(childrenByParentUri, uri, depth), } @@ -277,16 +286,19 @@ const getThreadData = async ( const getParentData = ( postsByUri: Record, + includedPosts: Set, uri: string, depth: number, ): PostThread | ParentNotFoundError | undefined => { if (depth < 1) return undefined + if (includedPosts.has(uri)) return undefined + includedPosts.add(uri) const post = postsByUri[uri] if (!post) return new ParentNotFoundError(uri) return { post, parent: post.replyParent - ? getParentData(postsByUri, post.replyParent, depth - 1) + ? getParentData(postsByUri, includedPosts, post.replyParent, depth - 1) : undefined, replies: [], } From bebc4bac8c42a97e91a8272e7deffcc6f46ceba0 Mon Sep 17 00:00:00 2001 From: devin ivy Date: Sun, 5 Nov 2023 00:33:14 -0400 Subject: [PATCH 25/59] Simplify moderation fanout to configured url (#1804) * simplify moderation fanout to configured url * fix bsky mod fanout tests --- .../atproto/admin/reverseModerationAction.ts | 6 +++--- .../com/atproto/admin/takeModerationAction.ts | 4 ++-- .../bsky/src/api/com/atproto/admin/util.ts | 3 ++- packages/bsky/src/config.ts | 12 +++++------ packages/bsky/src/context.ts | 20 ++++++++++--------- .../db/periodic-moderation-action-reversal.ts | 10 +--------- packages/dev-env/src/network.ts | 1 + 7 files changed, 26 insertions(+), 30 deletions(-) diff --git a/packages/bsky/src/api/com/atproto/admin/reverseModerationAction.ts b/packages/bsky/src/api/com/atproto/admin/reverseModerationAction.ts index ae76df5b0c7..a441d2b934c 100644 --- a/packages/bsky/src/api/com/atproto/admin/reverseModerationAction.ts +++ b/packages/bsky/src/api/com/atproto/admin/reverseModerationAction.ts @@ -85,9 +85,9 @@ export default function (server: Server, ctx: AppContext) { return { result, restored } }) - if (restored) { - const { did, subjects } = restored - const agent = await ctx.pdsAdminAgent(did) + if (restored && ctx.moderationPushAgent) { + const agent = ctx.moderationPushAgent + const { subjects } = restored const results = await Promise.allSettled( subjects.map((subject) => retryHttp(() => diff --git a/packages/bsky/src/api/com/atproto/admin/takeModerationAction.ts b/packages/bsky/src/api/com/atproto/admin/takeModerationAction.ts index a8d67fced9f..5239ddec42d 100644 --- a/packages/bsky/src/api/com/atproto/admin/takeModerationAction.ts +++ b/packages/bsky/src/api/com/atproto/admin/takeModerationAction.ts @@ -111,10 +111,10 @@ export default function (server: Server, ctx: AppContext) { return { result, takenDown } }) - if (takenDown) { + if (takenDown && ctx.moderationPushAgent) { + const agent = ctx.moderationPushAgent const { did, subjects } = takenDown if (did && subjects.length > 0) { - const agent = await ctx.pdsAdminAgent(did) const results = await Promise.allSettled( subjects.map((subject) => retryHttp(() => diff --git a/packages/bsky/src/api/com/atproto/admin/util.ts b/packages/bsky/src/api/com/atproto/admin/util.ts index eba3eaa1d1e..6217e71eb0f 100644 --- a/packages/bsky/src/api/com/atproto/admin/util.ts +++ b/packages/bsky/src/api/com/atproto/admin/util.ts @@ -9,8 +9,9 @@ export const getPdsAccountInfo = async ( ctx: AppContext, did: string, ): Promise => { + const agent = ctx.moderationPushAgent + if (!agent) return null try { - const agent = await ctx.pdsAdminAgent(did) const res = await agent.api.com.atproto.admin.getAccountInfo({ did }) return res.data } catch (err) { diff --git a/packages/bsky/src/config.ts b/packages/bsky/src/config.ts index 38d9c883760..679f99c59a4 100644 --- a/packages/bsky/src/config.ts +++ b/packages/bsky/src/config.ts @@ -23,7 +23,7 @@ export interface ServerConfigValues { adminPassword: string moderatorPassword?: string triagePassword?: string - moderationActionReverseUrl?: string + moderationPushUrl?: string } export class ServerConfig { @@ -78,8 +78,8 @@ export class ServerConfig { const moderatorPassword = process.env.MODERATOR_PASSWORD || undefined const triagePassword = process.env.TRIAGE_PASSWORD || undefined const labelerDid = process.env.LABELER_DID || 'did:example:labeler' - const moderationActionReverseUrl = - overrides?.moderationActionReverseUrl || + const moderationPushUrl = + overrides?.moderationPushUrl || process.env.MODERATION_PUSH_URL || undefined return new ServerConfig({ @@ -104,7 +104,7 @@ export class ServerConfig { adminPassword, moderatorPassword, triagePassword, - moderationActionReverseUrl, + moderationPushUrl, ...stripUndefineds(overrides ?? {}), }) } @@ -206,8 +206,8 @@ export class ServerConfig { return this.cfg.triagePassword } - get moderationActionReverseUrl() { - return this.cfg.moderationActionReverseUrl + get moderationPushUrl() { + return this.cfg.moderationPushUrl } } diff --git a/packages/bsky/src/context.ts b/packages/bsky/src/context.ts index 90e6cf60014..21c01c38fbd 100644 --- a/packages/bsky/src/context.ts +++ b/packages/bsky/src/context.ts @@ -15,6 +15,7 @@ import { LabelCache } from './label-cache' import { NotificationServer } from './notifications' export class AppContext { + public moderationPushAgent: AtpAgent | undefined constructor( private opts: { db: DatabaseCoordinator @@ -30,7 +31,16 @@ export class AppContext { algos: MountedAlgos notifServer: NotificationServer }, - ) {} + ) { + if (opts.cfg.moderationPushUrl) { + const url = new URL(opts.cfg.moderationPushUrl) + this.moderationPushAgent = new AtpAgent({ service: url.origin }) + this.moderationPushAgent.api.setHeader( + 'authorization', + auth.buildBasicAuth(url.username, url.password), + ) + } + } get db(): DatabaseCoordinator { return this.opts.db @@ -107,14 +117,6 @@ export class AppContext { }) } - async pdsAdminAgent(did: string): Promise { - const data = await this.idResolver.did.resolveAtprotoData(did) - const agent = new AtpAgent({ service: data.pds }) - const jwt = await this.serviceAuthJwt(did) - agent.api.setHeader('authorization', `Bearer ${jwt}`) - return agent - } - get backgroundQueue(): BackgroundQueue { return this.opts.backgroundQueue } diff --git a/packages/bsky/src/db/periodic-moderation-action-reversal.ts b/packages/bsky/src/db/periodic-moderation-action-reversal.ts index d15cef91afb..c148c408efe 100644 --- a/packages/bsky/src/db/periodic-moderation-action-reversal.ts +++ b/packages/bsky/src/db/periodic-moderation-action-reversal.ts @@ -3,7 +3,6 @@ import { Leader } from './leader' import { dbLogger } from '../logger' import AppContext from '../context' import AtpAgent from '@atproto/api' -import { buildBasicAuth } from '../auth' import { LabelService } from '../services/label' import { ModerationActionRow } from '../services/moderation' @@ -18,14 +17,7 @@ export class PeriodicModerationActionReversal { pushAgent?: AtpAgent constructor(private appContext: AppContext) { - if (appContext.cfg.moderationActionReverseUrl) { - const url = new URL(appContext.cfg.moderationActionReverseUrl) - this.pushAgent = new AtpAgent({ service: url.origin }) - this.pushAgent.api.setHeader( - 'authorization', - buildBasicAuth(url.username, url.password), - ) - } + this.pushAgent = appContext.moderationPushAgent } // invert label creation & negations diff --git a/packages/dev-env/src/network.ts b/packages/dev-env/src/network.ts index a6c150f0353..e1666935de7 100644 --- a/packages/dev-env/src/network.ts +++ b/packages/dev-env/src/network.ts @@ -40,6 +40,7 @@ export class TestNetwork extends TestNetworkNoAppView { dbPostgresSchema: `appview_${dbPostgresSchema}`, dbPrimaryPostgresUrl: dbPostgresUrl, redisHost, + moderationPushUrl: `http://admin:${ADMIN_PASSWORD}@localhost:${pdsPort}`, ...params.bsky, }) const pds = await TestPds.create({ From 697f5d36270f05891167b81a98e4c55cef3c1210 Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Tue, 7 Nov 2023 23:26:31 +0000 Subject: [PATCH 26/59] :sparkles: Expose emailConfirmedAt field from admin getRepo (#1757) * :sparkles: Expose emailConfirmedAt field from admin getRepo * :recycle: Fix typing for repo result * :broom: Cleanup unnecessary import * :sparkles: Adapt to the new pds based get account info method * :broom: Cleanup unused pds util --- lexicons/com/atproto/admin/defs.json | 4 ++- packages/api/src/client/lexicons.ts | 8 +++++ .../client/types/com/atproto/admin/defs.ts | 2 ++ .../bsky/src/api/com/atproto/admin/util.ts | 1 + packages/bsky/src/lexicon/lexicons.ts | 8 +++++ .../lexicon/types/com/atproto/admin/defs.ts | 2 ++ packages/bsky/tests/admin/get-repo.test.ts | 33 +++++++++++++++++++ .../pds/src/api/com/atproto/admin/util.ts | 17 ---------- packages/pds/src/lexicon/lexicons.ts | 8 +++++ .../lexicon/types/com/atproto/admin/defs.ts | 2 ++ packages/pds/src/services/account/index.ts | 2 ++ 11 files changed, 69 insertions(+), 18 deletions(-) diff --git a/lexicons/com/atproto/admin/defs.json b/lexicons/com/atproto/admin/defs.json index 318d1c33b5a..0d57da18ae8 100644 --- a/lexicons/com/atproto/admin/defs.json +++ b/lexicons/com/atproto/admin/defs.json @@ -248,7 +248,8 @@ } }, "invitesDisabled": { "type": "boolean" }, - "inviteNote": { "type": "string" } + "inviteNote": { "type": "string" }, + "emailConfirmedAt": { "type": "string", "format": "datetime" } } }, "accountView": { @@ -271,6 +272,7 @@ } }, "invitesDisabled": { "type": "boolean" }, + "emailConfirmedAt": { "type": "string", "format": "datetime" }, "inviteNote": { "type": "string" } } }, diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index f17885819a0..404526e7936 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -434,6 +434,10 @@ export const schemaDict = { inviteNote: { type: 'string', }, + emailConfirmedAt: { + type: 'string', + format: 'datetime', + }, }, }, accountView: { @@ -469,6 +473,10 @@ export const schemaDict = { invitesDisabled: { type: 'boolean', }, + emailConfirmedAt: { + type: 'string', + format: 'datetime', + }, inviteNote: { type: 'string', }, diff --git a/packages/api/src/client/types/com/atproto/admin/defs.ts b/packages/api/src/client/types/com/atproto/admin/defs.ts index 7c48fa87a3c..5ab3e3482ec 100644 --- a/packages/api/src/client/types/com/atproto/admin/defs.ts +++ b/packages/api/src/client/types/com/atproto/admin/defs.ts @@ -241,6 +241,7 @@ export interface RepoViewDetail { invites?: ComAtprotoServerDefs.InviteCode[] invitesDisabled?: boolean inviteNote?: string + emailConfirmedAt?: string [k: string]: unknown } @@ -264,6 +265,7 @@ export interface AccountView { invitedBy?: ComAtprotoServerDefs.InviteCode invites?: ComAtprotoServerDefs.InviteCode[] invitesDisabled?: boolean + emailConfirmedAt?: string inviteNote?: string [k: string]: unknown } diff --git a/packages/bsky/src/api/com/atproto/admin/util.ts b/packages/bsky/src/api/com/atproto/admin/util.ts index 6217e71eb0f..7dfd10cce5c 100644 --- a/packages/bsky/src/api/com/atproto/admin/util.ts +++ b/packages/bsky/src/api/com/atproto/admin/util.ts @@ -32,6 +32,7 @@ export const addAccountInfoToRepoViewDetail = ( invitesDisabled: accountInfo.invitesDisabled, inviteNote: accountInfo.inviteNote, invites: accountInfo.invites, + emailConfirmedAt: accountInfo.emailConfirmedAt, } } diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index f17885819a0..404526e7936 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -434,6 +434,10 @@ export const schemaDict = { inviteNote: { type: 'string', }, + emailConfirmedAt: { + type: 'string', + format: 'datetime', + }, }, }, accountView: { @@ -469,6 +473,10 @@ export const schemaDict = { invitesDisabled: { type: 'boolean', }, + emailConfirmedAt: { + type: 'string', + format: 'datetime', + }, inviteNote: { type: 'string', }, diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts index ea463368f8e..7c74f6b8b98 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts @@ -241,6 +241,7 @@ export interface RepoViewDetail { invites?: ComAtprotoServerDefs.InviteCode[] invitesDisabled?: boolean inviteNote?: string + emailConfirmedAt?: string [k: string]: unknown } @@ -264,6 +265,7 @@ export interface AccountView { invitedBy?: ComAtprotoServerDefs.InviteCode invites?: ComAtprotoServerDefs.InviteCode[] invitesDisabled?: boolean + emailConfirmedAt?: string inviteNote?: string [k: string]: unknown } diff --git a/packages/bsky/tests/admin/get-repo.test.ts b/packages/bsky/tests/admin/get-repo.test.ts index 3c1e909a4ab..9b4f6690ccd 100644 --- a/packages/bsky/tests/admin/get-repo.test.ts +++ b/packages/bsky/tests/admin/get-repo.test.ts @@ -91,6 +91,39 @@ describe('admin get repo view', () => { expect(triage).toEqual({ ...admin, email: undefined }) }) + it('includes emailConfirmedAt timestamp', async () => { + const { data: beforeEmailVerification } = + await agent.api.com.atproto.admin.getRepo( + { did: sc.dids.bob }, + { headers: network.pds.adminAuthHeaders() }, + ) + + expect(beforeEmailVerification.emailConfirmedAt).toBeUndefined() + const timestampBeforeVerification = Date.now() + const bobsAccount = sc.accounts[sc.dids.bob] + const verificationToken = await network.pds.ctx.services + .account(network.pds.ctx.db) + .createEmailToken(sc.dids.bob, 'confirm_email') + await agent.api.com.atproto.server.confirmEmail( + { email: bobsAccount.email, token: verificationToken }, + { + encoding: 'application/json', + + headers: sc.getHeaders(sc.dids.bob), + }, + ) + const { data: afterEmailVerification } = + await agent.api.com.atproto.admin.getRepo( + { did: sc.dids.bob }, + { headers: network.pds.adminAuthHeaders() }, + ) + + expect(afterEmailVerification.emailConfirmedAt).toBeTruthy() + expect( + new Date(afterEmailVerification.emailConfirmedAt as string).getTime(), + ).toBeGreaterThan(timestampBeforeVerification) + }) + it('fails when repo does not exist.', async () => { const promise = agent.api.com.atproto.admin.getRepo( { did: 'did:plc:doesnotexist' }, diff --git a/packages/pds/src/api/com/atproto/admin/util.ts b/packages/pds/src/api/com/atproto/admin/util.ts index f8bab4460a5..ee862236b4f 100644 --- a/packages/pds/src/api/com/atproto/admin/util.ts +++ b/packages/pds/src/api/com/atproto/admin/util.ts @@ -1,8 +1,4 @@ import express from 'express' -import { - RepoView, - RepoViewDetail, -} from '../../../../lexicon/types/com/atproto/admin/defs' // Output designed to passed as second arg to AtpAgent methods. // The encoding field here is a quirk of the AtpAgent. @@ -26,16 +22,3 @@ export function authPassthru(req: express.Request, withEncoding?: boolean) { } } } - -// @NOTE mutates. -// merges-in details that the pds knows about the repo. -export function mergeRepoViewPdsDetails( - other: T, - pds: T, -) { - other.email ??= pds.email - other.invites ??= pds.invites - other.invitedBy ??= pds.invitedBy - other.invitesDisabled ??= pds.invitesDisabled - return other -} diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index f17885819a0..404526e7936 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -434,6 +434,10 @@ export const schemaDict = { inviteNote: { type: 'string', }, + emailConfirmedAt: { + type: 'string', + format: 'datetime', + }, }, }, accountView: { @@ -469,6 +473,10 @@ export const schemaDict = { invitesDisabled: { type: 'boolean', }, + emailConfirmedAt: { + type: 'string', + format: 'datetime', + }, inviteNote: { type: 'string', }, diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts index ea463368f8e..7c74f6b8b98 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts @@ -241,6 +241,7 @@ export interface RepoViewDetail { invites?: ComAtprotoServerDefs.InviteCode[] invitesDisabled?: boolean inviteNote?: string + emailConfirmedAt?: string [k: string]: unknown } @@ -264,6 +265,7 @@ export interface AccountView { invitedBy?: ComAtprotoServerDefs.InviteCode invites?: ComAtprotoServerDefs.InviteCode[] invitesDisabled?: boolean + emailConfirmedAt?: string inviteNote?: string [k: string]: unknown } diff --git a/packages/pds/src/services/account/index.ts b/packages/pds/src/services/account/index.ts index 73518199c32..3893be03209 100644 --- a/packages/pds/src/services/account/index.ts +++ b/packages/pds/src/services/account/index.ts @@ -387,6 +387,7 @@ export class AccountService { 'did_handle.did', 'did_handle.handle', 'user_account.email', + 'user_account.emailConfirmedAt', 'user_account.invitesDisabled', 'user_account.inviteNote', 'user_account.createdAt as indexedAt', @@ -405,6 +406,7 @@ export class AccountService { handle: account?.handle ?? INVALID_HANDLE, invitesDisabled: account.invitesDisabled === 1, inviteNote: account.inviteNote ?? undefined, + emailConfirmedAt: account.emailConfirmedAt ?? undefined, invites, invitedBy: invitedBy[did], } From 74b7fdf7542b773fa4b5dea6ee193165d9627831 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Tue, 14 Nov 2023 18:14:08 -0600 Subject: [PATCH 27/59] Randomize suggestions (#1844) * randomize suggestions * fix snap * cursor fix * pr feedback --- .../src/api/app/bsky/actor/getSuggestions.ts | 66 +++++++++++++------ packages/bsky/tests/views/suggestions.test.ts | 22 +++++-- .../proxied/__snapshots__/views.test.ts.snap | 2 +- 3 files changed, 61 insertions(+), 29 deletions(-) diff --git a/packages/bsky/src/api/app/bsky/actor/getSuggestions.ts b/packages/bsky/src/api/app/bsky/actor/getSuggestions.ts index 18ab99debe2..f68ba68eb66 100644 --- a/packages/bsky/src/api/app/bsky/actor/getSuggestions.ts +++ b/packages/bsky/src/api/app/bsky/actor/getSuggestions.ts @@ -42,12 +42,12 @@ const skeleton = async ( ctx: Context, ): Promise => { const { db } = ctx - const { limit, cursor, viewer } = params + const { viewer } = params + const alreadyIncluded = parseCursor(params.cursor) const { ref } = db.db.dynamic - let suggestionsQb = db.db + const suggestions = await db.db .selectFrom('suggested_follow') .innerJoin('actor', 'actor.did', 'suggested_follow.did') - .innerJoin('profile_agg', 'profile_agg.did', 'actor.did') .where(notSoftDeletedClause(ref('actor'))) .where('suggested_follow.did', '!=', viewer ?? '') .whereNotExists((qb) => @@ -57,27 +57,30 @@ const skeleton = async ( .where('creator', '=', viewer ?? '') .whereRef('subjectDid', '=', ref('actor.did')), ) + .if(alreadyIncluded.length > 0, (qb) => + qb.where('suggested_follow.order', 'not in', alreadyIncluded), + ) .selectAll() - .select('profile_agg.postsCount as postsCount') - .limit(limit) .orderBy('suggested_follow.order', 'asc') + .execute() - if (cursor) { - const cursorRow = await db.db - .selectFrom('suggested_follow') - .where('did', '=', cursor) - .selectAll() - .executeTakeFirst() - if (cursorRow) { - suggestionsQb = suggestionsQb.where( - 'suggested_follow.order', - '>', - cursorRow.order, - ) - } - } - const suggestions = await suggestionsQb.execute() - return { params, suggestions, cursor: suggestions.at(-1)?.did } + // always include first two + const firstTwo = suggestions.filter( + (row) => row.order === 1 || row.order === 2, + ) + const rest = suggestions.filter((row) => row.order !== 1 && row.order !== 2) + const limited = firstTwo.concat(shuffle(rest)).slice(0, params.limit) + + // if the result set ends up getting larger, consider using a seed included in the cursor for for the randomized shuffle + const cursor = + limited.length > 0 + ? limited + .map((row) => row.order.toString()) + .concat(alreadyIncluded.map((id) => id.toString())) + .join(':') + : undefined + + return { params, suggestions: limited, cursor } } const hydration = async (state: SkeletonState, ctx: Context) => { @@ -110,6 +113,27 @@ const presentation = (state: HydrationState) => { return { actors: suggestedActors, cursor } } +const parseCursor = (cursor?: string): number[] => { + if (!cursor) { + return [] + } + try { + return cursor + .split(':') + .map((id) => parseInt(id, 10)) + .filter((id) => !isNaN(id)) + } catch { + return [] + } +} + +const shuffle = (arr: T[]): T[] => { + return arr + .map((value) => ({ value, sort: Math.random() })) + .sort((a, b) => a.sort - b.sort) + .map(({ value }) => value) +} + type Context = { db: Database actorService: ActorService diff --git a/packages/bsky/tests/views/suggestions.test.ts b/packages/bsky/tests/views/suggestions.test.ts index 2dcadf9e6ad..4253f528b13 100644 --- a/packages/bsky/tests/views/suggestions.test.ts +++ b/packages/bsky/tests/views/suggestions.test.ts @@ -19,10 +19,12 @@ describe('pds user search views', () => { await network.bsky.processAll() const suggestions = [ - { did: sc.dids.bob, order: 1 }, - { did: sc.dids.carol, order: 2 }, - { did: sc.dids.dan, order: 3 }, + { did: sc.dids.alice, order: 1 }, + { did: sc.dids.bob, order: 2 }, + { did: sc.dids.carol, order: 3 }, + { did: sc.dids.dan, order: 4 }, ] + await network.bsky.ctx.db .getPrimary() .db.insertInto('suggested_follow') @@ -63,16 +65,22 @@ describe('pds user search views', () => { { limit: 1 }, { headers: await network.serviceHeaders(sc.dids.carol) }, ) + expect(result1.data.actors.length).toBe(1) + expect(result1.data.actors[0].handle).toEqual('bob.test') + const result2 = await agent.api.app.bsky.actor.getSuggestions( { limit: 1, cursor: result1.data.cursor }, { headers: await network.serviceHeaders(sc.dids.carol) }, ) - - expect(result1.data.actors.length).toBe(1) - expect(result1.data.actors[0].handle).toEqual('bob.test') - expect(result2.data.actors.length).toBe(1) expect(result2.data.actors[0].handle).toEqual('dan.test') + + const result3 = await agent.api.app.bsky.actor.getSuggestions( + { limit: 1, cursor: result2.data.cursor }, + { headers: await network.serviceHeaders(sc.dids.carol) }, + ) + expect(result3.data.actors.length).toBe(0) + expect(result3.data.cursor).toBeUndefined() }) it('fetches suggestions unauthed', async () => { diff --git a/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap b/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap index fcf1063954c..0dbe9b5498d 100644 --- a/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap +++ b/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap @@ -114,7 +114,7 @@ Object { }, }, ], - "cursor": "user(2)", + "cursor": "1:3", } `; From a434d586dd91422a53416d9251b4f75805206d00 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Tue, 14 Nov 2023 18:14:22 -0600 Subject: [PATCH 28/59] Add searchPosts api to appview (#1845) * proxy search posts to search service * add search posts proxy * tidy * add type annotations --- .../bsky/src/api/app/bsky/feed/searchPosts.ts | 123 ++++++++++++++++++ packages/bsky/src/api/index.ts | 2 + packages/pds/src/api/app/bsky/feed/index.ts | 2 + .../pds/src/api/app/bsky/feed/searchPosts.ts | 19 +++ 4 files changed, 146 insertions(+) create mode 100644 packages/bsky/src/api/app/bsky/feed/searchPosts.ts create mode 100644 packages/pds/src/api/app/bsky/feed/searchPosts.ts diff --git a/packages/bsky/src/api/app/bsky/feed/searchPosts.ts b/packages/bsky/src/api/app/bsky/feed/searchPosts.ts new file mode 100644 index 00000000000..2edc9e2f138 --- /dev/null +++ b/packages/bsky/src/api/app/bsky/feed/searchPosts.ts @@ -0,0 +1,123 @@ +import AppContext from '../../../../context' +import { Server } from '../../../../lexicon' +import { InvalidRequestError } from '@atproto/xrpc-server' +import AtpAgent from '@atproto/api' +import { AtUri } from '@atproto/syntax' +import { mapDefined } from '@atproto/common' +import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/searchPosts' +import { Database } from '../../../../db' +import { FeedHydrationState, FeedService } from '../../../../services/feed' +import { ActorService } from '../../../../services/actor' +import { createPipeline } from '../../../../pipeline' + +export default function (server: Server, ctx: AppContext) { + const searchPosts = createPipeline( + skeleton, + hydration, + noBlocks, + presentation, + ) + server.app.bsky.feed.searchPosts({ + auth: ctx.authOptionalVerifier, + handler: async ({ auth, params }) => { + const viewer = auth.credentials.did + const db = ctx.db.getReplica('search') + const feedService = ctx.services.feed(db) + const actorService = ctx.services.actor(db) + const searchAgent = ctx.searchAgent + if (!searchAgent) { + throw new InvalidRequestError('Search not available') + } + + const results = await searchPosts( + { ...params, viewer }, + { db, feedService, actorService, searchAgent }, + ) + + return { + encoding: 'application/json', + body: results, + } + }, + }) +} + +const skeleton = async ( + params: Params, + ctx: Context, +): Promise => { + const res = await ctx.searchAgent.api.app.bsky.unspecced.searchPostsSkeleton( + params, + ) + return { + params, + postUris: res.data.posts.map((a) => a.uri), + cursor: res.data.cursor, + } +} + +const hydration = async ( + state: SkeletonState, + ctx: Context, +): Promise => { + const { feedService } = ctx + const { params, postUris } = state + const uris = new Set(postUris) + const dids = new Set(postUris.map((uri) => new AtUri(uri).hostname)) + const hydrated = await feedService.feedHydration({ + uris, + dids, + viewer: params.viewer, + }) + return { ...state, ...hydrated } +} + +const noBlocks = (state: HydrationState): HydrationState => { + const { viewer } = state.params + state.postUris = state.postUris.filter((uri) => { + const post = state.posts[uri] + if (!viewer || !post) return true + return !state.bam.block([viewer, post.creator]) + }) + return state +} + +const presentation = (state: HydrationState, ctx: Context) => { + const { feedService, actorService } = ctx + const { postUris, profiles, params } = state + const actors = actorService.views.profileBasicPresentation( + Object.keys(profiles), + state, + { viewer: params.viewer }, + ) + + const postViews = mapDefined(postUris, (uri) => + feedService.views.formatPostView( + uri, + actors, + state.posts, + state.threadgates, + state.embeds, + state.labels, + state.lists, + ), + ) + return { posts: postViews } +} + +type Context = { + db: Database + feedService: FeedService + actorService: ActorService + searchAgent: AtpAgent +} + +type Params = QueryParams & { viewer: string | null } + +type SkeletonState = { + params: Params + postUris: string[] + cursor?: string +} + +type HydrationState = SkeletonState & FeedHydrationState diff --git a/packages/bsky/src/api/index.ts b/packages/bsky/src/api/index.ts index 3768ed4da0b..cf2121b7792 100644 --- a/packages/bsky/src/api/index.ts +++ b/packages/bsky/src/api/index.ts @@ -13,6 +13,7 @@ import getLikes from './app/bsky/feed/getLikes' import getListFeed from './app/bsky/feed/getListFeed' import getPostThread from './app/bsky/feed/getPostThread' import getPosts from './app/bsky/feed/getPosts' +import searchPosts from './app/bsky/feed/searchPosts' import getActorLikes from './app/bsky/feed/getActorLikes' import getProfile from './app/bsky/actor/getProfile' import getProfiles from './app/bsky/actor/getProfiles' @@ -74,6 +75,7 @@ export default function (server: Server, ctx: AppContext) { getListFeed(server, ctx) getPostThread(server, ctx) getPosts(server, ctx) + searchPosts(server, ctx) getActorLikes(server, ctx) getProfile(server, ctx) getProfiles(server, ctx) diff --git a/packages/pds/src/api/app/bsky/feed/index.ts b/packages/pds/src/api/app/bsky/feed/index.ts index 8c4cfaa8b5f..026ce86f612 100644 --- a/packages/pds/src/api/app/bsky/feed/index.ts +++ b/packages/pds/src/api/app/bsky/feed/index.ts @@ -13,6 +13,7 @@ import getPostThread from './getPostThread' import getRepostedBy from './getRepostedBy' import getSuggestedFeeds from './getSuggestedFeeds' import getTimeline from './getTimeline' +import searchPosts from './searchPosts' export default function (server: Server, ctx: AppContext) { getActorFeeds(server, ctx) @@ -28,4 +29,5 @@ export default function (server: Server, ctx: AppContext) { getRepostedBy(server, ctx) getSuggestedFeeds(server, ctx) getTimeline(server, ctx) + searchPosts(server, ctx) } diff --git a/packages/pds/src/api/app/bsky/feed/searchPosts.ts b/packages/pds/src/api/app/bsky/feed/searchPosts.ts new file mode 100644 index 00000000000..85384751ea1 --- /dev/null +++ b/packages/pds/src/api/app/bsky/feed/searchPosts.ts @@ -0,0 +1,19 @@ +import { Server } from '../../../../lexicon' +import AppContext from '../../../../context' + +export default function (server: Server, ctx: AppContext) { + server.app.bsky.feed.searchPosts({ + auth: ctx.authVerifier.access, + handler: async ({ params, auth }) => { + const requester = auth.credentials.did + const res = await ctx.appViewAgent.api.app.bsky.feed.searchPosts( + params, + await ctx.serviceAuthHeaders(requester), + ) + return { + encoding: 'application/json', + body: res.data, + } + }, + }) +} From 681e35d4ef0f42171ab101dbf1caf81a459556c5 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 15 Nov 2023 03:37:15 +0000 Subject: [PATCH 29/59] Add a note to README about depending on jq and docker (#1854) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5a0a18ce853..a54fa91fe5f 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ make build make test # run a local PDS and AppView with fake test accounts and data +# (this requires a global installation of `jq` and `docker`) make run-dev-env # show all other commands From 599cb449b54ae0d72db1492852d4d63b3fbcf3f9 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Tue, 14 Nov 2023 22:18:47 -0600 Subject: [PATCH 30/59] Bugfix: don't pass the viewer as a param to searchPosts skeleton (#1855) dont pass the viewer as a param to searchPosts skeleton --- packages/bsky/src/api/app/bsky/feed/searchPosts.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/bsky/src/api/app/bsky/feed/searchPosts.ts b/packages/bsky/src/api/app/bsky/feed/searchPosts.ts index 2edc9e2f138..1195d319e82 100644 --- a/packages/bsky/src/api/app/bsky/feed/searchPosts.ts +++ b/packages/bsky/src/api/app/bsky/feed/searchPosts.ts @@ -46,9 +46,11 @@ const skeleton = async ( params: Params, ctx: Context, ): Promise => { - const res = await ctx.searchAgent.api.app.bsky.unspecced.searchPostsSkeleton( - params, - ) + const res = await ctx.searchAgent.api.app.bsky.unspecced.searchPostsSkeleton({ + q: params.q, + cursor: params.cursor, + limit: params.limit, + }) return { params, postUris: res.data.posts.map((a) => a.uri), From 1ef6da191b6409eb5c2e87aa534392522a12e067 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Wed, 15 Nov 2023 15:31:14 -0600 Subject: [PATCH 31/59] Pass along hitsTotal and cursor from searchPostsSkeleton (#1857) pass along hitsTotal and cursor from searchPostsSkeleton --- packages/bsky/src/api/app/bsky/feed/searchPosts.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/bsky/src/api/app/bsky/feed/searchPosts.ts b/packages/bsky/src/api/app/bsky/feed/searchPosts.ts index 1195d319e82..718bfa7afa4 100644 --- a/packages/bsky/src/api/app/bsky/feed/searchPosts.ts +++ b/packages/bsky/src/api/app/bsky/feed/searchPosts.ts @@ -55,6 +55,7 @@ const skeleton = async ( params, postUris: res.data.posts.map((a) => a.uri), cursor: res.data.cursor, + hitsTotal: res.data.hitsTotal, } } @@ -104,7 +105,7 @@ const presentation = (state: HydrationState, ctx: Context) => { state.lists, ), ) - return { posts: postViews } + return { posts: postViews, cursor: state.cursor, hitsTotal: state.hitsTotal } } type Context = { @@ -119,6 +120,7 @@ type Params = QueryParams & { viewer: string | null } type SkeletonState = { params: Params postUris: string[] + hitsTotal?: number cursor?: string } From b05130db6897013d7ef5483da6ecece1b1140932 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Thu, 16 Nov 2023 11:07:52 -0600 Subject: [PATCH 32/59] Add temp.fetchLabels method (#1858) * add temp.fetchLabels route * update description --- lexicons/com/atproto/temp/fetchLabels.json | 35 ++++++++++++++ packages/api/src/client/index.ts | 23 +++++++++ packages/api/src/client/lexicons.ts | 42 +++++++++++++++++ .../types/com/atproto/temp/fetchLabels.ts | 37 +++++++++++++++ .../src/api/com/atproto/temp/fetchLabels.ts | 25 ++++++++++ packages/bsky/src/api/index.ts | 2 + packages/bsky/src/lexicon/index.ts | 22 +++++++++ packages/bsky/src/lexicon/lexicons.ts | 42 +++++++++++++++++ .../types/com/atproto/temp/fetchLabels.ts | 47 +++++++++++++++++++ packages/pds/src/lexicon/index.ts | 22 +++++++++ packages/pds/src/lexicon/lexicons.ts | 42 +++++++++++++++++ .../types/com/atproto/temp/fetchLabels.ts | 47 +++++++++++++++++++ 12 files changed, 386 insertions(+) create mode 100644 lexicons/com/atproto/temp/fetchLabels.json create mode 100644 packages/api/src/client/types/com/atproto/temp/fetchLabels.ts create mode 100644 packages/bsky/src/api/com/atproto/temp/fetchLabels.ts create mode 100644 packages/bsky/src/lexicon/types/com/atproto/temp/fetchLabels.ts create mode 100644 packages/pds/src/lexicon/types/com/atproto/temp/fetchLabels.ts diff --git a/lexicons/com/atproto/temp/fetchLabels.json b/lexicons/com/atproto/temp/fetchLabels.json new file mode 100644 index 00000000000..14e392fd5e7 --- /dev/null +++ b/lexicons/com/atproto/temp/fetchLabels.json @@ -0,0 +1,35 @@ +{ + "lexicon": 1, + "id": "com.atproto.temp.fetchLabels", + "defs": { + "main": { + "type": "query", + "description": "Fetch all labels from a labeler created after a certain date.", + "parameters": { + "type": "params", + "properties": { + "since": { "type": "integer" }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 250, + "default": 50 + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["labels"], + "properties": { + "labels": { + "type": "array", + "items": { "type": "ref", "ref": "com.atproto.label.defs#label" } + } + } + } + } + } + } +} diff --git a/packages/api/src/client/index.ts b/packages/api/src/client/index.ts index 3fd82222639..519ec284dc4 100644 --- a/packages/api/src/client/index.ts +++ b/packages/api/src/client/index.ts @@ -78,6 +78,7 @@ import * as ComAtprotoSyncListRepos from './types/com/atproto/sync/listRepos' import * as ComAtprotoSyncNotifyOfUpdate from './types/com/atproto/sync/notifyOfUpdate' import * as ComAtprotoSyncRequestCrawl from './types/com/atproto/sync/requestCrawl' import * as ComAtprotoSyncSubscribeRepos from './types/com/atproto/sync/subscribeRepos' +import * as ComAtprotoTempFetchLabels from './types/com/atproto/temp/fetchLabels' import * as AppBskyActorDefs from './types/app/bsky/actor/defs' import * as AppBskyActorGetPreferences from './types/app/bsky/actor/getPreferences' import * as AppBskyActorGetProfile from './types/app/bsky/actor/getProfile' @@ -215,6 +216,7 @@ export * as ComAtprotoSyncListRepos from './types/com/atproto/sync/listRepos' export * as ComAtprotoSyncNotifyOfUpdate from './types/com/atproto/sync/notifyOfUpdate' export * as ComAtprotoSyncRequestCrawl from './types/com/atproto/sync/requestCrawl' export * as ComAtprotoSyncSubscribeRepos from './types/com/atproto/sync/subscribeRepos' +export * as ComAtprotoTempFetchLabels from './types/com/atproto/temp/fetchLabels' export * as AppBskyActorDefs from './types/app/bsky/actor/defs' export * as AppBskyActorGetPreferences from './types/app/bsky/actor/getPreferences' export * as AppBskyActorGetProfile from './types/app/bsky/actor/getProfile' @@ -349,6 +351,7 @@ export class AtprotoNS { repo: RepoNS server: ServerNS sync: SyncNS + temp: TempNS constructor(service: AtpServiceClient) { this._service = service @@ -359,6 +362,7 @@ export class AtprotoNS { this.repo = new RepoNS(service) this.server = new ServerNS(service) this.sync = new SyncNS(service) + this.temp = new TempNS(service) } } @@ -1122,6 +1126,25 @@ export class SyncNS { } } +export class TempNS { + _service: AtpServiceClient + + constructor(service: AtpServiceClient) { + this._service = service + } + + fetchLabels( + params?: ComAtprotoTempFetchLabels.QueryParams, + opts?: ComAtprotoTempFetchLabels.CallOptions, + ): Promise { + return this._service.xrpc + .call('com.atproto.temp.fetchLabels', params, undefined, opts) + .catch((e) => { + throw ComAtprotoTempFetchLabels.toKnownErr(e) + }) + } +} + export class AppNS { _service: AtpServiceClient bsky: BskyNS diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index 404526e7936..545d8e9d43d 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -3895,6 +3895,47 @@ export const schemaDict = { }, }, }, + ComAtprotoTempFetchLabels: { + lexicon: 1, + id: 'com.atproto.temp.fetchLabels', + defs: { + main: { + type: 'query', + description: + 'Fetch all labels from a labeler created after a certain date.', + parameters: { + type: 'params', + properties: { + since: { + type: 'integer', + }, + limit: { + type: 'integer', + minimum: 1, + maximum: 250, + default: 50, + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['labels'], + properties: { + labels: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.label.defs#label', + }, + }, + }, + }, + }, + }, + }, + }, AppBskyActorDefs: { lexicon: 1, id: 'app.bsky.actor.defs', @@ -7659,6 +7700,7 @@ export const ids = { ComAtprotoSyncNotifyOfUpdate: 'com.atproto.sync.notifyOfUpdate', ComAtprotoSyncRequestCrawl: 'com.atproto.sync.requestCrawl', ComAtprotoSyncSubscribeRepos: 'com.atproto.sync.subscribeRepos', + ComAtprotoTempFetchLabels: 'com.atproto.temp.fetchLabels', AppBskyActorDefs: 'app.bsky.actor.defs', AppBskyActorGetPreferences: 'app.bsky.actor.getPreferences', AppBskyActorGetProfile: 'app.bsky.actor.getProfile', diff --git a/packages/api/src/client/types/com/atproto/temp/fetchLabels.ts b/packages/api/src/client/types/com/atproto/temp/fetchLabels.ts new file mode 100644 index 00000000000..408d39cdab3 --- /dev/null +++ b/packages/api/src/client/types/com/atproto/temp/fetchLabels.ts @@ -0,0 +1,37 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import { Headers, XRPCError } from '@atproto/xrpc' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { isObj, hasProp } from '../../../../util' +import { lexicons } from '../../../../lexicons' +import { CID } from 'multiformats/cid' +import * as ComAtprotoLabelDefs from '../label/defs' + +export interface QueryParams { + since?: number + limit?: number +} + +export type InputSchema = undefined + +export interface OutputSchema { + labels: ComAtprotoLabelDefs.Label[] + [k: string]: unknown +} + +export interface CallOptions { + headers?: Headers +} + +export interface Response { + success: boolean + headers: Headers + data: OutputSchema +} + +export function toKnownErr(e: any) { + if (e instanceof XRPCError) { + } + return e +} diff --git a/packages/bsky/src/api/com/atproto/temp/fetchLabels.ts b/packages/bsky/src/api/com/atproto/temp/fetchLabels.ts new file mode 100644 index 00000000000..77addd24cc1 --- /dev/null +++ b/packages/bsky/src/api/com/atproto/temp/fetchLabels.ts @@ -0,0 +1,25 @@ +import { Server } from '../../../../lexicon' +import AppContext from '../../../../context' + +export default function (server: Server, ctx: AppContext) { + server.com.atproto.temp.fetchLabels(async ({ params }) => { + const { limit } = params + const db = ctx.db.getReplica() + const since = + params.since !== undefined ? new Date(params.since).toISOString() : '' + const labels = await db.db + .selectFrom('label') + .selectAll() + .orderBy('label.cts', 'asc') + .where('cts', '>', since) + .limit(limit) + .execute() + + return { + encoding: 'application/json', + body: { + labels, + }, + } + }) +} diff --git a/packages/bsky/src/api/index.ts b/packages/bsky/src/api/index.ts index cf2121b7792..786fdd00e5d 100644 --- a/packages/bsky/src/api/index.ts +++ b/packages/bsky/src/api/index.ts @@ -53,6 +53,7 @@ import getModerationReport from './com/atproto/admin/getModerationReport' import getModerationReports from './com/atproto/admin/getModerationReports' import resolveHandle from './com/atproto/identity/resolveHandle' import getRecord from './com/atproto/repo/getRecord' +import fetchLabels from './com/atproto/temp/fetchLabels' export * as health from './health' @@ -116,5 +117,6 @@ export default function (server: Server, ctx: AppContext) { getModerationReports(server, ctx) resolveHandle(server, ctx) getRecord(server, ctx) + fetchLabels(server, ctx) return server } diff --git a/packages/bsky/src/lexicon/index.ts b/packages/bsky/src/lexicon/index.ts index bf69ebafa68..6474220edad 100644 --- a/packages/bsky/src/lexicon/index.ts +++ b/packages/bsky/src/lexicon/index.ts @@ -75,6 +75,7 @@ import * as ComAtprotoSyncListRepos from './types/com/atproto/sync/listRepos' import * as ComAtprotoSyncNotifyOfUpdate from './types/com/atproto/sync/notifyOfUpdate' import * as ComAtprotoSyncRequestCrawl from './types/com/atproto/sync/requestCrawl' import * as ComAtprotoSyncSubscribeRepos from './types/com/atproto/sync/subscribeRepos' +import * as ComAtprotoTempFetchLabels from './types/com/atproto/temp/fetchLabels' import * as AppBskyActorGetPreferences from './types/app/bsky/actor/getPreferences' import * as AppBskyActorGetProfile from './types/app/bsky/actor/getProfile' import * as AppBskyActorGetProfiles from './types/app/bsky/actor/getProfiles' @@ -175,6 +176,7 @@ export class AtprotoNS { repo: RepoNS server: ServerNS sync: SyncNS + temp: TempNS constructor(server: Server) { this._server = server @@ -185,6 +187,7 @@ export class AtprotoNS { this.repo = new RepoNS(server) this.server = new ServerNS(server) this.sync = new SyncNS(server) + this.temp = new TempNS(server) } } @@ -970,6 +973,25 @@ export class SyncNS { } } +export class TempNS { + _server: Server + + constructor(server: Server) { + this._server = server + } + + fetchLabels( + cfg: ConfigOf< + AV, + ComAtprotoTempFetchLabels.Handler>, + ComAtprotoTempFetchLabels.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.temp.fetchLabels' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } +} + export class AppNS { _server: Server bsky: BskyNS diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index 404526e7936..545d8e9d43d 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -3895,6 +3895,47 @@ export const schemaDict = { }, }, }, + ComAtprotoTempFetchLabels: { + lexicon: 1, + id: 'com.atproto.temp.fetchLabels', + defs: { + main: { + type: 'query', + description: + 'Fetch all labels from a labeler created after a certain date.', + parameters: { + type: 'params', + properties: { + since: { + type: 'integer', + }, + limit: { + type: 'integer', + minimum: 1, + maximum: 250, + default: 50, + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['labels'], + properties: { + labels: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.label.defs#label', + }, + }, + }, + }, + }, + }, + }, + }, AppBskyActorDefs: { lexicon: 1, id: 'app.bsky.actor.defs', @@ -7659,6 +7700,7 @@ export const ids = { ComAtprotoSyncNotifyOfUpdate: 'com.atproto.sync.notifyOfUpdate', ComAtprotoSyncRequestCrawl: 'com.atproto.sync.requestCrawl', ComAtprotoSyncSubscribeRepos: 'com.atproto.sync.subscribeRepos', + ComAtprotoTempFetchLabels: 'com.atproto.temp.fetchLabels', AppBskyActorDefs: 'app.bsky.actor.defs', AppBskyActorGetPreferences: 'app.bsky.actor.getPreferences', AppBskyActorGetProfile: 'app.bsky.actor.getProfile', diff --git a/packages/bsky/src/lexicon/types/com/atproto/temp/fetchLabels.ts b/packages/bsky/src/lexicon/types/com/atproto/temp/fetchLabels.ts new file mode 100644 index 00000000000..39341fd3a0e --- /dev/null +++ b/packages/bsky/src/lexicon/types/com/atproto/temp/fetchLabels.ts @@ -0,0 +1,47 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' +import * as ComAtprotoLabelDefs from '../label/defs' + +export interface QueryParams { + since?: number + limit: number +} + +export type InputSchema = undefined + +export interface OutputSchema { + labels: ComAtprotoLabelDefs.Label[] + [k: string]: unknown +} + +export type HandlerInput = undefined + +export interface HandlerSuccess { + encoding: 'application/json' + body: OutputSchema + headers?: { [key: string]: string } +} + +export interface HandlerError { + status: number + message?: string +} + +export type HandlerOutput = HandlerError | HandlerSuccess +export type HandlerReqCtx = { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +} +export type Handler = ( + ctx: HandlerReqCtx, +) => Promise | HandlerOutput diff --git a/packages/pds/src/lexicon/index.ts b/packages/pds/src/lexicon/index.ts index bf69ebafa68..6474220edad 100644 --- a/packages/pds/src/lexicon/index.ts +++ b/packages/pds/src/lexicon/index.ts @@ -75,6 +75,7 @@ import * as ComAtprotoSyncListRepos from './types/com/atproto/sync/listRepos' import * as ComAtprotoSyncNotifyOfUpdate from './types/com/atproto/sync/notifyOfUpdate' import * as ComAtprotoSyncRequestCrawl from './types/com/atproto/sync/requestCrawl' import * as ComAtprotoSyncSubscribeRepos from './types/com/atproto/sync/subscribeRepos' +import * as ComAtprotoTempFetchLabels from './types/com/atproto/temp/fetchLabels' import * as AppBskyActorGetPreferences from './types/app/bsky/actor/getPreferences' import * as AppBskyActorGetProfile from './types/app/bsky/actor/getProfile' import * as AppBskyActorGetProfiles from './types/app/bsky/actor/getProfiles' @@ -175,6 +176,7 @@ export class AtprotoNS { repo: RepoNS server: ServerNS sync: SyncNS + temp: TempNS constructor(server: Server) { this._server = server @@ -185,6 +187,7 @@ export class AtprotoNS { this.repo = new RepoNS(server) this.server = new ServerNS(server) this.sync = new SyncNS(server) + this.temp = new TempNS(server) } } @@ -970,6 +973,25 @@ export class SyncNS { } } +export class TempNS { + _server: Server + + constructor(server: Server) { + this._server = server + } + + fetchLabels( + cfg: ConfigOf< + AV, + ComAtprotoTempFetchLabels.Handler>, + ComAtprotoTempFetchLabels.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.temp.fetchLabels' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } +} + export class AppNS { _server: Server bsky: BskyNS diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index 404526e7936..545d8e9d43d 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -3895,6 +3895,47 @@ export const schemaDict = { }, }, }, + ComAtprotoTempFetchLabels: { + lexicon: 1, + id: 'com.atproto.temp.fetchLabels', + defs: { + main: { + type: 'query', + description: + 'Fetch all labels from a labeler created after a certain date.', + parameters: { + type: 'params', + properties: { + since: { + type: 'integer', + }, + limit: { + type: 'integer', + minimum: 1, + maximum: 250, + default: 50, + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['labels'], + properties: { + labels: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.label.defs#label', + }, + }, + }, + }, + }, + }, + }, + }, AppBskyActorDefs: { lexicon: 1, id: 'app.bsky.actor.defs', @@ -7659,6 +7700,7 @@ export const ids = { ComAtprotoSyncNotifyOfUpdate: 'com.atproto.sync.notifyOfUpdate', ComAtprotoSyncRequestCrawl: 'com.atproto.sync.requestCrawl', ComAtprotoSyncSubscribeRepos: 'com.atproto.sync.subscribeRepos', + ComAtprotoTempFetchLabels: 'com.atproto.temp.fetchLabels', AppBskyActorDefs: 'app.bsky.actor.defs', AppBskyActorGetPreferences: 'app.bsky.actor.getPreferences', AppBskyActorGetProfile: 'app.bsky.actor.getProfile', diff --git a/packages/pds/src/lexicon/types/com/atproto/temp/fetchLabels.ts b/packages/pds/src/lexicon/types/com/atproto/temp/fetchLabels.ts new file mode 100644 index 00000000000..39341fd3a0e --- /dev/null +++ b/packages/pds/src/lexicon/types/com/atproto/temp/fetchLabels.ts @@ -0,0 +1,47 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' +import * as ComAtprotoLabelDefs from '../label/defs' + +export interface QueryParams { + since?: number + limit: number +} + +export type InputSchema = undefined + +export interface OutputSchema { + labels: ComAtprotoLabelDefs.Label[] + [k: string]: unknown +} + +export type HandlerInput = undefined + +export interface HandlerSuccess { + encoding: 'application/json' + body: OutputSchema + headers?: { [key: string]: string } +} + +export interface HandlerError { + status: number + message?: string +} + +export type HandlerOutput = HandlerError | HandlerSuccess +export type HandlerReqCtx = { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +} +export type Handler = ( + ctx: HandlerReqCtx, +) => Promise | HandlerOutput From e6fb39dc8453bf95942305e8e69aa035062448cf Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Thu, 16 Nov 2023 12:27:46 -0600 Subject: [PATCH 33/59] Bugfix: fetchLabels (#1862) * fix fetch labels * tidy * update cast to parseInt --- packages/bsky/src/api/com/atproto/temp/fetchLabels.ts | 7 ++++++- packages/xrpc-server/src/util.ts | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/bsky/src/api/com/atproto/temp/fetchLabels.ts b/packages/bsky/src/api/com/atproto/temp/fetchLabels.ts index 77addd24cc1..8a6cacc2fbd 100644 --- a/packages/bsky/src/api/com/atproto/temp/fetchLabels.ts +++ b/packages/bsky/src/api/com/atproto/temp/fetchLabels.ts @@ -7,7 +7,7 @@ export default function (server: Server, ctx: AppContext) { const db = ctx.db.getReplica() const since = params.since !== undefined ? new Date(params.since).toISOString() : '' - const labels = await db.db + const labelRes = await db.db .selectFrom('label') .selectAll() .orderBy('label.cts', 'asc') @@ -15,6 +15,11 @@ export default function (server: Server, ctx: AppContext) { .limit(limit) .execute() + const labels = labelRes.map((l) => ({ + ...l, + cid: l.cid === '' ? undefined : l.cid, + })) + return { encoding: 'application/json', body: { diff --git a/packages/xrpc-server/src/util.ts b/packages/xrpc-server/src/util.ts index 730db950fbd..f306d1944aa 100644 --- a/packages/xrpc-server/src/util.ts +++ b/packages/xrpc-server/src/util.ts @@ -60,7 +60,7 @@ export function decodeQueryParam( if (type === 'float') { return Number(String(value)) } else if (type === 'integer') { - return Number(String(value)) | 0 + return parseInt(String(value), 10) || 0 } else if (type === 'boolean') { return value === 'true' } From c28754f681e383f58d180c1e67b1c0681dc8271f Mon Sep 17 00:00:00 2001 From: devin ivy Date: Thu, 16 Nov 2023 18:21:57 -0500 Subject: [PATCH 34/59] Upgrade sharp on pds and appview (#1864) upgrade sharp to 0.32.6 --- packages/bsky/package.json | 3 +- packages/dev-env/package.json | 2 +- packages/pds/package.json | 3 +- pnpm-lock.yaml | 71 +++++++++++++++++++++++------------ 4 files changed, 50 insertions(+), 29 deletions(-) diff --git a/packages/bsky/package.json b/packages/bsky/package.json index 60991c5f791..81e02626895 100644 --- a/packages/bsky/package.json +++ b/packages/bsky/package.json @@ -55,7 +55,7 @@ "pg": "^8.10.0", "pino": "^8.15.0", "pino-http": "^8.2.1", - "sharp": "^0.31.2", + "sharp": "^0.32.6", "typed-emitter": "^2.1.0", "uint8arrays": "3.0.0" }, @@ -71,7 +71,6 @@ "@types/express-serve-static-core": "^4.17.36", "@types/pg": "^8.6.6", "@types/qs": "^6.9.7", - "@types/sharp": "^0.31.0", "axios": "^0.27.2" } } diff --git a/packages/dev-env/package.json b/packages/dev-env/package.json index 3140d8ff189..90fd0824e9d 100644 --- a/packages/dev-env/package.json +++ b/packages/dev-env/package.json @@ -43,7 +43,7 @@ "express": "^4.18.2", "get-port": "^6.1.2", "multiformats": "^9.9.0", - "sharp": "^0.31.2", + "sharp": "^0.32.6", "uint8arrays": "3.0.0" }, "devDependencies": { diff --git a/packages/pds/package.json b/packages/pds/package.json index adc0ed9e508..c5bf32f9357 100644 --- a/packages/pds/package.json +++ b/packages/pds/package.json @@ -65,7 +65,7 @@ "pg": "^8.10.0", "pino": "^8.15.0", "pino-http": "^8.2.1", - "sharp": "^0.31.2", + "sharp": "^0.32.6", "typed-emitter": "^2.1.0", "uint8arrays": "3.0.0", "zod": "^3.21.4" @@ -84,7 +84,6 @@ "@types/nodemailer": "^6.4.6", "@types/pg": "^8.6.6", "@types/qs": "^6.9.7", - "@types/sharp": "^0.31.0", "axios": "^0.27.2", "ws": "^8.12.0" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index aac3ef2bcba..422867c73f4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -235,8 +235,8 @@ importers: specifier: ^8.2.1 version: 8.2.1 sharp: - specifier: ^0.31.2 - version: 0.31.2 + specifier: ^0.32.6 + version: 0.32.6 typed-emitter: specifier: ^2.1.0 version: 2.1.0 @@ -274,9 +274,6 @@ importers: '@types/qs': specifier: ^6.9.7 version: 6.9.7 - '@types/sharp': - specifier: ^0.31.0 - version: 0.31.0 axios: specifier: ^0.27.2 version: 0.27.2 @@ -389,8 +386,8 @@ importers: specifier: ^9.9.0 version: 9.9.0 sharp: - specifier: ^0.31.2 - version: 0.31.2 + specifier: ^0.32.6 + version: 0.32.6 uint8arrays: specifier: 3.0.0 version: 3.0.0 @@ -571,8 +568,8 @@ importers: specifier: ^8.2.1 version: 8.2.1 sharp: - specifier: ^0.31.2 - version: 0.31.2 + specifier: ^0.32.6 + version: 0.32.6 typed-emitter: specifier: ^2.1.0 version: 2.1.0 @@ -619,9 +616,6 @@ importers: '@types/qs': specifier: ^6.9.7 version: 6.9.7 - '@types/sharp': - specifier: ^0.31.0 - version: 0.31.0 axios: specifier: ^0.27.2 version: 0.27.2 @@ -5466,12 +5460,6 @@ packages: '@types/node': 18.17.8 dev: true - /@types/sharp@0.31.0: - resolution: {integrity: sha512-nwivOU101fYInCwdDcH/0/Ru6yIRXOpORx25ynEOc6/IakuCmjOAGpaO5VfUl4QkDtUC6hj+Z2eCQvgXOioknw==} - dependencies: - '@types/node': 18.17.8 - dev: true - /@types/stack-utils@2.0.1: resolution: {integrity: sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==} dev: true @@ -5867,6 +5855,10 @@ packages: transitivePeerDependencies: - debug + /b4a@1.6.4: + resolution: {integrity: sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw==} + dev: false + /babel-eslint@10.1.0(eslint@8.24.0): resolution: {integrity: sha512-ifWaTHQ0ce+448CYop8AdrQiBsGrnC+bMgfyKFdi6EsPLTAWG+QfyDeM6OH+FmWnKvEq5NnBMLvlBUPKQZoDSg==} engines: {node: '>=6'} @@ -7395,6 +7387,10 @@ packages: resolution: {integrity: sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==} dev: true + /fast-fifo@1.3.2: + resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + dev: false + /fast-glob@3.3.1: resolution: {integrity: sha512-kNFPyjhh5cKjrUltxs+wFx+ZkbRaxxmZ+X0ZU31SOsxCEtP9VPgtq2teZw1DebupL5GmDaNQ6yKMMVcM41iqDg==} engines: {node: '>=8.6.0'} @@ -9295,8 +9291,8 @@ packages: resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} dev: false - /node-addon-api@5.1.0: - resolution: {integrity: sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA==} + /node-addon-api@6.1.0: + resolution: {integrity: sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==} dev: false /node-fetch@2.7.0: @@ -9931,6 +9927,10 @@ packages: /queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} + /queue-tick@1.0.1: + resolution: {integrity: sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==} + dev: false + /quick-format-unescaped@4.0.4: resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} @@ -10287,18 +10287,18 @@ packages: /setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} - /sharp@0.31.2: - resolution: {integrity: sha512-DUdNVEXgS5A97cTagSLIIp8dUZ/lZtk78iNVZgHdHbx1qnQR7JAHY0BnXnwwH39Iw+VKhO08CTYhIg0p98vQ5Q==} + /sharp@0.32.6: + resolution: {integrity: sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w==} engines: {node: '>=14.15.0'} requiresBuild: true dependencies: color: 4.2.3 detect-libc: 2.0.2 - node-addon-api: 5.1.0 + node-addon-api: 6.1.0 prebuild-install: 7.1.1 semver: 7.5.4 simple-get: 4.0.1 - tar-fs: 2.1.1 + tar-fs: 3.0.4 tunnel-agent: 0.6.0 dev: false @@ -10500,6 +10500,13 @@ packages: mixme: 0.5.9 dev: true + /streamx@2.15.5: + resolution: {integrity: sha512-9thPGMkKC2GctCzyCUjME3yR03x2xNo0GPKGkRw2UMYN+gqWa9uqpyNWhmsNCutU5zHmkUum0LsCRQTXUgUCAg==} + dependencies: + fast-fifo: 1.3.2 + queue-tick: 1.0.1 + dev: false + /string-length@4.0.2: resolution: {integrity: sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==} engines: {node: '>=10'} @@ -10641,6 +10648,14 @@ packages: tar-stream: 2.2.0 dev: false + /tar-fs@3.0.4: + resolution: {integrity: sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w==} + dependencies: + mkdirp-classic: 0.5.3 + pump: 3.0.0 + tar-stream: 3.1.6 + dev: false + /tar-stream@2.2.0: resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} engines: {node: '>=6'} @@ -10652,6 +10667,14 @@ packages: readable-stream: 3.6.2 dev: false + /tar-stream@3.1.6: + resolution: {integrity: sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==} + dependencies: + b4a: 1.6.4 + fast-fifo: 1.3.2 + streamx: 2.15.5 + dev: false + /tar@6.1.15: resolution: {integrity: sha512-/zKt9UyngnxIT/EAGYuxaMYgOIJiP81ab9ZfkILq4oNLPFX50qyYmu7jRj9qeXoxmJHjGlbH0+cm2uy1WCs10A==} engines: {node: '>=10'} From 59f70dbe802d62224036521bccb9c582805c2412 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Mon, 20 Nov 2023 14:41:25 -0800 Subject: [PATCH 35/59] Tweak lexicon descriptions (#1859) * Tweak lexicon descriptions * Tweak birthdate * Update lexicons/app/bsky/actor/getProfiles.json Co-authored-by: devin ivy * Regenerate packages from lexicons --------- Co-authored-by: devin ivy --- lexicons/app/bsky/actor/defs.json | 4 +- lexicons/app/bsky/actor/getProfile.json | 1 + lexicons/app/bsky/actor/getProfiles.json | 1 + lexicons/app/bsky/actor/getSuggestions.json | 2 +- lexicons/app/bsky/actor/profile.json | 1 + lexicons/app/bsky/actor/putPreferences.json | 2 +- lexicons/app/bsky/actor/searchActors.json | 4 +- .../app/bsky/actor/searchActorsTypeahead.json | 6 +- lexicons/app/bsky/embed/external.json | 2 +- lexicons/app/bsky/embed/images.json | 2 +- lexicons/app/bsky/embed/record.json | 2 +- lexicons/app/bsky/embed/recordWithMedia.json | 2 +- .../app/bsky/feed/describeFeedGenerator.json | 2 +- lexicons/app/bsky/feed/generator.json | 2 +- lexicons/app/bsky/feed/getActorFeeds.json | 2 +- lexicons/app/bsky/feed/getActorLikes.json | 2 +- lexicons/app/bsky/feed/getAuthorFeed.json | 2 +- lexicons/app/bsky/feed/getFeed.json | 2 +- lexicons/app/bsky/feed/getFeedGenerator.json | 2 +- lexicons/app/bsky/feed/getFeedGenerators.json | 2 +- lexicons/app/bsky/feed/getFeedSkeleton.json | 2 +- lexicons/app/bsky/feed/getLikes.json | 1 + lexicons/app/bsky/feed/getListFeed.json | 2 +- lexicons/app/bsky/feed/getPostThread.json | 1 + lexicons/app/bsky/feed/getPosts.json | 2 +- lexicons/app/bsky/feed/getRepostedBy.json | 1 + lexicons/app/bsky/feed/getTimeline.json | 2 +- lexicons/app/bsky/feed/like.json | 1 + lexicons/app/bsky/feed/post.json | 1 + lexicons/app/bsky/feed/repost.json | 1 + lexicons/app/bsky/feed/searchPosts.json | 8 +- lexicons/app/bsky/graph/block.json | 2 +- lexicons/app/bsky/graph/defs.json | 4 +- lexicons/app/bsky/graph/follow.json | 2 +- lexicons/app/bsky/graph/getBlocks.json | 2 +- lexicons/app/bsky/graph/getFollowers.json | 2 +- lexicons/app/bsky/graph/getFollows.json | 2 +- lexicons/app/bsky/graph/getList.json | 2 +- lexicons/app/bsky/graph/getListBlocks.json | 2 +- lexicons/app/bsky/graph/getListMutes.json | 2 +- lexicons/app/bsky/graph/getLists.json | 2 +- lexicons/app/bsky/graph/getMutes.json | 2 +- lexicons/app/bsky/graph/listitem.json | 2 +- lexicons/app/bsky/graph/muteActor.json | 2 +- lexicons/app/bsky/graph/unmuteActor.json | 2 +- .../app/bsky/notification/getUnreadCount.json | 1 + .../bsky/notification/listNotifications.json | 1 + .../app/bsky/notification/registerPush.json | 2 +- lexicons/app/bsky/unspecced/getPopular.json | 2 +- .../unspecced/getPopularFeedGenerators.json | 2 +- .../bsky/unspecced/getTimelineSkeleton.json | 2 +- .../bsky/unspecced/searchActorsSkeleton.json | 10 +- .../bsky/unspecced/searchPostsSkeleton.json | 8 +- lexicons/com/atproto/admin/defs.json | 6 +- .../atproto/admin/disableAccountInvites.json | 4 +- .../com/atproto/admin/disableInviteCodes.json | 2 +- .../atproto/admin/enableAccountInvites.json | 4 +- .../com/atproto/admin/getAccountInfo.json | 2 +- .../com/atproto/admin/getInviteCodes.json | 2 +- .../atproto/admin/getModerationAction.json | 2 +- .../atproto/admin/getModerationActions.json | 2 +- .../atproto/admin/getModerationReport.json | 2 +- .../atproto/admin/getModerationReports.json | 8 +- lexicons/com/atproto/admin/getRecord.json | 2 +- lexicons/com/atproto/admin/getRepo.json | 2 +- .../com/atproto/admin/getSubjectStatus.json | 2 +- lexicons/com/atproto/admin/sendEmail.json | 2 +- .../atproto/admin/takeModerationAction.json | 4 +- .../com/atproto/admin/updateAccountEmail.json | 2 +- .../atproto/admin/updateAccountHandle.json | 2 +- .../atproto/admin/updateSubjectStatus.json | 2 +- .../com/atproto/identity/updateHandle.json | 2 +- lexicons/com/atproto/label/defs.json | 18 +- lexicons/com/atproto/label/queryLabels.json | 4 +- .../com/atproto/label/subscribeLabels.json | 2 +- lexicons/com/atproto/repo/applyWrites.json | 2 +- lexicons/com/atproto/repo/createRecord.json | 4 +- lexicons/com/atproto/repo/deleteRecord.json | 4 +- lexicons/com/atproto/repo/listRecords.json | 2 +- lexicons/com/atproto/repo/putRecord.json | 6 +- .../com/atproto/server/createAppPassword.json | 2 +- .../com/atproto/server/createInviteCodes.json | 2 +- .../com/atproto/server/deleteAccount.json | 2 +- .../atproto/server/getAccountInviteCodes.json | 2 +- .../com/atproto/server/listAppPasswords.json | 2 +- .../server/requestEmailConfirmation.json | 2 +- .../com/atproto/server/revokeAppPassword.json | 2 +- lexicons/com/atproto/sync/getBlocks.json | 2 +- .../com/atproto/sync/getLatestCommit.json | 2 +- lexicons/com/atproto/sync/getRecord.json | 2 +- lexicons/com/atproto/sync/getRepo.json | 2 +- lexicons/com/atproto/sync/listBlobs.json | 4 +- lexicons/com/atproto/sync/listRepos.json | 2 +- lexicons/com/atproto/sync/notifyOfUpdate.json | 2 +- lexicons/com/atproto/sync/subscribeRepos.json | 10 +- packages/api/src/client/lexicons.ts | 268 +++++++++--------- .../src/client/types/app/bsky/actor/defs.ts | 4 +- .../types/app/bsky/actor/searchActors.ts | 4 +- .../app/bsky/actor/searchActorsTypeahead.ts | 4 +- .../client/types/app/bsky/feed/searchPosts.ts | 6 +- .../src/client/types/app/bsky/graph/defs.ts | 4 +- .../bsky/unspecced/searchActorsSkeleton.ts | 8 +- .../app/bsky/unspecced/searchPostsSkeleton.ts | 6 +- .../client/types/com/atproto/admin/defs.ts | 6 +- .../atproto/admin/disableAccountInvites.ts | 2 +- .../com/atproto/admin/enableAccountInvites.ts | 2 +- .../com/atproto/admin/getModerationReports.ts | 6 +- .../com/atproto/admin/takeModerationAction.ts | 2 +- .../client/types/com/atproto/label/defs.ts | 18 +- .../types/com/atproto/label/queryLabels.ts | 4 +- .../types/com/atproto/repo/applyWrites.ts | 2 +- .../types/com/atproto/repo/createRecord.ts | 4 +- .../types/com/atproto/repo/deleteRecord.ts | 4 +- .../types/com/atproto/repo/listRecords.ts | 2 +- .../types/com/atproto/repo/putRecord.ts | 6 +- .../types/com/atproto/sync/listBlobs.ts | 2 +- .../types/com/atproto/sync/subscribeRepos.ts | 8 +- packages/bsky/src/lexicon/lexicons.ts | 268 +++++++++--------- .../src/lexicon/types/app/bsky/actor/defs.ts | 4 +- .../types/app/bsky/actor/searchActors.ts | 4 +- .../app/bsky/actor/searchActorsTypeahead.ts | 4 +- .../types/app/bsky/feed/searchPosts.ts | 6 +- .../src/lexicon/types/app/bsky/graph/defs.ts | 4 +- .../bsky/unspecced/searchActorsSkeleton.ts | 8 +- .../app/bsky/unspecced/searchPostsSkeleton.ts | 6 +- .../lexicon/types/com/atproto/admin/defs.ts | 6 +- .../atproto/admin/disableAccountInvites.ts | 2 +- .../com/atproto/admin/enableAccountInvites.ts | 2 +- .../com/atproto/admin/getModerationReports.ts | 6 +- .../com/atproto/admin/takeModerationAction.ts | 2 +- .../lexicon/types/com/atproto/label/defs.ts | 18 +- .../types/com/atproto/label/queryLabels.ts | 4 +- .../types/com/atproto/repo/applyWrites.ts | 2 +- .../types/com/atproto/repo/createRecord.ts | 4 +- .../types/com/atproto/repo/deleteRecord.ts | 4 +- .../types/com/atproto/repo/listRecords.ts | 2 +- .../types/com/atproto/repo/putRecord.ts | 6 +- .../types/com/atproto/sync/listBlobs.ts | 2 +- .../types/com/atproto/sync/subscribeRepos.ts | 8 +- packages/pds/src/lexicon/lexicons.ts | 268 +++++++++--------- .../src/lexicon/types/app/bsky/actor/defs.ts | 4 +- .../types/app/bsky/actor/searchActors.ts | 4 +- .../app/bsky/actor/searchActorsTypeahead.ts | 4 +- .../types/app/bsky/feed/searchPosts.ts | 6 +- .../src/lexicon/types/app/bsky/graph/defs.ts | 4 +- .../bsky/unspecced/searchActorsSkeleton.ts | 8 +- .../app/bsky/unspecced/searchPostsSkeleton.ts | 6 +- .../lexicon/types/com/atproto/admin/defs.ts | 6 +- .../atproto/admin/disableAccountInvites.ts | 2 +- .../com/atproto/admin/enableAccountInvites.ts | 2 +- .../com/atproto/admin/getModerationReports.ts | 6 +- .../com/atproto/admin/takeModerationAction.ts | 2 +- .../lexicon/types/com/atproto/label/defs.ts | 18 +- .../types/com/atproto/label/queryLabels.ts | 4 +- .../types/com/atproto/repo/applyWrites.ts | 2 +- .../types/com/atproto/repo/createRecord.ts | 4 +- .../types/com/atproto/repo/deleteRecord.ts | 4 +- .../types/com/atproto/repo/listRecords.ts | 2 +- .../types/com/atproto/repo/putRecord.ts | 6 +- .../types/com/atproto/sync/listBlobs.ts | 2 +- .../types/com/atproto/sync/subscribeRepos.ts | 8 +- 161 files changed, 709 insertions(+), 668 deletions(-) diff --git a/lexicons/app/bsky/actor/defs.json b/lexicons/app/bsky/actor/defs.json index 063072ba181..913957f1291 100644 --- a/lexicons/app/bsky/actor/defs.json +++ b/lexicons/app/bsky/actor/defs.json @@ -152,7 +152,7 @@ "birthDate": { "type": "string", "format": "datetime", - "description": "The birth date of the owner of the account." + "description": "The birth date of account owner." } } }, @@ -191,7 +191,7 @@ "properties": { "sort": { "type": "string", - "description": "Sorting mode.", + "description": "Sorting mode for threads.", "knownValues": ["oldest", "newest", "most-likes", "random"] }, "prioritizeFollowedUsers": { diff --git a/lexicons/app/bsky/actor/getProfile.json b/lexicons/app/bsky/actor/getProfile.json index d04ed0e159b..1bb2ad2fea1 100644 --- a/lexicons/app/bsky/actor/getProfile.json +++ b/lexicons/app/bsky/actor/getProfile.json @@ -4,6 +4,7 @@ "defs": { "main": { "type": "query", + "description": "Get detailed profile view of an actor.", "parameters": { "type": "params", "required": ["actor"], diff --git a/lexicons/app/bsky/actor/getProfiles.json b/lexicons/app/bsky/actor/getProfiles.json index ded213b6671..c89622cb18c 100644 --- a/lexicons/app/bsky/actor/getProfiles.json +++ b/lexicons/app/bsky/actor/getProfiles.json @@ -4,6 +4,7 @@ "defs": { "main": { "type": "query", + "description": "Get detailed profile views of multiple actors.", "parameters": { "type": "params", "required": ["actors"], diff --git a/lexicons/app/bsky/actor/getSuggestions.json b/lexicons/app/bsky/actor/getSuggestions.json index 38c30c2c9a6..74465dfdf2e 100644 --- a/lexicons/app/bsky/actor/getSuggestions.json +++ b/lexicons/app/bsky/actor/getSuggestions.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Get a list of actors suggested for following. Used in discovery UIs.", + "description": "Get a list of suggested actors, used for discovery.", "parameters": { "type": "params", "properties": { diff --git a/lexicons/app/bsky/actor/profile.json b/lexicons/app/bsky/actor/profile.json index e0f476deb73..e1b7c6a2b96 100644 --- a/lexicons/app/bsky/actor/profile.json +++ b/lexicons/app/bsky/actor/profile.json @@ -4,6 +4,7 @@ "defs": { "main": { "type": "record", + "description": "A declaration of a profile.", "key": "literal:self", "record": { "type": "object", diff --git a/lexicons/app/bsky/actor/putPreferences.json b/lexicons/app/bsky/actor/putPreferences.json index fff2cdbe495..425ea28c48b 100644 --- a/lexicons/app/bsky/actor/putPreferences.json +++ b/lexicons/app/bsky/actor/putPreferences.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Sets the private preferences attached to the account.", + "description": "Set the private preferences attached to the account.", "input": { "encoding": "application/json", "schema": { diff --git a/lexicons/app/bsky/actor/searchActors.json b/lexicons/app/bsky/actor/searchActors.json index f65e2fc953b..48fbacf4fcc 100644 --- a/lexicons/app/bsky/actor/searchActors.json +++ b/lexicons/app/bsky/actor/searchActors.json @@ -10,11 +10,11 @@ "properties": { "term": { "type": "string", - "description": "DEPRECATED: use 'q' instead" + "description": "DEPRECATED: use 'q' instead." }, "q": { "type": "string", - "description": "search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended" + "description": "Search query string. Syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended." }, "limit": { "type": "integer", diff --git a/lexicons/app/bsky/actor/searchActorsTypeahead.json b/lexicons/app/bsky/actor/searchActorsTypeahead.json index f94dd6c3f69..495b7081c38 100644 --- a/lexicons/app/bsky/actor/searchActorsTypeahead.json +++ b/lexicons/app/bsky/actor/searchActorsTypeahead.json @@ -4,17 +4,17 @@ "defs": { "main": { "type": "query", - "description": "Find actor suggestions for a search term.", + "description": "Find actor suggestions for a prefix search term.", "parameters": { "type": "params", "properties": { "term": { "type": "string", - "description": "DEPRECATED: use 'q' instead" + "description": "DEPRECATED: use 'q' instead." }, "q": { "type": "string", - "description": "search query prefix; not a full query string" + "description": "Search query prefix; not a full query string." }, "limit": { "type": "integer", diff --git a/lexicons/app/bsky/embed/external.json b/lexicons/app/bsky/embed/external.json index 694787eb507..8946382835f 100644 --- a/lexicons/app/bsky/embed/external.json +++ b/lexicons/app/bsky/embed/external.json @@ -1,7 +1,7 @@ { "lexicon": 1, "id": "app.bsky.embed.external", - "description": "A representation of some externally linked content, embedded in another form of content", + "description": "A representation of some externally linked content, embedded in another form of content.", "defs": { "main": { "type": "object", diff --git a/lexicons/app/bsky/embed/images.json b/lexicons/app/bsky/embed/images.json index 48975a60ae2..5baa7ab3f74 100644 --- a/lexicons/app/bsky/embed/images.json +++ b/lexicons/app/bsky/embed/images.json @@ -1,7 +1,7 @@ { "lexicon": 1, "id": "app.bsky.embed.images", - "description": "A set of images embedded in some other form of content", + "description": "A set of images embedded in some other form of content.", "defs": { "main": { "type": "object", diff --git a/lexicons/app/bsky/embed/record.json b/lexicons/app/bsky/embed/record.json index 0d7cb830ba8..4b3d4f814a5 100644 --- a/lexicons/app/bsky/embed/record.json +++ b/lexicons/app/bsky/embed/record.json @@ -1,7 +1,7 @@ { "lexicon": 1, "id": "app.bsky.embed.record", - "description": "A representation of a record embedded in another form of content", + "description": "A representation of a record embedded in another form of content.", "defs": { "main": { "type": "object", diff --git a/lexicons/app/bsky/embed/recordWithMedia.json b/lexicons/app/bsky/embed/recordWithMedia.json index 296494d9ebc..9bc5fe09048 100644 --- a/lexicons/app/bsky/embed/recordWithMedia.json +++ b/lexicons/app/bsky/embed/recordWithMedia.json @@ -1,7 +1,7 @@ { "lexicon": 1, "id": "app.bsky.embed.recordWithMedia", - "description": "A representation of a record embedded in another form of content, alongside other compatible embeds", + "description": "A representation of a record embedded in another form of content, alongside other compatible embeds.", "defs": { "main": { "type": "object", diff --git a/lexicons/app/bsky/feed/describeFeedGenerator.json b/lexicons/app/bsky/feed/describeFeedGenerator.json index 69c143e6016..f95027183a1 100644 --- a/lexicons/app/bsky/feed/describeFeedGenerator.json +++ b/lexicons/app/bsky/feed/describeFeedGenerator.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Returns information about a given feed generator including TOS & offered feed URIs", + "description": "Get information about a feed generator, including policies and offered feed URIs.", "output": { "encoding": "application/json", "schema": { diff --git a/lexicons/app/bsky/feed/generator.json b/lexicons/app/bsky/feed/generator.json index e31bb4484e1..8c00884ad28 100644 --- a/lexicons/app/bsky/feed/generator.json +++ b/lexicons/app/bsky/feed/generator.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "record", - "description": "A declaration of the existence of a feed generator", + "description": "A declaration of the existence of a feed generator.", "key": "any", "record": { "type": "object", diff --git a/lexicons/app/bsky/feed/getActorFeeds.json b/lexicons/app/bsky/feed/getActorFeeds.json index f34aece1609..a0620477bc3 100644 --- a/lexicons/app/bsky/feed/getActorFeeds.json +++ b/lexicons/app/bsky/feed/getActorFeeds.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Retrieve a list of feeds created by a given actor", + "description": "Get a list of feeds created by the actor.", "parameters": { "type": "params", "required": ["actor"], diff --git a/lexicons/app/bsky/feed/getActorLikes.json b/lexicons/app/bsky/feed/getActorLikes.json index df5e45a4295..b3baa58a457 100644 --- a/lexicons/app/bsky/feed/getActorLikes.json +++ b/lexicons/app/bsky/feed/getActorLikes.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "A view of the posts liked by an actor.", + "description": "Get a list of posts liked by an actor.", "parameters": { "type": "params", "required": ["actor"], diff --git a/lexicons/app/bsky/feed/getAuthorFeed.json b/lexicons/app/bsky/feed/getAuthorFeed.json index c1edfb21330..27dccf63e20 100644 --- a/lexicons/app/bsky/feed/getAuthorFeed.json +++ b/lexicons/app/bsky/feed/getAuthorFeed.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "A view of an actor's feed.", + "description": "Get a view of an actor's feed.", "parameters": { "type": "params", "required": ["actor"], diff --git a/lexicons/app/bsky/feed/getFeed.json b/lexicons/app/bsky/feed/getFeed.json index 9aaeb24c7db..84407bde155 100644 --- a/lexicons/app/bsky/feed/getFeed.json +++ b/lexicons/app/bsky/feed/getFeed.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Compose and hydrate a feed from a user's selected feed generator", + "description": "Get a hydrated feed from an actor's selected feed generator.", "parameters": { "type": "params", "required": ["feed"], diff --git a/lexicons/app/bsky/feed/getFeedGenerator.json b/lexicons/app/bsky/feed/getFeedGenerator.json index 5af09254f93..8b3d4d0551a 100644 --- a/lexicons/app/bsky/feed/getFeedGenerator.json +++ b/lexicons/app/bsky/feed/getFeedGenerator.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Get information about a specific feed offered by a feed generator, such as its online status", + "description": "Get information about a feed generator.", "parameters": { "type": "params", "required": ["feed"], diff --git a/lexicons/app/bsky/feed/getFeedGenerators.json b/lexicons/app/bsky/feed/getFeedGenerators.json index d0cae3b39d2..b8e926db00d 100644 --- a/lexicons/app/bsky/feed/getFeedGenerators.json +++ b/lexicons/app/bsky/feed/getFeedGenerators.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Get information about a list of feed generators", + "description": "Get information about a list of feed generators.", "parameters": { "type": "params", "required": ["feeds"], diff --git a/lexicons/app/bsky/feed/getFeedSkeleton.json b/lexicons/app/bsky/feed/getFeedSkeleton.json index d12b4bdc2b8..03f3ba04c0f 100644 --- a/lexicons/app/bsky/feed/getFeedSkeleton.json +++ b/lexicons/app/bsky/feed/getFeedSkeleton.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "A skeleton of a feed provided by a feed generator", + "description": "Get a skeleton of a feed provided by a feed generator.", "parameters": { "type": "params", "required": ["feed"], diff --git a/lexicons/app/bsky/feed/getLikes.json b/lexicons/app/bsky/feed/getLikes.json index e9b632684a8..ffcbc01ac53 100644 --- a/lexicons/app/bsky/feed/getLikes.json +++ b/lexicons/app/bsky/feed/getLikes.json @@ -4,6 +4,7 @@ "defs": { "main": { "type": "query", + "description": "Get the list of likes.", "parameters": { "type": "params", "required": ["uri"], diff --git a/lexicons/app/bsky/feed/getListFeed.json b/lexicons/app/bsky/feed/getListFeed.json index f7b778bda28..4c5358fcfd7 100644 --- a/lexicons/app/bsky/feed/getListFeed.json +++ b/lexicons/app/bsky/feed/getListFeed.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "A view of a recent posts from actors in a list", + "description": "Get a view of a recent posts from actors in a list.", "parameters": { "type": "params", "required": ["list"], diff --git a/lexicons/app/bsky/feed/getPostThread.json b/lexicons/app/bsky/feed/getPostThread.json index 2d5c2e29a11..b983617041f 100644 --- a/lexicons/app/bsky/feed/getPostThread.json +++ b/lexicons/app/bsky/feed/getPostThread.json @@ -4,6 +4,7 @@ "defs": { "main": { "type": "query", + "description": "Get posts in a thread.", "parameters": { "type": "params", "required": ["uri"], diff --git a/lexicons/app/bsky/feed/getPosts.json b/lexicons/app/bsky/feed/getPosts.json index 37056417a27..c985a5cf033 100644 --- a/lexicons/app/bsky/feed/getPosts.json +++ b/lexicons/app/bsky/feed/getPosts.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "A view of an actor's feed.", + "description": "Get a view of an actor's feed.", "parameters": { "type": "params", "required": ["uris"], diff --git a/lexicons/app/bsky/feed/getRepostedBy.json b/lexicons/app/bsky/feed/getRepostedBy.json index 247ca305e67..99abc6d5cde 100644 --- a/lexicons/app/bsky/feed/getRepostedBy.json +++ b/lexicons/app/bsky/feed/getRepostedBy.json @@ -4,6 +4,7 @@ "defs": { "main": { "type": "query", + "description": "Get a list of reposts.", "parameters": { "type": "params", "required": ["uri"], diff --git a/lexicons/app/bsky/feed/getTimeline.json b/lexicons/app/bsky/feed/getTimeline.json index 49e1ae84b37..c3b116381c6 100644 --- a/lexicons/app/bsky/feed/getTimeline.json +++ b/lexicons/app/bsky/feed/getTimeline.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "A view of the user's home timeline.", + "description": "Get a view of the actor's home timeline.", "parameters": { "type": "params", "properties": { diff --git a/lexicons/app/bsky/feed/like.json b/lexicons/app/bsky/feed/like.json index 01d9a08a76c..d82f93bbb1b 100644 --- a/lexicons/app/bsky/feed/like.json +++ b/lexicons/app/bsky/feed/like.json @@ -4,6 +4,7 @@ "defs": { "main": { "type": "record", + "description": "A declaration of a like.", "key": "tid", "record": { "type": "object", diff --git a/lexicons/app/bsky/feed/post.json b/lexicons/app/bsky/feed/post.json index b21f01b6050..d5f92969253 100644 --- a/lexicons/app/bsky/feed/post.json +++ b/lexicons/app/bsky/feed/post.json @@ -4,6 +4,7 @@ "defs": { "main": { "type": "record", + "description": "A declaration of a post.", "key": "tid", "record": { "type": "object", diff --git a/lexicons/app/bsky/feed/repost.json b/lexicons/app/bsky/feed/repost.json index 3b809a53a7e..4dbef10b319 100644 --- a/lexicons/app/bsky/feed/repost.json +++ b/lexicons/app/bsky/feed/repost.json @@ -3,6 +3,7 @@ "id": "app.bsky.feed.repost", "defs": { "main": { + "description": "A declaration of a repost.", "type": "record", "key": "tid", "record": { diff --git a/lexicons/app/bsky/feed/searchPosts.json b/lexicons/app/bsky/feed/searchPosts.json index 34eb38f686e..a3e0bc47f03 100644 --- a/lexicons/app/bsky/feed/searchPosts.json +++ b/lexicons/app/bsky/feed/searchPosts.json @@ -4,14 +4,14 @@ "defs": { "main": { "type": "query", - "description": "Find posts matching search criteria", + "description": "Find posts matching search criteria.", "parameters": { "type": "params", "required": ["q"], "properties": { "q": { "type": "string", - "description": "search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended" + "description": "Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended." }, "limit": { "type": "integer", @@ -21,7 +21,7 @@ }, "cursor": { "type": "string", - "description": "optional pagination mechanism; may not necessarily allow scrolling through entire result set" + "description": "Optional pagination mechanism; may not necessarily allow scrolling through entire result set." } } }, @@ -34,7 +34,7 @@ "cursor": { "type": "string" }, "hitsTotal": { "type": "integer", - "description": "count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits" + "description": "Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits." }, "posts": { "type": "array", diff --git a/lexicons/app/bsky/graph/block.json b/lexicons/app/bsky/graph/block.json index 67821908d6a..6231eb04e10 100644 --- a/lexicons/app/bsky/graph/block.json +++ b/lexicons/app/bsky/graph/block.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "record", - "description": "A block.", + "description": "A declaration of a block.", "key": "tid", "record": { "type": "object", diff --git a/lexicons/app/bsky/graph/defs.json b/lexicons/app/bsky/graph/defs.json index 894cfadc0ef..c957f211670 100644 --- a/lexicons/app/bsky/graph/defs.json +++ b/lexicons/app/bsky/graph/defs.json @@ -54,11 +54,11 @@ }, "modlist": { "type": "token", - "description": "A list of actors to apply an aggregate moderation action (mute/block) on" + "description": "A list of actors to apply an aggregate moderation action (mute/block) on." }, "curatelist": { "type": "token", - "description": "A list of actors used for curation purposes such as list feeds or interaction gating" + "description": "A list of actors used for curation purposes such as list feeds or interaction gating." }, "listViewerState": { "type": "object", diff --git a/lexicons/app/bsky/graph/follow.json b/lexicons/app/bsky/graph/follow.json index 64783ae1d1b..df4f4319d92 100644 --- a/lexicons/app/bsky/graph/follow.json +++ b/lexicons/app/bsky/graph/follow.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "record", - "description": "A social follow.", + "description": "A declaration of a social follow.", "key": "tid", "record": { "type": "object", diff --git a/lexicons/app/bsky/graph/getBlocks.json b/lexicons/app/bsky/graph/getBlocks.json index 1573e57faa3..bbfe956fbe0 100644 --- a/lexicons/app/bsky/graph/getBlocks.json +++ b/lexicons/app/bsky/graph/getBlocks.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Who is the requester's account blocking?", + "description": "Get a list of who the actor is blocking.", "parameters": { "type": "params", "properties": { diff --git a/lexicons/app/bsky/graph/getFollowers.json b/lexicons/app/bsky/graph/getFollowers.json index f3824bbd699..378c7a7a339 100644 --- a/lexicons/app/bsky/graph/getFollowers.json +++ b/lexicons/app/bsky/graph/getFollowers.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Who is following an actor?", + "description": "Get a list of an actor's followers.", "parameters": { "type": "params", "required": ["actor"], diff --git a/lexicons/app/bsky/graph/getFollows.json b/lexicons/app/bsky/graph/getFollows.json index 04ce9fef8bf..b90f7613889 100644 --- a/lexicons/app/bsky/graph/getFollows.json +++ b/lexicons/app/bsky/graph/getFollows.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Who is an actor following?", + "description": "Get a list of who the actor follows.", "parameters": { "type": "params", "required": ["actor"], diff --git a/lexicons/app/bsky/graph/getList.json b/lexicons/app/bsky/graph/getList.json index 0241ee1e7f1..fd24668e5bd 100644 --- a/lexicons/app/bsky/graph/getList.json +++ b/lexicons/app/bsky/graph/getList.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Fetch a list of actors", + "description": "Get a list of actors.", "parameters": { "type": "params", "required": ["list"], diff --git a/lexicons/app/bsky/graph/getListBlocks.json b/lexicons/app/bsky/graph/getListBlocks.json index 709d77aa68b..9f9f59821f2 100644 --- a/lexicons/app/bsky/graph/getListBlocks.json +++ b/lexicons/app/bsky/graph/getListBlocks.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Which lists is the requester's account blocking?", + "description": "Get lists that the actor is blocking.", "parameters": { "type": "params", "properties": { diff --git a/lexicons/app/bsky/graph/getListMutes.json b/lexicons/app/bsky/graph/getListMutes.json index 44b3b652fde..8d42ac40f9c 100644 --- a/lexicons/app/bsky/graph/getListMutes.json +++ b/lexicons/app/bsky/graph/getListMutes.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Which lists is the requester's account muting?", + "description": "Get lists that the actor is muting.", "parameters": { "type": "params", "properties": { diff --git a/lexicons/app/bsky/graph/getLists.json b/lexicons/app/bsky/graph/getLists.json index de83a50d3eb..602dd15307d 100644 --- a/lexicons/app/bsky/graph/getLists.json +++ b/lexicons/app/bsky/graph/getLists.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Fetch a list of lists that belong to an actor", + "description": "Get a list of lists that belong to an actor.", "parameters": { "type": "params", "required": ["actor"], diff --git a/lexicons/app/bsky/graph/getMutes.json b/lexicons/app/bsky/graph/getMutes.json index aba63ea3043..8ceae00f607 100644 --- a/lexicons/app/bsky/graph/getMutes.json +++ b/lexicons/app/bsky/graph/getMutes.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Who does the viewer mute?", + "description": "Get a list of who the actor mutes.", "parameters": { "type": "params", "properties": { diff --git a/lexicons/app/bsky/graph/listitem.json b/lexicons/app/bsky/graph/listitem.json index f05a1641e6f..2eafb1340be 100644 --- a/lexicons/app/bsky/graph/listitem.json +++ b/lexicons/app/bsky/graph/listitem.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "record", - "description": "An item under a declared list of actors", + "description": "An item under a declared list of actors.", "key": "tid", "record": { "type": "object", diff --git a/lexicons/app/bsky/graph/muteActor.json b/lexicons/app/bsky/graph/muteActor.json index 33dc75a7d29..f1c3dd18f64 100644 --- a/lexicons/app/bsky/graph/muteActor.json +++ b/lexicons/app/bsky/graph/muteActor.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Mute an actor by did or handle.", + "description": "Mute an actor by DID or handle.", "input": { "encoding": "application/json", "schema": { diff --git a/lexicons/app/bsky/graph/unmuteActor.json b/lexicons/app/bsky/graph/unmuteActor.json index a71585a95c8..114af204890 100644 --- a/lexicons/app/bsky/graph/unmuteActor.json +++ b/lexicons/app/bsky/graph/unmuteActor.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Unmute an actor by did or handle.", + "description": "Unmute an actor by DID or handle.", "input": { "encoding": "application/json", "schema": { diff --git a/lexicons/app/bsky/notification/getUnreadCount.json b/lexicons/app/bsky/notification/getUnreadCount.json index 8b0cdc3af46..ab716b2a436 100644 --- a/lexicons/app/bsky/notification/getUnreadCount.json +++ b/lexicons/app/bsky/notification/getUnreadCount.json @@ -4,6 +4,7 @@ "defs": { "main": { "type": "query", + "description": "Get the count of unread notifications.", "parameters": { "type": "params", "properties": { diff --git a/lexicons/app/bsky/notification/listNotifications.json b/lexicons/app/bsky/notification/listNotifications.json index c5a9aee0fd4..41f92cad4bc 100644 --- a/lexicons/app/bsky/notification/listNotifications.json +++ b/lexicons/app/bsky/notification/listNotifications.json @@ -4,6 +4,7 @@ "defs": { "main": { "type": "query", + "description": "Get a list of notifications.", "parameters": { "type": "params", "properties": { diff --git a/lexicons/app/bsky/notification/registerPush.json b/lexicons/app/bsky/notification/registerPush.json index 159dd370049..80819ece46f 100644 --- a/lexicons/app/bsky/notification/registerPush.json +++ b/lexicons/app/bsky/notification/registerPush.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Register for push notifications with a service", + "description": "Register for push notifications with a service.", "input": { "encoding": "application/json", "schema": { diff --git a/lexicons/app/bsky/unspecced/getPopular.json b/lexicons/app/bsky/unspecced/getPopular.json index 2aac00fed14..61fe26e889f 100644 --- a/lexicons/app/bsky/unspecced/getPopular.json +++ b/lexicons/app/bsky/unspecced/getPopular.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "DEPRECATED: will be removed soon, please find a feed generator alternative", + "description": "DEPRECATED: will be removed soon. Use a feed generator alternative.", "parameters": { "type": "params", "properties": { diff --git a/lexicons/app/bsky/unspecced/getPopularFeedGenerators.json b/lexicons/app/bsky/unspecced/getPopularFeedGenerators.json index ddeb7e7fab2..ee05c16a1c1 100644 --- a/lexicons/app/bsky/unspecced/getPopularFeedGenerators.json +++ b/lexicons/app/bsky/unspecced/getPopularFeedGenerators.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "An unspecced view of globally popular feed generators", + "description": "An unspecced view of globally popular feed generators.", "parameters": { "type": "params", "properties": { diff --git a/lexicons/app/bsky/unspecced/getTimelineSkeleton.json b/lexicons/app/bsky/unspecced/getTimelineSkeleton.json index c720b81832d..391c3718dc1 100644 --- a/lexicons/app/bsky/unspecced/getTimelineSkeleton.json +++ b/lexicons/app/bsky/unspecced/getTimelineSkeleton.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "A skeleton of a timeline - UNSPECCED & WILL GO AWAY SOON", + "description": "DEPRECATED: a skeleton of a timeline. Unspecced and will be unavailable soon.", "parameters": { "type": "params", "properties": { diff --git a/lexicons/app/bsky/unspecced/searchActorsSkeleton.json b/lexicons/app/bsky/unspecced/searchActorsSkeleton.json index 108dacf9b14..ea73577d57d 100644 --- a/lexicons/app/bsky/unspecced/searchActorsSkeleton.json +++ b/lexicons/app/bsky/unspecced/searchActorsSkeleton.json @@ -4,18 +4,18 @@ "defs": { "main": { "type": "query", - "description": "Backend Actors (profile) search, returning only skeleton", + "description": "Backend Actors (profile) search, returns only skeleton.", "parameters": { "type": "params", "required": ["q"], "properties": { "q": { "type": "string", - "description": "search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. For typeahead search, only simple term match is supported, not full syntax" + "description": "Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. For typeahead search, only simple term match is supported, not full syntax." }, "typeahead": { "type": "boolean", - "description": "if true, acts as fast/simple 'typeahead' query" + "description": "If true, acts as fast/simple 'typeahead' query." }, "limit": { "type": "integer", @@ -25,7 +25,7 @@ }, "cursor": { "type": "string", - "description": "optional pagination mechanism; may not necessarily allow scrolling through entire result set" + "description": "Optional pagination mechanism; may not necessarily allow scrolling through entire result set." } } }, @@ -38,7 +38,7 @@ "cursor": { "type": "string" }, "hitsTotal": { "type": "integer", - "description": "count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits" + "description": "Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits." }, "actors": { "type": "array", diff --git a/lexicons/app/bsky/unspecced/searchPostsSkeleton.json b/lexicons/app/bsky/unspecced/searchPostsSkeleton.json index 532bfea79f9..3305dd98734 100644 --- a/lexicons/app/bsky/unspecced/searchPostsSkeleton.json +++ b/lexicons/app/bsky/unspecced/searchPostsSkeleton.json @@ -4,14 +4,14 @@ "defs": { "main": { "type": "query", - "description": "Backend Posts search, returning only skeleton", + "description": "Backend Posts search, returns only skeleton", "parameters": { "type": "params", "required": ["q"], "properties": { "q": { "type": "string", - "description": "search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended" + "description": "Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended." }, "limit": { "type": "integer", @@ -21,7 +21,7 @@ }, "cursor": { "type": "string", - "description": "optional pagination mechanism; may not necessarily allow scrolling through entire result set" + "description": "Optional pagination mechanism; may not necessarily allow scrolling through entire result set." } } }, @@ -34,7 +34,7 @@ "cursor": { "type": "string" }, "hitsTotal": { "type": "integer", - "description": "count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits" + "description": "Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits." }, "posts": { "type": "array", diff --git a/lexicons/com/atproto/admin/defs.json b/lexicons/com/atproto/admin/defs.json index 0d57da18ae8..42dc7165423 100644 --- a/lexicons/com/atproto/admin/defs.json +++ b/lexicons/com/atproto/admin/defs.json @@ -27,7 +27,7 @@ "action": { "type": "ref", "ref": "#actionType" }, "durationInHours": { "type": "integer", - "description": "Indicates how long this action was meant to be in effect before automatically expiring." + "description": "Indicates how long this action is meant to be in effect before automatically expiring." }, "subject": { "type": "union", @@ -60,7 +60,7 @@ "action": { "type": "ref", "ref": "#actionType" }, "durationInHours": { "type": "integer", - "description": "Indicates how long this action was meant to be in effect before automatically expiring." + "description": "Indicates how long this action is meant to be in effect before automatically expiring." }, "subject": { "type": "union", @@ -95,7 +95,7 @@ "action": { "type": "ref", "ref": "#actionType" }, "durationInHours": { "type": "integer", - "description": "Indicates how long this action was meant to be in effect before automatically expiring." + "description": "Indicates how long this action is meant to be in effect before automatically expiring." } } }, diff --git a/lexicons/com/atproto/admin/disableAccountInvites.json b/lexicons/com/atproto/admin/disableAccountInvites.json index 2006271d089..8bd6cabbc92 100644 --- a/lexicons/com/atproto/admin/disableAccountInvites.json +++ b/lexicons/com/atproto/admin/disableAccountInvites.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Disable an account from receiving new invite codes, but does not invalidate existing codes", + "description": "Disable an account from receiving new invite codes, but does not invalidate existing codes.", "input": { "encoding": "application/json", "schema": { @@ -14,7 +14,7 @@ "account": { "type": "string", "format": "did" }, "note": { "type": "string", - "description": "Additionally add a note describing why the invites were disabled" + "description": "Optional reason for disabled invites." } } } diff --git a/lexicons/com/atproto/admin/disableInviteCodes.json b/lexicons/com/atproto/admin/disableInviteCodes.json index f84bc05f203..914c8cb29e6 100644 --- a/lexicons/com/atproto/admin/disableInviteCodes.json +++ b/lexicons/com/atproto/admin/disableInviteCodes.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Disable some set of codes and/or all codes associated with a set of users", + "description": "Disable some set of codes and/or all codes associated with a set of users.", "input": { "encoding": "application/json", "schema": { diff --git a/lexicons/com/atproto/admin/enableAccountInvites.json b/lexicons/com/atproto/admin/enableAccountInvites.json index ac42d727879..97443cd9895 100644 --- a/lexicons/com/atproto/admin/enableAccountInvites.json +++ b/lexicons/com/atproto/admin/enableAccountInvites.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Re-enable an accounts ability to receive invite codes", + "description": "Re-enable an account's ability to receive invite codes.", "input": { "encoding": "application/json", "schema": { @@ -14,7 +14,7 @@ "account": { "type": "string", "format": "did" }, "note": { "type": "string", - "description": "Additionally add a note describing why the invites were enabled" + "description": "Optional reason for enabled invites." } } } diff --git a/lexicons/com/atproto/admin/getAccountInfo.json b/lexicons/com/atproto/admin/getAccountInfo.json index da8e839fdfa..e5d33fdcf10 100644 --- a/lexicons/com/atproto/admin/getAccountInfo.json +++ b/lexicons/com/atproto/admin/getAccountInfo.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "View details about an account.", + "description": "Get details about an account.", "parameters": { "type": "params", "required": ["did"], diff --git a/lexicons/com/atproto/admin/getInviteCodes.json b/lexicons/com/atproto/admin/getInviteCodes.json index 895f1259d7c..883604dc7e1 100644 --- a/lexicons/com/atproto/admin/getInviteCodes.json +++ b/lexicons/com/atproto/admin/getInviteCodes.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Admin view of invite codes", + "description": "Get an admin view of invite codes.", "parameters": { "type": "params", "properties": { diff --git a/lexicons/com/atproto/admin/getModerationAction.json b/lexicons/com/atproto/admin/getModerationAction.json index b25ccc227e1..eae0736bb3d 100644 --- a/lexicons/com/atproto/admin/getModerationAction.json +++ b/lexicons/com/atproto/admin/getModerationAction.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "View details about a moderation action.", + "description": "Get details about a moderation action.", "parameters": { "type": "params", "required": ["id"], diff --git a/lexicons/com/atproto/admin/getModerationActions.json b/lexicons/com/atproto/admin/getModerationActions.json index 89e16fd6919..370ba7d2f72 100644 --- a/lexicons/com/atproto/admin/getModerationActions.json +++ b/lexicons/com/atproto/admin/getModerationActions.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "List moderation actions related to a subject.", + "description": "Get a list of moderation actions related to a subject.", "parameters": { "type": "params", "properties": { diff --git a/lexicons/com/atproto/admin/getModerationReport.json b/lexicons/com/atproto/admin/getModerationReport.json index 3e84e13e676..0e7efc16fde 100644 --- a/lexicons/com/atproto/admin/getModerationReport.json +++ b/lexicons/com/atproto/admin/getModerationReport.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "View details about a moderation report.", + "description": "Get details about a moderation report.", "parameters": { "type": "params", "required": ["id"], diff --git a/lexicons/com/atproto/admin/getModerationReports.json b/lexicons/com/atproto/admin/getModerationReports.json index ad930389147..0caeac1a8d6 100644 --- a/lexicons/com/atproto/admin/getModerationReports.json +++ b/lexicons/com/atproto/admin/getModerationReports.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "List moderation reports related to a subject.", + "description": "Get moderation reports related to a subject.", "parameters": { "type": "params", "properties": { @@ -13,12 +13,12 @@ "actionedBy": { "type": "string", "format": "did", - "description": "Get all reports that were actioned by a specific moderator" + "description": "Get all reports that were actioned by a specific moderator." }, "reporters": { "type": "array", "items": { "type": "string" }, - "description": "Filter reports made by one or more DIDs" + "description": "Filter reports made by one or more DIDs." }, "resolved": { "type": "boolean" }, "actionType": { @@ -39,7 +39,7 @@ "cursor": { "type": "string" }, "reverse": { "type": "boolean", - "description": "Reverse the order of the returned records? when true, returns reports in chronological order" + "description": "Reverse the order of the returned records. When true, returns reports in chronological order." } } }, diff --git a/lexicons/com/atproto/admin/getRecord.json b/lexicons/com/atproto/admin/getRecord.json index a16fe484feb..7c513bec72c 100644 --- a/lexicons/com/atproto/admin/getRecord.json +++ b/lexicons/com/atproto/admin/getRecord.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "View details about a record.", + "description": "Get details about a record.", "parameters": { "type": "params", "required": ["uri"], diff --git a/lexicons/com/atproto/admin/getRepo.json b/lexicons/com/atproto/admin/getRepo.json index d8cfd883f72..c7966158f11 100644 --- a/lexicons/com/atproto/admin/getRepo.json +++ b/lexicons/com/atproto/admin/getRepo.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "View details about a repository.", + "description": "Get details about a repository.", "parameters": { "type": "params", "required": ["did"], diff --git a/lexicons/com/atproto/admin/getSubjectStatus.json b/lexicons/com/atproto/admin/getSubjectStatus.json index a6ce340c009..0668aad611f 100644 --- a/lexicons/com/atproto/admin/getSubjectStatus.json +++ b/lexicons/com/atproto/admin/getSubjectStatus.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Fetch the service-specific the admin status of a subject (account, record, or blob)", + "description": "Get the service-specific admin status of a subject (account, record, or blob).", "parameters": { "type": "params", "properties": { diff --git a/lexicons/com/atproto/admin/sendEmail.json b/lexicons/com/atproto/admin/sendEmail.json index 8df082258dd..c6af697edd2 100644 --- a/lexicons/com/atproto/admin/sendEmail.json +++ b/lexicons/com/atproto/admin/sendEmail.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Send email to a user's primary email address", + "description": "Send email to a user's account email address.", "input": { "encoding": "application/json", "schema": { diff --git a/lexicons/com/atproto/admin/takeModerationAction.json b/lexicons/com/atproto/admin/takeModerationAction.json index 76600735251..70b650aa4b1 100644 --- a/lexicons/com/atproto/admin/takeModerationAction.json +++ b/lexicons/com/atproto/admin/takeModerationAction.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Take a moderation action on a repo.", + "description": "Take a moderation action on an actor.", "input": { "encoding": "application/json", "schema": { @@ -41,7 +41,7 @@ "reason": { "type": "string" }, "durationInHours": { "type": "integer", - "description": "Indicates how long this action was meant to be in effect before automatically expiring." + "description": "Indicates how long this action is meant to be in effect before automatically expiring." }, "createdBy": { "type": "string", "format": "did" } } diff --git a/lexicons/com/atproto/admin/updateAccountEmail.json b/lexicons/com/atproto/admin/updateAccountEmail.json index a4cf711dee9..f0c3eeb7991 100644 --- a/lexicons/com/atproto/admin/updateAccountEmail.json +++ b/lexicons/com/atproto/admin/updateAccountEmail.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Administrative action to update an account's email", + "description": "Administrative action to update an account's email.", "input": { "encoding": "application/json", "schema": { diff --git a/lexicons/com/atproto/admin/updateAccountHandle.json b/lexicons/com/atproto/admin/updateAccountHandle.json index 15442e98beb..23b8656627b 100644 --- a/lexicons/com/atproto/admin/updateAccountHandle.json +++ b/lexicons/com/atproto/admin/updateAccountHandle.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Administrative action to update an account's handle", + "description": "Administrative action to update an account's handle.", "input": { "encoding": "application/json", "schema": { diff --git a/lexicons/com/atproto/admin/updateSubjectStatus.json b/lexicons/com/atproto/admin/updateSubjectStatus.json index 5273aea4da6..9483532e0a8 100644 --- a/lexicons/com/atproto/admin/updateSubjectStatus.json +++ b/lexicons/com/atproto/admin/updateSubjectStatus.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Update the service-specific admin status of a subject (account, record, or blob)", + "description": "Update the service-specific admin status of a subject (account, record, or blob).", "input": { "encoding": "application/json", "schema": { diff --git a/lexicons/com/atproto/identity/updateHandle.json b/lexicons/com/atproto/identity/updateHandle.json index 2b595d189f5..5fe392bb838 100644 --- a/lexicons/com/atproto/identity/updateHandle.json +++ b/lexicons/com/atproto/identity/updateHandle.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Updates the handle of the account", + "description": "Updates the handle of the account.", "input": { "encoding": "application/json", "schema": { diff --git a/lexicons/com/atproto/label/defs.json b/lexicons/com/atproto/label/defs.json index 88a262b11c8..06e4e8f9cd2 100644 --- a/lexicons/com/atproto/label/defs.json +++ b/lexicons/com/atproto/label/defs.json @@ -4,37 +4,37 @@ "defs": { "label": { "type": "object", - "description": "Metadata tag on an atproto resource (eg, repo or record)", + "description": "Metadata tag on an atproto resource (eg, repo or record).", "required": ["src", "uri", "val", "cts"], "properties": { "src": { "type": "string", "format": "did", - "description": "DID of the actor who created this label" + "description": "DID of the actor who created this label." }, "uri": { "type": "string", "format": "uri", - "description": "AT URI of the record, repository (account), or other resource which this label applies to" + "description": "AT URI of the record, repository (account), or other resource that this label applies to." }, "cid": { "type": "string", "format": "cid", - "description": "optionally, CID specifying the specific version of 'uri' resource this label applies to" + "description": "Optionally, CID specifying the specific version of 'uri' resource this label applies to." }, "val": { "type": "string", "maxLength": 128, - "description": "the short string name of the value or type of this label" + "description": "The short string name of the value or type of this label." }, "neg": { "type": "boolean", - "description": "if true, this is a negation label, overwriting a previous label" + "description": "If true, this is a negation label, overwriting a previous label." }, "cts": { "type": "string", "format": "datetime", - "description": "timestamp when this label was created" + "description": "Timestamp when this label was created." } } }, @@ -52,13 +52,13 @@ }, "selfLabel": { "type": "object", - "description": "Metadata tag on an atproto record, published by the author within the record. Note -- schemas should use #selfLabels, not #selfLabel.", + "description": "Metadata tag on an atproto record, published by the author within the record. Note that schemas should use #selfLabels, not #selfLabel.", "required": ["val"], "properties": { "val": { "type": "string", "maxLength": 128, - "description": "the short string name of the value or type of this label" + "description": "The short string name of the value or type of this label." } } } diff --git a/lexicons/com/atproto/label/queryLabels.json b/lexicons/com/atproto/label/queryLabels.json index f4773f255e3..7b6fbe23d54 100644 --- a/lexicons/com/atproto/label/queryLabels.json +++ b/lexicons/com/atproto/label/queryLabels.json @@ -12,12 +12,12 @@ "uriPatterns": { "type": "array", "items": { "type": "string" }, - "description": "List of AT URI patterns to match (boolean 'OR'). Each may be a prefix (ending with '*'; will match inclusive of the string leading to '*'), or a full URI" + "description": "List of AT URI patterns to match (boolean 'OR'). Each may be a prefix (ending with '*'; will match inclusive of the string leading to '*'), or a full URI." }, "sources": { "type": "array", "items": { "type": "string", "format": "did" }, - "description": "Optional list of label sources (DIDs) to filter on" + "description": "Optional list of label sources (DIDs) to filter on." }, "limit": { "type": "integer", diff --git a/lexicons/com/atproto/label/subscribeLabels.json b/lexicons/com/atproto/label/subscribeLabels.json index 044036cfad4..9813ffc192e 100644 --- a/lexicons/com/atproto/label/subscribeLabels.json +++ b/lexicons/com/atproto/label/subscribeLabels.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "subscription", - "description": "Subscribe to label updates", + "description": "Subscribe to label updates.", "parameters": { "type": "params", "properties": { diff --git a/lexicons/com/atproto/repo/applyWrites.json b/lexicons/com/atproto/repo/applyWrites.json index 5c37340cf3f..050b6efbfab 100644 --- a/lexicons/com/atproto/repo/applyWrites.json +++ b/lexicons/com/atproto/repo/applyWrites.json @@ -19,7 +19,7 @@ "validate": { "type": "boolean", "default": true, - "description": "Validate the records?" + "description": "Flag for validating the records." }, "writes": { "type": "array", diff --git a/lexicons/com/atproto/repo/createRecord.json b/lexicons/com/atproto/repo/createRecord.json index e594db58d51..baef20c88f0 100644 --- a/lexicons/com/atproto/repo/createRecord.json +++ b/lexicons/com/atproto/repo/createRecord.json @@ -29,7 +29,7 @@ "validate": { "type": "boolean", "default": true, - "description": "Validate the record?" + "description": "Flag for validating the record." }, "record": { "type": "unknown", @@ -38,7 +38,7 @@ "swapCommit": { "type": "string", "format": "cid", - "description": "Compare and swap with the previous commit by cid." + "description": "Compare and swap with the previous commit by CID." } } } diff --git a/lexicons/com/atproto/repo/deleteRecord.json b/lexicons/com/atproto/repo/deleteRecord.json index 53ef72b190e..d8d7955b6a9 100644 --- a/lexicons/com/atproto/repo/deleteRecord.json +++ b/lexicons/com/atproto/repo/deleteRecord.json @@ -28,12 +28,12 @@ "swapRecord": { "type": "string", "format": "cid", - "description": "Compare and swap with the previous record by cid." + "description": "Compare and swap with the previous record by CID." }, "swapCommit": { "type": "string", "format": "cid", - "description": "Compare and swap with the previous commit by cid." + "description": "Compare and swap with the previous commit by CID." } } } diff --git a/lexicons/com/atproto/repo/listRecords.json b/lexicons/com/atproto/repo/listRecords.json index 8bcee4fcb58..ac04e3e8782 100644 --- a/lexicons/com/atproto/repo/listRecords.json +++ b/lexicons/com/atproto/repo/listRecords.json @@ -37,7 +37,7 @@ }, "reverse": { "type": "boolean", - "description": "Reverse the order of the returned records?" + "description": "Flag to reverse the order of the returned records." } } }, diff --git a/lexicons/com/atproto/repo/putRecord.json b/lexicons/com/atproto/repo/putRecord.json index 118b41dc49c..ae39bd95ead 100644 --- a/lexicons/com/atproto/repo/putRecord.json +++ b/lexicons/com/atproto/repo/putRecord.json @@ -30,7 +30,7 @@ "validate": { "type": "boolean", "default": true, - "description": "Validate the record?" + "description": "Flag for validating the record." }, "record": { "type": "unknown", @@ -39,12 +39,12 @@ "swapRecord": { "type": "string", "format": "cid", - "description": "Compare and swap with the previous record by cid." + "description": "Compare and swap with the previous record by CID." }, "swapCommit": { "type": "string", "format": "cid", - "description": "Compare and swap with the previous commit by cid." + "description": "Compare and swap with the previous commit by CID." } } } diff --git a/lexicons/com/atproto/server/createAppPassword.json b/lexicons/com/atproto/server/createAppPassword.json index 4dfdcca1f6b..f12e8e2557e 100644 --- a/lexicons/com/atproto/server/createAppPassword.json +++ b/lexicons/com/atproto/server/createAppPassword.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Create an app-specific password.", + "description": "Create an App Password.", "input": { "encoding": "application/json", "schema": { diff --git a/lexicons/com/atproto/server/createInviteCodes.json b/lexicons/com/atproto/server/createInviteCodes.json index 827d798a088..1965fe995f6 100644 --- a/lexicons/com/atproto/server/createInviteCodes.json +++ b/lexicons/com/atproto/server/createInviteCodes.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Create an invite code.", + "description": "Create invite codes.", "input": { "encoding": "application/json", "schema": { diff --git a/lexicons/com/atproto/server/deleteAccount.json b/lexicons/com/atproto/server/deleteAccount.json index 2cef799aefe..3747189dca3 100644 --- a/lexicons/com/atproto/server/deleteAccount.json +++ b/lexicons/com/atproto/server/deleteAccount.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Delete a user account with a token and password.", + "description": "Delete an actor's account with a token and password.", "input": { "encoding": "application/json", "schema": { diff --git a/lexicons/com/atproto/server/getAccountInviteCodes.json b/lexicons/com/atproto/server/getAccountInviteCodes.json index a99f78e0d7b..ac23b11f23f 100644 --- a/lexicons/com/atproto/server/getAccountInviteCodes.json +++ b/lexicons/com/atproto/server/getAccountInviteCodes.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Get all invite codes for a given account", + "description": "Get all invite codes for a given account.", "parameters": { "type": "params", "properties": { diff --git a/lexicons/com/atproto/server/listAppPasswords.json b/lexicons/com/atproto/server/listAppPasswords.json index f50a13d6b82..13188897e00 100644 --- a/lexicons/com/atproto/server/listAppPasswords.json +++ b/lexicons/com/atproto/server/listAppPasswords.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "List all app-specific passwords.", + "description": "List all App Passwords.", "output": { "encoding": "application/json", "schema": { diff --git a/lexicons/com/atproto/server/requestEmailConfirmation.json b/lexicons/com/atproto/server/requestEmailConfirmation.json index 4b2470bf59b..0020ed31607 100644 --- a/lexicons/com/atproto/server/requestEmailConfirmation.json +++ b/lexicons/com/atproto/server/requestEmailConfirmation.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Request an email with a code to confirm ownership of email" + "description": "Request an email with a code to confirm ownership of email." } } } diff --git a/lexicons/com/atproto/server/revokeAppPassword.json b/lexicons/com/atproto/server/revokeAppPassword.json index 52094c4459c..bcfd3d864e3 100644 --- a/lexicons/com/atproto/server/revokeAppPassword.json +++ b/lexicons/com/atproto/server/revokeAppPassword.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Revoke an app-specific password by name.", + "description": "Revoke an App Password by name.", "input": { "encoding": "application/json", "schema": { diff --git a/lexicons/com/atproto/sync/getBlocks.json b/lexicons/com/atproto/sync/getBlocks.json index 0b6c25f8252..cf776a0c88f 100644 --- a/lexicons/com/atproto/sync/getBlocks.json +++ b/lexicons/com/atproto/sync/getBlocks.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Gets blocks from a given repo.", + "description": "Get blocks from a given repo.", "parameters": { "type": "params", "required": ["did", "cids"], diff --git a/lexicons/com/atproto/sync/getLatestCommit.json b/lexicons/com/atproto/sync/getLatestCommit.json index 602cc2dac59..d8754f09062 100644 --- a/lexicons/com/atproto/sync/getLatestCommit.json +++ b/lexicons/com/atproto/sync/getLatestCommit.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Gets the current commit CID & revision of the repo.", + "description": "Get the current commit CID & revision of the repo.", "parameters": { "type": "params", "required": ["did"], diff --git a/lexicons/com/atproto/sync/getRecord.json b/lexicons/com/atproto/sync/getRecord.json index ea14ba0f75e..cbd0ad3a5ac 100644 --- a/lexicons/com/atproto/sync/getRecord.json +++ b/lexicons/com/atproto/sync/getRecord.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Gets blocks needed for existence or non-existence of record.", + "description": "Get blocks needed for existence or non-existence of record.", "parameters": { "type": "params", "required": ["did", "collection", "rkey"], diff --git a/lexicons/com/atproto/sync/getRepo.json b/lexicons/com/atproto/sync/getRepo.json index 6bb6de2b1e4..fb68ab670ee 100644 --- a/lexicons/com/atproto/sync/getRepo.json +++ b/lexicons/com/atproto/sync/getRepo.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Gets the did's repo, optionally catching up from a specific revision.", + "description": "Gets the DID's repo, optionally catching up from a specific revision.", "parameters": { "type": "params", "required": ["did"], diff --git a/lexicons/com/atproto/sync/listBlobs.json b/lexicons/com/atproto/sync/listBlobs.json index 9f25b1330a6..46815eeb49a 100644 --- a/lexicons/com/atproto/sync/listBlobs.json +++ b/lexicons/com/atproto/sync/listBlobs.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "List blob cids since some revision", + "description": "List blob CIDs since some revision.", "parameters": { "type": "params", "required": ["did"], @@ -16,7 +16,7 @@ }, "since": { "type": "string", - "description": "Optional revision of the repo to list blobs since" + "description": "Optional revision of the repo to list blobs since." }, "limit": { "type": "integer", diff --git a/lexicons/com/atproto/sync/listRepos.json b/lexicons/com/atproto/sync/listRepos.json index ad4eabdabf1..440e8693d5e 100644 --- a/lexicons/com/atproto/sync/listRepos.json +++ b/lexicons/com/atproto/sync/listRepos.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "List dids and root cids of hosted repos", + "description": "List DIDs and root CIDs of hosted repos.", "parameters": { "type": "params", "properties": { diff --git a/lexicons/com/atproto/sync/notifyOfUpdate.json b/lexicons/com/atproto/sync/notifyOfUpdate.json index caca6fe9f30..48cb4b24678 100644 --- a/lexicons/com/atproto/sync/notifyOfUpdate.json +++ b/lexicons/com/atproto/sync/notifyOfUpdate.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Notify a crawling service of a recent update. Often when a long break between updates causes the connection with the crawling service to break.", + "description": "Notify a crawling service of a recent update; often when a long break between updates causes the connection with the crawling service to break.", "input": { "encoding": "application/json", "schema": { diff --git a/lexicons/com/atproto/sync/subscribeRepos.json b/lexicons/com/atproto/sync/subscribeRepos.json index b8feecad5b8..9a5c0f6153c 100644 --- a/lexicons/com/atproto/sync/subscribeRepos.json +++ b/lexicons/com/atproto/sync/subscribeRepos.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "subscription", - "description": "Subscribe to repo updates", + "description": "Subscribe to repo updates.", "parameters": { "type": "params", "properties": { @@ -47,15 +47,15 @@ "prev": { "type": "cid-link" }, "rev": { "type": "string", - "description": "The rev of the emitted commit" + "description": "The rev of the emitted commit." }, "since": { "type": "string", - "description": "The rev of the last emitted commit from this repo" + "description": "The rev of the last emitted commit from this repo." }, "blocks": { "type": "bytes", - "description": "CAR file containing relevant blocks", + "description": "CAR file containing relevant blocks.", "maxLength": 1000000 }, "ops": { @@ -115,7 +115,7 @@ }, "repoOp": { "type": "object", - "description": "A repo operation, ie a write of a single record. For creates and updates, cid is the record's CID as of this operation. For deletes, it's null.", + "description": "A repo operation, ie a write of a single record. For creates and updates, CID is the record's CID as of this operation. For deletes, it's null.", "required": ["action", "path", "cid"], "nullable": ["cid"], "properties": { diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index 545d8e9d43d..14c1be9f477 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -43,7 +43,7 @@ export const schemaDict = { durationInHours: { type: 'integer', description: - 'Indicates how long this action was meant to be in effect before automatically expiring.', + 'Indicates how long this action is meant to be in effect before automatically expiring.', }, subject: { type: 'union', @@ -116,7 +116,7 @@ export const schemaDict = { durationInHours: { type: 'integer', description: - 'Indicates how long this action was meant to be in effect before automatically expiring.', + 'Indicates how long this action is meant to be in effect before automatically expiring.', }, subject: { type: 'union', @@ -184,7 +184,7 @@ export const schemaDict = { durationInHours: { type: 'integer', description: - 'Indicates how long this action was meant to be in effect before automatically expiring.', + 'Indicates how long this action is meant to be in effect before automatically expiring.', }, }, }, @@ -725,7 +725,7 @@ export const schemaDict = { main: { type: 'procedure', description: - 'Disable an account from receiving new invite codes, but does not invalidate existing codes', + 'Disable an account from receiving new invite codes, but does not invalidate existing codes.', input: { encoding: 'application/json', schema: { @@ -738,8 +738,7 @@ export const schemaDict = { }, note: { type: 'string', - description: - 'Additionally add a note describing why the invites were disabled', + description: 'Optional reason for disabled invites.', }, }, }, @@ -754,7 +753,7 @@ export const schemaDict = { main: { type: 'procedure', description: - 'Disable some set of codes and/or all codes associated with a set of users', + 'Disable some set of codes and/or all codes associated with a set of users.', input: { encoding: 'application/json', schema: { @@ -784,7 +783,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Re-enable an accounts ability to receive invite codes', + description: "Re-enable an account's ability to receive invite codes.", input: { encoding: 'application/json', schema: { @@ -797,8 +796,7 @@ export const schemaDict = { }, note: { type: 'string', - description: - 'Additionally add a note describing why the invites were enabled', + description: 'Optional reason for enabled invites.', }, }, }, @@ -812,7 +810,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'View details about an account.', + description: 'Get details about an account.', parameters: { type: 'params', required: ['did'], @@ -839,7 +837,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Admin view of invite codes', + description: 'Get an admin view of invite codes.', parameters: { type: 'params', properties: { @@ -887,7 +885,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'View details about a moderation action.', + description: 'Get details about a moderation action.', parameters: { type: 'params', required: ['id'], @@ -913,7 +911,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'List moderation actions related to a subject.', + description: 'Get a list of moderation actions related to a subject.', parameters: { type: 'params', properties: { @@ -959,7 +957,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'View details about a moderation report.', + description: 'Get details about a moderation report.', parameters: { type: 'params', required: ['id'], @@ -985,7 +983,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'List moderation reports related to a subject.', + description: 'Get moderation reports related to a subject.', parameters: { type: 'params', properties: { @@ -1002,14 +1000,14 @@ export const schemaDict = { type: 'string', format: 'did', description: - 'Get all reports that were actioned by a specific moderator', + 'Get all reports that were actioned by a specific moderator.', }, reporters: { type: 'array', items: { type: 'string', }, - description: 'Filter reports made by one or more DIDs', + description: 'Filter reports made by one or more DIDs.', }, resolved: { type: 'boolean', @@ -1035,7 +1033,7 @@ export const schemaDict = { reverse: { type: 'boolean', description: - 'Reverse the order of the returned records? when true, returns reports in chronological order', + 'Reverse the order of the returned records. When true, returns reports in chronological order.', }, }, }, @@ -1067,7 +1065,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'View details about a record.', + description: 'Get details about a record.', parameters: { type: 'params', required: ['uri'], @@ -1103,7 +1101,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'View details about a repository.', + description: 'Get details about a repository.', parameters: { type: 'params', required: ['did'], @@ -1136,7 +1134,7 @@ export const schemaDict = { main: { type: 'query', description: - 'Fetch the service-specific the admin status of a subject (account, record, or blob)', + 'Get the service-specific admin status of a subject (account, record, or blob).', parameters: { type: 'params', properties: { @@ -1309,7 +1307,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: "Send email to a user's primary email address", + description: "Send email to a user's account email address.", input: { encoding: 'application/json', schema: { @@ -1350,7 +1348,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Take a moderation action on a repo.', + description: 'Take a moderation action on an actor.', input: { encoding: 'application/json', schema: { @@ -1397,7 +1395,7 @@ export const schemaDict = { durationInHours: { type: 'integer', description: - 'Indicates how long this action was meant to be in effect before automatically expiring.', + 'Indicates how long this action is meant to be in effect before automatically expiring.', }, createdBy: { type: 'string', @@ -1427,7 +1425,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: "Administrative action to update an account's email", + description: "Administrative action to update an account's email.", input: { encoding: 'application/json', schema: { @@ -1454,7 +1452,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: "Administrative action to update an account's handle", + description: "Administrative action to update an account's handle.", input: { encoding: 'application/json', schema: { @@ -1482,7 +1480,7 @@ export const schemaDict = { main: { type: 'procedure', description: - 'Update the service-specific admin status of a subject (account, record, or blob)', + 'Update the service-specific admin status of a subject (account, record, or blob).', input: { encoding: 'application/json', schema: { @@ -1568,7 +1566,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Updates the handle of the account', + description: 'Updates the handle of the account.', input: { encoding: 'application/json', schema: { @@ -1591,41 +1589,42 @@ export const schemaDict = { defs: { label: { type: 'object', - description: 'Metadata tag on an atproto resource (eg, repo or record)', + description: + 'Metadata tag on an atproto resource (eg, repo or record).', required: ['src', 'uri', 'val', 'cts'], properties: { src: { type: 'string', format: 'did', - description: 'DID of the actor who created this label', + description: 'DID of the actor who created this label.', }, uri: { type: 'string', format: 'uri', description: - 'AT URI of the record, repository (account), or other resource which this label applies to', + 'AT URI of the record, repository (account), or other resource that this label applies to.', }, cid: { type: 'string', format: 'cid', description: - "optionally, CID specifying the specific version of 'uri' resource this label applies to", + "Optionally, CID specifying the specific version of 'uri' resource this label applies to.", }, val: { type: 'string', maxLength: 128, description: - 'the short string name of the value or type of this label', + 'The short string name of the value or type of this label.', }, neg: { type: 'boolean', description: - 'if true, this is a negation label, overwriting a previous label', + 'If true, this is a negation label, overwriting a previous label.', }, cts: { type: 'string', format: 'datetime', - description: 'timestamp when this label was created', + description: 'Timestamp when this label was created.', }, }, }, @@ -1648,14 +1647,14 @@ export const schemaDict = { selfLabel: { type: 'object', description: - 'Metadata tag on an atproto record, published by the author within the record. Note -- schemas should use #selfLabels, not #selfLabel.', + 'Metadata tag on an atproto record, published by the author within the record. Note that schemas should use #selfLabels, not #selfLabel.', required: ['val'], properties: { val: { type: 'string', maxLength: 128, description: - 'the short string name of the value or type of this label', + 'The short string name of the value or type of this label.', }, }, }, @@ -1678,7 +1677,7 @@ export const schemaDict = { type: 'string', }, description: - "List of AT URI patterns to match (boolean 'OR'). Each may be a prefix (ending with '*'; will match inclusive of the string leading to '*'), or a full URI", + "List of AT URI patterns to match (boolean 'OR'). Each may be a prefix (ending with '*'; will match inclusive of the string leading to '*'), or a full URI.", }, sources: { type: 'array', @@ -1686,7 +1685,8 @@ export const schemaDict = { type: 'string', format: 'did', }, - description: 'Optional list of label sources (DIDs) to filter on', + description: + 'Optional list of label sources (DIDs) to filter on.', }, limit: { type: 'integer', @@ -1727,7 +1727,7 @@ export const schemaDict = { defs: { main: { type: 'subscription', - description: 'Subscribe to label updates', + description: 'Subscribe to label updates.', parameters: { type: 'params', properties: { @@ -1922,7 +1922,7 @@ export const schemaDict = { validate: { type: 'boolean', default: true, - description: 'Validate the records?', + description: 'Flag for validating the records.', }, writes: { type: 'array', @@ -2031,7 +2031,7 @@ export const schemaDict = { validate: { type: 'boolean', default: true, - description: 'Validate the record?', + description: 'Flag for validating the record.', }, record: { type: 'unknown', @@ -2041,7 +2041,7 @@ export const schemaDict = { type: 'string', format: 'cid', description: - 'Compare and swap with the previous commit by cid.', + 'Compare and swap with the previous commit by CID.', }, }, }, @@ -2102,13 +2102,13 @@ export const schemaDict = { type: 'string', format: 'cid', description: - 'Compare and swap with the previous record by cid.', + 'Compare and swap with the previous record by CID.', }, swapCommit: { type: 'string', format: 'cid', description: - 'Compare and swap with the previous commit by cid.', + 'Compare and swap with the previous commit by CID.', }, }, }, @@ -2278,7 +2278,7 @@ export const schemaDict = { }, reverse: { type: 'boolean', - description: 'Reverse the order of the returned records?', + description: 'Flag to reverse the order of the returned records.', }, }, }, @@ -2353,7 +2353,7 @@ export const schemaDict = { validate: { type: 'boolean', default: true, - description: 'Validate the record?', + description: 'Flag for validating the record.', }, record: { type: 'unknown', @@ -2363,13 +2363,13 @@ export const schemaDict = { type: 'string', format: 'cid', description: - 'Compare and swap with the previous record by cid.', + 'Compare and swap with the previous record by CID.', }, swapCommit: { type: 'string', format: 'cid', description: - 'Compare and swap with the previous commit by cid.', + 'Compare and swap with the previous commit by CID.', }, }, }, @@ -2583,7 +2583,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Create an app-specific password.', + description: 'Create an App Password.', input: { encoding: 'application/json', schema: { @@ -2671,7 +2671,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Create an invite code.', + description: 'Create invite codes.', input: { encoding: 'application/json', schema: { @@ -2859,7 +2859,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Delete a user account with a token and password.', + description: "Delete an actor's account with a token and password.", input: { encoding: 'application/json', schema: { @@ -2950,7 +2950,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Get all invite codes for a given account', + description: 'Get all invite codes for a given account.', parameters: { type: 'params', properties: { @@ -3030,7 +3030,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'List all app-specific passwords.', + description: 'List all App Passwords.', output: { encoding: 'application/json', schema: { @@ -3126,7 +3126,7 @@ export const schemaDict = { main: { type: 'procedure', description: - 'Request an email with a code to confirm ownership of email', + 'Request an email with a code to confirm ownership of email.', }, }, }, @@ -3248,7 +3248,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Revoke an app-specific password by name.', + description: 'Revoke an App Password by name.', input: { encoding: 'application/json', schema: { @@ -3337,7 +3337,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Gets blocks from a given repo.', + description: 'Get blocks from a given repo.', parameters: { type: 'params', required: ['did', 'cids'], @@ -3432,7 +3432,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Gets the current commit CID & revision of the repo.', + description: 'Get the current commit CID & revision of the repo.', parameters: { type: 'params', required: ['did'], @@ -3475,7 +3475,7 @@ export const schemaDict = { main: { type: 'query', description: - 'Gets blocks needed for existence or non-existence of record.', + 'Get blocks needed for existence or non-existence of record.', parameters: { type: 'params', required: ['did', 'collection', 'rkey'], @@ -3512,7 +3512,7 @@ export const schemaDict = { main: { type: 'query', description: - "Gets the did's repo, optionally catching up from a specific revision.", + "Gets the DID's repo, optionally catching up from a specific revision.", parameters: { type: 'params', required: ['did'], @@ -3540,7 +3540,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'List blob cids since some revision', + description: 'List blob CIDs since some revision.', parameters: { type: 'params', required: ['did'], @@ -3552,7 +3552,7 @@ export const schemaDict = { }, since: { type: 'string', - description: 'Optional revision of the repo to list blobs since', + description: 'Optional revision of the repo to list blobs since.', }, limit: { type: 'integer', @@ -3593,7 +3593,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'List dids and root cids of hosted repos', + description: 'List DIDs and root CIDs of hosted repos.', parameters: { type: 'params', properties: { @@ -3654,7 +3654,7 @@ export const schemaDict = { main: { type: 'procedure', description: - 'Notify a crawling service of a recent update. Often when a long break between updates causes the connection with the crawling service to break.', + 'Notify a crawling service of a recent update; often when a long break between updates causes the connection with the crawling service to break.', input: { encoding: 'application/json', schema: { @@ -3702,7 +3702,7 @@ export const schemaDict = { defs: { main: { type: 'subscription', - description: 'Subscribe to repo updates', + description: 'Subscribe to repo updates.', parameters: { type: 'params', properties: { @@ -3771,15 +3771,15 @@ export const schemaDict = { }, rev: { type: 'string', - description: 'The rev of the emitted commit', + description: 'The rev of the emitted commit.', }, since: { type: 'string', - description: 'The rev of the last emitted commit from this repo', + description: 'The rev of the last emitted commit from this repo.', }, blocks: { type: 'bytes', - description: 'CAR file containing relevant blocks', + description: 'CAR file containing relevant blocks.', maxLength: 1000000, }, ops: { @@ -3877,7 +3877,7 @@ export const schemaDict = { repoOp: { type: 'object', description: - "A repo operation, ie a write of a single record. For creates and updates, cid is the record's CID as of this operation. For deletes, it's null.", + "A repo operation, ie a write of a single record. For creates and updates, CID is the record's CID as of this operation. For deletes, it's null.", required: ['action', 'path', 'cid'], nullable: ['cid'], properties: { @@ -4164,7 +4164,7 @@ export const schemaDict = { birthDate: { type: 'string', format: 'datetime', - description: 'The birth date of the owner of the account.', + description: 'The birth date of account owner.', }, }, }, @@ -4206,7 +4206,7 @@ export const schemaDict = { properties: { sort: { type: 'string', - description: 'Sorting mode.', + description: 'Sorting mode for threads.', knownValues: ['oldest', 'newest', 'most-likes', 'random'], }, prioritizeFollowedUsers: { @@ -4250,6 +4250,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get detailed profile view of an actor.', parameters: { type: 'params', required: ['actor'], @@ -4276,6 +4277,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get detailed profile views of multiple actors.', parameters: { type: 'params', required: ['actors'], @@ -4315,8 +4317,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: - 'Get a list of actors suggested for following. Used in discovery UIs.', + description: 'Get a list of suggested actors, used for discovery.', parameters: { type: 'params', properties: { @@ -4359,6 +4360,7 @@ export const schemaDict = { defs: { main: { type: 'record', + description: 'A declaration of a profile.', key: 'literal:self', record: { type: 'object', @@ -4398,7 +4400,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Sets the private preferences attached to the account.', + description: 'Set the private preferences attached to the account.', input: { encoding: 'application/json', schema: { @@ -4427,12 +4429,12 @@ export const schemaDict = { properties: { term: { type: 'string', - description: "DEPRECATED: use 'q' instead", + description: "DEPRECATED: use 'q' instead.", }, q: { type: 'string', description: - 'search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended', + 'Search query string. Syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended.', }, limit: { type: 'integer', @@ -4473,17 +4475,17 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Find actor suggestions for a search term.', + description: 'Find actor suggestions for a prefix search term.', parameters: { type: 'params', properties: { term: { type: 'string', - description: "DEPRECATED: use 'q' instead", + description: "DEPRECATED: use 'q' instead.", }, q: { type: 'string', - description: 'search query prefix; not a full query string', + description: 'Search query prefix; not a full query string.', }, limit: { type: 'integer', @@ -4516,7 +4518,7 @@ export const schemaDict = { lexicon: 1, id: 'app.bsky.embed.external', description: - 'A representation of some externally linked content, embedded in another form of content', + 'A representation of some externally linked content, embedded in another form of content.', defs: { main: { type: 'object', @@ -4583,7 +4585,7 @@ export const schemaDict = { AppBskyEmbedImages: { lexicon: 1, id: 'app.bsky.embed.images', - description: 'A set of images embedded in some other form of content', + description: 'A set of images embedded in some other form of content.', defs: { main: { type: 'object', @@ -4672,7 +4674,7 @@ export const schemaDict = { lexicon: 1, id: 'app.bsky.embed.record', description: - 'A representation of a record embedded in another form of content', + 'A representation of a record embedded in another form of content.', defs: { main: { type: 'object', @@ -4782,7 +4784,7 @@ export const schemaDict = { lexicon: 1, id: 'app.bsky.embed.recordWithMedia', description: - 'A representation of a record embedded in another form of content, alongside other compatible embeds', + 'A representation of a record embedded in another form of content, alongside other compatible embeds.', defs: { main: { type: 'object', @@ -5150,7 +5152,7 @@ export const schemaDict = { main: { type: 'query', description: - 'Returns information about a given feed generator including TOS & offered feed URIs', + 'Get information about a feed generator, including policies and offered feed URIs.', output: { encoding: 'application/json', schema: { @@ -5205,7 +5207,7 @@ export const schemaDict = { defs: { main: { type: 'record', - description: 'A declaration of the existence of a feed generator', + description: 'A declaration of the existence of a feed generator.', key: 'any', record: { type: 'object', @@ -5256,7 +5258,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Retrieve a list of feeds created by a given actor', + description: 'Get a list of feeds created by the actor.', parameters: { type: 'params', required: ['actor'], @@ -5304,7 +5306,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'A view of the posts liked by an actor.', + description: 'Get a list of posts liked by an actor.', parameters: { type: 'params', required: ['actor'], @@ -5360,7 +5362,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: "A view of an actor's feed.", + description: "Get a view of an actor's feed.", parameters: { type: 'params', required: ['actor'], @@ -5426,7 +5428,7 @@ export const schemaDict = { main: { type: 'query', description: - "Compose and hydrate a feed from a user's selected feed generator", + "Get a hydrated feed from an actor's selected feed generator.", parameters: { type: 'params', required: ['feed'], @@ -5479,8 +5481,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: - 'Get information about a specific feed offered by a feed generator, such as its online status', + description: 'Get information about a feed generator.', parameters: { type: 'params', required: ['feed'], @@ -5519,7 +5520,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Get information about a list of feed generators', + description: 'Get information about a list of feed generators.', parameters: { type: 'params', required: ['feeds'], @@ -5558,7 +5559,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'A skeleton of a feed provided by a feed generator', + description: 'Get a skeleton of a feed provided by a feed generator.', parameters: { type: 'params', required: ['feed'], @@ -5611,6 +5612,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get the list of likes.', parameters: { type: 'params', required: ['uri'], @@ -5688,7 +5690,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'A view of a recent posts from actors in a list', + description: 'Get a view of a recent posts from actors in a list.', parameters: { type: 'params', required: ['list'], @@ -5741,6 +5743,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get posts in a thread.', parameters: { type: 'params', required: ['uri'], @@ -5794,7 +5797,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: "A view of an actor's feed.", + description: "Get a view of an actor's feed.", parameters: { type: 'params', required: ['uris'], @@ -5834,6 +5837,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get a list of reposts.', parameters: { type: 'params', required: ['uri'], @@ -5936,7 +5940,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: "A view of the user's home timeline.", + description: "Get a view of the actor's home timeline.", parameters: { type: 'params', properties: { @@ -5982,6 +5986,7 @@ export const schemaDict = { defs: { main: { type: 'record', + description: 'A declaration of a like.', key: 'tid', record: { type: 'object', @@ -6006,6 +6011,7 @@ export const schemaDict = { defs: { main: { type: 'record', + description: 'A declaration of a post.', key: 'tid', record: { type: 'object', @@ -6128,6 +6134,7 @@ export const schemaDict = { id: 'app.bsky.feed.repost', defs: { main: { + description: 'A declaration of a repost.', type: 'record', key: 'tid', record: { @@ -6153,7 +6160,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Find posts matching search criteria', + description: 'Find posts matching search criteria.', parameters: { type: 'params', required: ['q'], @@ -6161,7 +6168,7 @@ export const schemaDict = { q: { type: 'string', description: - 'search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended', + 'Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended.', }, limit: { type: 'integer', @@ -6172,7 +6179,7 @@ export const schemaDict = { cursor: { type: 'string', description: - 'optional pagination mechanism; may not necessarily allow scrolling through entire result set', + 'Optional pagination mechanism; may not necessarily allow scrolling through entire result set.', }, }, }, @@ -6188,7 +6195,7 @@ export const schemaDict = { hitsTotal: { type: 'integer', description: - 'count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits', + 'Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits.', }, posts: { type: 'array', @@ -6273,7 +6280,7 @@ export const schemaDict = { defs: { main: { type: 'record', - description: 'A block.', + description: 'A declaration of a block.', key: 'tid', record: { type: 'object', @@ -6400,12 +6407,12 @@ export const schemaDict = { modlist: { type: 'token', description: - 'A list of actors to apply an aggregate moderation action (mute/block) on', + 'A list of actors to apply an aggregate moderation action (mute/block) on.', }, curatelist: { type: 'token', description: - 'A list of actors used for curation purposes such as list feeds or interaction gating', + 'A list of actors used for curation purposes such as list feeds or interaction gating.', }, listViewerState: { type: 'object', @@ -6427,7 +6434,7 @@ export const schemaDict = { defs: { main: { type: 'record', - description: 'A social follow.', + description: 'A declaration of a social follow.', key: 'tid', record: { type: 'object', @@ -6452,7 +6459,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: "Who is the requester's account blocking?", + description: 'Get a list of who the actor is blocking.', parameters: { type: 'params', properties: { @@ -6495,7 +6502,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Who is following an actor?', + description: "Get a list of an actor's followers.", parameters: { type: 'params', required: ['actor'], @@ -6547,7 +6554,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Who is an actor following?', + description: 'Get a list of who the actor follows.', parameters: { type: 'params', required: ['actor'], @@ -6599,7 +6606,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Fetch a list of actors', + description: 'Get a list of actors.', parameters: { type: 'params', required: ['list'], @@ -6651,7 +6658,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: "Which lists is the requester's account blocking?", + description: 'Get lists that the actor is blocking.', parameters: { type: 'params', properties: { @@ -6694,7 +6701,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: "Which lists is the requester's account muting?", + description: 'Get lists that the actor is muting.', parameters: { type: 'params', properties: { @@ -6737,7 +6744,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Fetch a list of lists that belong to an actor', + description: 'Get a list of lists that belong to an actor.', parameters: { type: 'params', required: ['actor'], @@ -6785,7 +6792,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Who does the viewer mute?', + description: 'Get a list of who the actor mutes.', parameters: { type: 'params', properties: { @@ -6940,7 +6947,7 @@ export const schemaDict = { defs: { main: { type: 'record', - description: 'An item under a declared list of actors', + description: 'An item under a declared list of actors.', key: 'tid', record: { type: 'object', @@ -6969,7 +6976,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Mute an actor by did or handle.', + description: 'Mute an actor by DID or handle.', input: { encoding: 'application/json', schema: { @@ -7015,7 +7022,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Unmute an actor by did or handle.', + description: 'Unmute an actor by DID or handle.', input: { encoding: 'application/json', schema: { @@ -7061,6 +7068,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get the count of unread notifications.', parameters: { type: 'params', properties: { @@ -7091,6 +7099,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get a list of notifications.', parameters: { type: 'params', properties: { @@ -7197,7 +7206,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Register for push notifications with a service', + description: 'Register for push notifications with a service.', input: { encoding: 'application/json', schema: { @@ -7357,7 +7366,7 @@ export const schemaDict = { main: { type: 'query', description: - 'DEPRECATED: will be removed soon, please find a feed generator alternative', + 'DEPRECATED: will be removed soon. Use a feed generator alternative.', parameters: { type: 'params', properties: { @@ -7404,7 +7413,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'An unspecced view of globally popular feed generators', + description: 'An unspecced view of globally popular feed generators.', parameters: { type: 'params', properties: { @@ -7450,7 +7459,8 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'A skeleton of a timeline - UNSPECCED & WILL GO AWAY SOON', + description: + 'DEPRECATED: a skeleton of a timeline. Unspecced and will be unavailable soon.', parameters: { type: 'params', properties: { @@ -7498,7 +7508,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Backend Actors (profile) search, returning only skeleton', + description: 'Backend Actors (profile) search, returns only skeleton.', parameters: { type: 'params', required: ['q'], @@ -7506,11 +7516,11 @@ export const schemaDict = { q: { type: 'string', description: - 'search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. For typeahead search, only simple term match is supported, not full syntax', + 'Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. For typeahead search, only simple term match is supported, not full syntax.', }, typeahead: { type: 'boolean', - description: "if true, acts as fast/simple 'typeahead' query", + description: "If true, acts as fast/simple 'typeahead' query.", }, limit: { type: 'integer', @@ -7521,7 +7531,7 @@ export const schemaDict = { cursor: { type: 'string', description: - 'optional pagination mechanism; may not necessarily allow scrolling through entire result set', + 'Optional pagination mechanism; may not necessarily allow scrolling through entire result set.', }, }, }, @@ -7537,7 +7547,7 @@ export const schemaDict = { hitsTotal: { type: 'integer', description: - 'count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits', + 'Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits.', }, actors: { type: 'array', @@ -7563,7 +7573,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Backend Posts search, returning only skeleton', + description: 'Backend Posts search, returns only skeleton', parameters: { type: 'params', required: ['q'], @@ -7571,7 +7581,7 @@ export const schemaDict = { q: { type: 'string', description: - 'search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended', + 'Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended.', }, limit: { type: 'integer', @@ -7582,7 +7592,7 @@ export const schemaDict = { cursor: { type: 'string', description: - 'optional pagination mechanism; may not necessarily allow scrolling through entire result set', + 'Optional pagination mechanism; may not necessarily allow scrolling through entire result set.', }, }, }, @@ -7598,7 +7608,7 @@ export const schemaDict = { hitsTotal: { type: 'integer', description: - 'count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits', + 'Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits.', }, posts: { type: 'array', diff --git a/packages/api/src/client/types/app/bsky/actor/defs.ts b/packages/api/src/client/types/app/bsky/actor/defs.ts index 5dd765a8373..0cb22ea5797 100644 --- a/packages/api/src/client/types/app/bsky/actor/defs.ts +++ b/packages/api/src/client/types/app/bsky/actor/defs.ts @@ -169,7 +169,7 @@ export function validateSavedFeedsPref(v: unknown): ValidationResult { } export interface PersonalDetailsPref { - /** The birth date of the owner of the account. */ + /** The birth date of account owner. */ birthDate?: string [k: string]: unknown } @@ -215,7 +215,7 @@ export function validateFeedViewPref(v: unknown): ValidationResult { } export interface ThreadViewPref { - /** Sorting mode. */ + /** Sorting mode for threads. */ sort?: 'oldest' | 'newest' | 'most-likes' | 'random' | (string & {}) /** Show followed users at the top of all replies. */ prioritizeFollowedUsers?: boolean diff --git a/packages/api/src/client/types/app/bsky/actor/searchActors.ts b/packages/api/src/client/types/app/bsky/actor/searchActors.ts index 5e6527606d8..63395418e11 100644 --- a/packages/api/src/client/types/app/bsky/actor/searchActors.ts +++ b/packages/api/src/client/types/app/bsky/actor/searchActors.ts @@ -9,9 +9,9 @@ import { CID } from 'multiformats/cid' import * as AppBskyActorDefs from './defs' export interface QueryParams { - /** DEPRECATED: use 'q' instead */ + /** DEPRECATED: use 'q' instead. */ term?: string - /** search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended */ + /** Search query string. Syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. */ q?: string limit?: number cursor?: string diff --git a/packages/api/src/client/types/app/bsky/actor/searchActorsTypeahead.ts b/packages/api/src/client/types/app/bsky/actor/searchActorsTypeahead.ts index 5818d6f64ad..a91e0ce7dcd 100644 --- a/packages/api/src/client/types/app/bsky/actor/searchActorsTypeahead.ts +++ b/packages/api/src/client/types/app/bsky/actor/searchActorsTypeahead.ts @@ -9,9 +9,9 @@ import { CID } from 'multiformats/cid' import * as AppBskyActorDefs from './defs' export interface QueryParams { - /** DEPRECATED: use 'q' instead */ + /** DEPRECATED: use 'q' instead. */ term?: string - /** search query prefix; not a full query string */ + /** Search query prefix; not a full query string. */ q?: string limit?: number } diff --git a/packages/api/src/client/types/app/bsky/feed/searchPosts.ts b/packages/api/src/client/types/app/bsky/feed/searchPosts.ts index 6b8613a2e1f..c94abf40221 100644 --- a/packages/api/src/client/types/app/bsky/feed/searchPosts.ts +++ b/packages/api/src/client/types/app/bsky/feed/searchPosts.ts @@ -9,10 +9,10 @@ import { CID } from 'multiformats/cid' import * as AppBskyFeedDefs from './defs' export interface QueryParams { - /** search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended */ + /** Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. */ q: string limit?: number - /** optional pagination mechanism; may not necessarily allow scrolling through entire result set */ + /** Optional pagination mechanism; may not necessarily allow scrolling through entire result set. */ cursor?: string } @@ -20,7 +20,7 @@ export type InputSchema = undefined export interface OutputSchema { cursor?: string - /** count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits */ + /** Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits. */ hitsTotal?: number posts: AppBskyFeedDefs.PostView[] [k: string]: unknown diff --git a/packages/api/src/client/types/app/bsky/graph/defs.ts b/packages/api/src/client/types/app/bsky/graph/defs.ts index fb40758534f..19c2cc81337 100644 --- a/packages/api/src/client/types/app/bsky/graph/defs.ts +++ b/packages/api/src/client/types/app/bsky/graph/defs.ts @@ -79,9 +79,9 @@ export type ListPurpose = | 'app.bsky.graph.defs#curatelist' | (string & {}) -/** A list of actors to apply an aggregate moderation action (mute/block) on */ +/** A list of actors to apply an aggregate moderation action (mute/block) on. */ export const MODLIST = 'app.bsky.graph.defs#modlist' -/** A list of actors used for curation purposes such as list feeds or interaction gating */ +/** A list of actors used for curation purposes such as list feeds or interaction gating. */ export const CURATELIST = 'app.bsky.graph.defs#curatelist' export interface ListViewerState { diff --git a/packages/api/src/client/types/app/bsky/unspecced/searchActorsSkeleton.ts b/packages/api/src/client/types/app/bsky/unspecced/searchActorsSkeleton.ts index 7cc2729620e..aa882a0e74d 100644 --- a/packages/api/src/client/types/app/bsky/unspecced/searchActorsSkeleton.ts +++ b/packages/api/src/client/types/app/bsky/unspecced/searchActorsSkeleton.ts @@ -9,12 +9,12 @@ import { CID } from 'multiformats/cid' import * as AppBskyUnspeccedDefs from './defs' export interface QueryParams { - /** search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. For typeahead search, only simple term match is supported, not full syntax */ + /** Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. For typeahead search, only simple term match is supported, not full syntax. */ q: string - /** if true, acts as fast/simple 'typeahead' query */ + /** If true, acts as fast/simple 'typeahead' query. */ typeahead?: boolean limit?: number - /** optional pagination mechanism; may not necessarily allow scrolling through entire result set */ + /** Optional pagination mechanism; may not necessarily allow scrolling through entire result set. */ cursor?: string } @@ -22,7 +22,7 @@ export type InputSchema = undefined export interface OutputSchema { cursor?: string - /** count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits */ + /** Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits. */ hitsTotal?: number actors: AppBskyUnspeccedDefs.SkeletonSearchActor[] [k: string]: unknown diff --git a/packages/api/src/client/types/app/bsky/unspecced/searchPostsSkeleton.ts b/packages/api/src/client/types/app/bsky/unspecced/searchPostsSkeleton.ts index 07391886f8f..9d118d08518 100644 --- a/packages/api/src/client/types/app/bsky/unspecced/searchPostsSkeleton.ts +++ b/packages/api/src/client/types/app/bsky/unspecced/searchPostsSkeleton.ts @@ -9,10 +9,10 @@ import { CID } from 'multiformats/cid' import * as AppBskyUnspeccedDefs from './defs' export interface QueryParams { - /** search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended */ + /** Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. */ q: string limit?: number - /** optional pagination mechanism; may not necessarily allow scrolling through entire result set */ + /** Optional pagination mechanism; may not necessarily allow scrolling through entire result set. */ cursor?: string } @@ -20,7 +20,7 @@ export type InputSchema = undefined export interface OutputSchema { cursor?: string - /** count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits */ + /** Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits. */ hitsTotal?: number posts: AppBskyUnspeccedDefs.SkeletonSearchPost[] [k: string]: unknown diff --git a/packages/api/src/client/types/com/atproto/admin/defs.ts b/packages/api/src/client/types/com/atproto/admin/defs.ts index 5ab3e3482ec..8b8197a06d0 100644 --- a/packages/api/src/client/types/com/atproto/admin/defs.ts +++ b/packages/api/src/client/types/com/atproto/admin/defs.ts @@ -31,7 +31,7 @@ export function validateStatusAttr(v: unknown): ValidationResult { export interface ActionView { id: number action: ActionType - /** Indicates how long this action was meant to be in effect before automatically expiring. */ + /** Indicates how long this action is meant to be in effect before automatically expiring. */ durationInHours?: number subject: | RepoRef @@ -63,7 +63,7 @@ export function validateActionView(v: unknown): ValidationResult { export interface ActionViewDetail { id: number action: ActionType - /** Indicates how long this action was meant to be in effect before automatically expiring. */ + /** Indicates how long this action is meant to be in effect before automatically expiring. */ durationInHours?: number subject: | RepoView @@ -97,7 +97,7 @@ export function validateActionViewDetail(v: unknown): ValidationResult { export interface ActionViewCurrent { id: number action: ActionType - /** Indicates how long this action was meant to be in effect before automatically expiring. */ + /** Indicates how long this action is meant to be in effect before automatically expiring. */ durationInHours?: number [k: string]: unknown } diff --git a/packages/api/src/client/types/com/atproto/admin/disableAccountInvites.ts b/packages/api/src/client/types/com/atproto/admin/disableAccountInvites.ts index cf61d3026d4..7d4a59e503f 100644 --- a/packages/api/src/client/types/com/atproto/admin/disableAccountInvites.ts +++ b/packages/api/src/client/types/com/atproto/admin/disableAccountInvites.ts @@ -11,7 +11,7 @@ export interface QueryParams {} export interface InputSchema { account: string - /** Additionally add a note describing why the invites were disabled */ + /** Optional reason for disabled invites. */ note?: string [k: string]: unknown } diff --git a/packages/api/src/client/types/com/atproto/admin/enableAccountInvites.ts b/packages/api/src/client/types/com/atproto/admin/enableAccountInvites.ts index e20c3f14d49..c39d1990b26 100644 --- a/packages/api/src/client/types/com/atproto/admin/enableAccountInvites.ts +++ b/packages/api/src/client/types/com/atproto/admin/enableAccountInvites.ts @@ -11,7 +11,7 @@ export interface QueryParams {} export interface InputSchema { account: string - /** Additionally add a note describing why the invites were enabled */ + /** Optional reason for enabled invites. */ note?: string [k: string]: unknown } diff --git a/packages/api/src/client/types/com/atproto/admin/getModerationReports.ts b/packages/api/src/client/types/com/atproto/admin/getModerationReports.ts index cc6c6f00f3c..3a3e52f59b3 100644 --- a/packages/api/src/client/types/com/atproto/admin/getModerationReports.ts +++ b/packages/api/src/client/types/com/atproto/admin/getModerationReports.ts @@ -11,9 +11,9 @@ import * as ComAtprotoAdminDefs from './defs' export interface QueryParams { subject?: string ignoreSubjects?: string[] - /** Get all reports that were actioned by a specific moderator */ + /** Get all reports that were actioned by a specific moderator. */ actionedBy?: string - /** Filter reports made by one or more DIDs */ + /** Filter reports made by one or more DIDs. */ reporters?: string[] resolved?: boolean actionType?: @@ -24,7 +24,7 @@ export interface QueryParams { | (string & {}) limit?: number cursor?: string - /** Reverse the order of the returned records? when true, returns reports in chronological order */ + /** Reverse the order of the returned records. When true, returns reports in chronological order. */ reverse?: boolean } diff --git a/packages/api/src/client/types/com/atproto/admin/takeModerationAction.ts b/packages/api/src/client/types/com/atproto/admin/takeModerationAction.ts index 6e253d6b0ef..49fba249af7 100644 --- a/packages/api/src/client/types/com/atproto/admin/takeModerationAction.ts +++ b/packages/api/src/client/types/com/atproto/admin/takeModerationAction.ts @@ -25,7 +25,7 @@ export interface InputSchema { createLabelVals?: string[] negateLabelVals?: string[] reason: string - /** Indicates how long this action was meant to be in effect before automatically expiring. */ + /** Indicates how long this action is meant to be in effect before automatically expiring. */ durationInHours?: number createdBy: string [k: string]: unknown diff --git a/packages/api/src/client/types/com/atproto/label/defs.ts b/packages/api/src/client/types/com/atproto/label/defs.ts index 13f90cd80c5..54402204c61 100644 --- a/packages/api/src/client/types/com/atproto/label/defs.ts +++ b/packages/api/src/client/types/com/atproto/label/defs.ts @@ -6,19 +6,19 @@ import { isObj, hasProp } from '../../../../util' import { lexicons } from '../../../../lexicons' import { CID } from 'multiformats/cid' -/** Metadata tag on an atproto resource (eg, repo or record) */ +/** Metadata tag on an atproto resource (eg, repo or record). */ export interface Label { - /** DID of the actor who created this label */ + /** DID of the actor who created this label. */ src: string - /** AT URI of the record, repository (account), or other resource which this label applies to */ + /** AT URI of the record, repository (account), or other resource that this label applies to. */ uri: string - /** optionally, CID specifying the specific version of 'uri' resource this label applies to */ + /** Optionally, CID specifying the specific version of 'uri' resource this label applies to. */ cid?: string - /** the short string name of the value or type of this label */ + /** The short string name of the value or type of this label. */ val: string - /** if true, this is a negation label, overwriting a previous label */ + /** If true, this is a negation label, overwriting a previous label. */ neg?: boolean - /** timestamp when this label was created */ + /** Timestamp when this label was created. */ cts: string [k: string]: unknown } @@ -53,9 +53,9 @@ export function validateSelfLabels(v: unknown): ValidationResult { return lexicons.validate('com.atproto.label.defs#selfLabels', v) } -/** Metadata tag on an atproto record, published by the author within the record. Note -- schemas should use #selfLabels, not #selfLabel. */ +/** Metadata tag on an atproto record, published by the author within the record. Note that schemas should use #selfLabels, not #selfLabel. */ export interface SelfLabel { - /** the short string name of the value or type of this label */ + /** The short string name of the value or type of this label. */ val: string [k: string]: unknown } diff --git a/packages/api/src/client/types/com/atproto/label/queryLabels.ts b/packages/api/src/client/types/com/atproto/label/queryLabels.ts index 9b0a283943d..3fd08d1b2b4 100644 --- a/packages/api/src/client/types/com/atproto/label/queryLabels.ts +++ b/packages/api/src/client/types/com/atproto/label/queryLabels.ts @@ -9,9 +9,9 @@ import { CID } from 'multiformats/cid' import * as ComAtprotoLabelDefs from './defs' export interface QueryParams { - /** List of AT URI patterns to match (boolean 'OR'). Each may be a prefix (ending with '*'; will match inclusive of the string leading to '*'), or a full URI */ + /** List of AT URI patterns to match (boolean 'OR'). Each may be a prefix (ending with '*'; will match inclusive of the string leading to '*'), or a full URI. */ uriPatterns: string[] - /** Optional list of label sources (DIDs) to filter on */ + /** Optional list of label sources (DIDs) to filter on. */ sources?: string[] limit?: number cursor?: string diff --git a/packages/api/src/client/types/com/atproto/repo/applyWrites.ts b/packages/api/src/client/types/com/atproto/repo/applyWrites.ts index b25c3716e1b..f4a8a269201 100644 --- a/packages/api/src/client/types/com/atproto/repo/applyWrites.ts +++ b/packages/api/src/client/types/com/atproto/repo/applyWrites.ts @@ -12,7 +12,7 @@ export interface QueryParams {} export interface InputSchema { /** The handle or DID of the repo. */ repo: string - /** Validate the records? */ + /** Flag for validating the records. */ validate?: boolean writes: (Create | Update | Delete)[] swapCommit?: string diff --git a/packages/api/src/client/types/com/atproto/repo/createRecord.ts b/packages/api/src/client/types/com/atproto/repo/createRecord.ts index 27662fc4929..2056778c71c 100644 --- a/packages/api/src/client/types/com/atproto/repo/createRecord.ts +++ b/packages/api/src/client/types/com/atproto/repo/createRecord.ts @@ -16,11 +16,11 @@ export interface InputSchema { collection: string /** The key of the record. */ rkey?: string - /** Validate the record? */ + /** Flag for validating the record. */ validate?: boolean /** The record to create. */ record: {} - /** Compare and swap with the previous commit by cid. */ + /** Compare and swap with the previous commit by CID. */ swapCommit?: string [k: string]: unknown } diff --git a/packages/api/src/client/types/com/atproto/repo/deleteRecord.ts b/packages/api/src/client/types/com/atproto/repo/deleteRecord.ts index 4bbe1fd2341..5bf9237abbb 100644 --- a/packages/api/src/client/types/com/atproto/repo/deleteRecord.ts +++ b/packages/api/src/client/types/com/atproto/repo/deleteRecord.ts @@ -16,9 +16,9 @@ export interface InputSchema { collection: string /** The key of the record. */ rkey: string - /** Compare and swap with the previous record by cid. */ + /** Compare and swap with the previous record by CID. */ swapRecord?: string - /** Compare and swap with the previous commit by cid. */ + /** Compare and swap with the previous commit by CID. */ swapCommit?: string [k: string]: unknown } diff --git a/packages/api/src/client/types/com/atproto/repo/listRecords.ts b/packages/api/src/client/types/com/atproto/repo/listRecords.ts index 57bcf55df2a..6322c782008 100644 --- a/packages/api/src/client/types/com/atproto/repo/listRecords.ts +++ b/packages/api/src/client/types/com/atproto/repo/listRecords.ts @@ -19,7 +19,7 @@ export interface QueryParams { rkeyStart?: string /** DEPRECATED: The highest sort-ordered rkey to stop at (exclusive) */ rkeyEnd?: string - /** Reverse the order of the returned records? */ + /** Flag to reverse the order of the returned records. */ reverse?: boolean } diff --git a/packages/api/src/client/types/com/atproto/repo/putRecord.ts b/packages/api/src/client/types/com/atproto/repo/putRecord.ts index 7fbf2630b81..269ef759401 100644 --- a/packages/api/src/client/types/com/atproto/repo/putRecord.ts +++ b/packages/api/src/client/types/com/atproto/repo/putRecord.ts @@ -16,13 +16,13 @@ export interface InputSchema { collection: string /** The key of the record. */ rkey: string - /** Validate the record? */ + /** Flag for validating the record. */ validate?: boolean /** The record to write. */ record: {} - /** Compare and swap with the previous record by cid. */ + /** Compare and swap with the previous record by CID. */ swapRecord?: string | null - /** Compare and swap with the previous commit by cid. */ + /** Compare and swap with the previous commit by CID. */ swapCommit?: string [k: string]: unknown } diff --git a/packages/api/src/client/types/com/atproto/sync/listBlobs.ts b/packages/api/src/client/types/com/atproto/sync/listBlobs.ts index 72ddd99cb2d..d775ce35a6a 100644 --- a/packages/api/src/client/types/com/atproto/sync/listBlobs.ts +++ b/packages/api/src/client/types/com/atproto/sync/listBlobs.ts @@ -10,7 +10,7 @@ import { CID } from 'multiformats/cid' export interface QueryParams { /** The DID of the repo. */ did: string - /** Optional revision of the repo to list blobs since */ + /** Optional revision of the repo to list blobs since. */ since?: string limit?: number cursor?: string diff --git a/packages/api/src/client/types/com/atproto/sync/subscribeRepos.ts b/packages/api/src/client/types/com/atproto/sync/subscribeRepos.ts index f54c8d45631..a4fec035874 100644 --- a/packages/api/src/client/types/com/atproto/sync/subscribeRepos.ts +++ b/packages/api/src/client/types/com/atproto/sync/subscribeRepos.ts @@ -14,11 +14,11 @@ export interface Commit { repo: string commit: CID prev?: CID | null - /** The rev of the emitted commit */ + /** The rev of the emitted commit. */ rev: string - /** The rev of the last emitted commit from this repo */ + /** The rev of the last emitted commit from this repo. */ since: string | null - /** CAR file containing relevant blocks */ + /** CAR file containing relevant blocks. */ blocks: Uint8Array ops: RepoOp[] blobs: CID[] @@ -115,7 +115,7 @@ export function validateInfo(v: unknown): ValidationResult { return lexicons.validate('com.atproto.sync.subscribeRepos#info', v) } -/** A repo operation, ie a write of a single record. For creates and updates, cid is the record's CID as of this operation. For deletes, it's null. */ +/** A repo operation, ie a write of a single record. For creates and updates, CID is the record's CID as of this operation. For deletes, it's null. */ export interface RepoOp { action: 'create' | 'update' | 'delete' | (string & {}) path: string diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index 545d8e9d43d..14c1be9f477 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -43,7 +43,7 @@ export const schemaDict = { durationInHours: { type: 'integer', description: - 'Indicates how long this action was meant to be in effect before automatically expiring.', + 'Indicates how long this action is meant to be in effect before automatically expiring.', }, subject: { type: 'union', @@ -116,7 +116,7 @@ export const schemaDict = { durationInHours: { type: 'integer', description: - 'Indicates how long this action was meant to be in effect before automatically expiring.', + 'Indicates how long this action is meant to be in effect before automatically expiring.', }, subject: { type: 'union', @@ -184,7 +184,7 @@ export const schemaDict = { durationInHours: { type: 'integer', description: - 'Indicates how long this action was meant to be in effect before automatically expiring.', + 'Indicates how long this action is meant to be in effect before automatically expiring.', }, }, }, @@ -725,7 +725,7 @@ export const schemaDict = { main: { type: 'procedure', description: - 'Disable an account from receiving new invite codes, but does not invalidate existing codes', + 'Disable an account from receiving new invite codes, but does not invalidate existing codes.', input: { encoding: 'application/json', schema: { @@ -738,8 +738,7 @@ export const schemaDict = { }, note: { type: 'string', - description: - 'Additionally add a note describing why the invites were disabled', + description: 'Optional reason for disabled invites.', }, }, }, @@ -754,7 +753,7 @@ export const schemaDict = { main: { type: 'procedure', description: - 'Disable some set of codes and/or all codes associated with a set of users', + 'Disable some set of codes and/or all codes associated with a set of users.', input: { encoding: 'application/json', schema: { @@ -784,7 +783,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Re-enable an accounts ability to receive invite codes', + description: "Re-enable an account's ability to receive invite codes.", input: { encoding: 'application/json', schema: { @@ -797,8 +796,7 @@ export const schemaDict = { }, note: { type: 'string', - description: - 'Additionally add a note describing why the invites were enabled', + description: 'Optional reason for enabled invites.', }, }, }, @@ -812,7 +810,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'View details about an account.', + description: 'Get details about an account.', parameters: { type: 'params', required: ['did'], @@ -839,7 +837,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Admin view of invite codes', + description: 'Get an admin view of invite codes.', parameters: { type: 'params', properties: { @@ -887,7 +885,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'View details about a moderation action.', + description: 'Get details about a moderation action.', parameters: { type: 'params', required: ['id'], @@ -913,7 +911,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'List moderation actions related to a subject.', + description: 'Get a list of moderation actions related to a subject.', parameters: { type: 'params', properties: { @@ -959,7 +957,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'View details about a moderation report.', + description: 'Get details about a moderation report.', parameters: { type: 'params', required: ['id'], @@ -985,7 +983,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'List moderation reports related to a subject.', + description: 'Get moderation reports related to a subject.', parameters: { type: 'params', properties: { @@ -1002,14 +1000,14 @@ export const schemaDict = { type: 'string', format: 'did', description: - 'Get all reports that were actioned by a specific moderator', + 'Get all reports that were actioned by a specific moderator.', }, reporters: { type: 'array', items: { type: 'string', }, - description: 'Filter reports made by one or more DIDs', + description: 'Filter reports made by one or more DIDs.', }, resolved: { type: 'boolean', @@ -1035,7 +1033,7 @@ export const schemaDict = { reverse: { type: 'boolean', description: - 'Reverse the order of the returned records? when true, returns reports in chronological order', + 'Reverse the order of the returned records. When true, returns reports in chronological order.', }, }, }, @@ -1067,7 +1065,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'View details about a record.', + description: 'Get details about a record.', parameters: { type: 'params', required: ['uri'], @@ -1103,7 +1101,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'View details about a repository.', + description: 'Get details about a repository.', parameters: { type: 'params', required: ['did'], @@ -1136,7 +1134,7 @@ export const schemaDict = { main: { type: 'query', description: - 'Fetch the service-specific the admin status of a subject (account, record, or blob)', + 'Get the service-specific admin status of a subject (account, record, or blob).', parameters: { type: 'params', properties: { @@ -1309,7 +1307,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: "Send email to a user's primary email address", + description: "Send email to a user's account email address.", input: { encoding: 'application/json', schema: { @@ -1350,7 +1348,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Take a moderation action on a repo.', + description: 'Take a moderation action on an actor.', input: { encoding: 'application/json', schema: { @@ -1397,7 +1395,7 @@ export const schemaDict = { durationInHours: { type: 'integer', description: - 'Indicates how long this action was meant to be in effect before automatically expiring.', + 'Indicates how long this action is meant to be in effect before automatically expiring.', }, createdBy: { type: 'string', @@ -1427,7 +1425,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: "Administrative action to update an account's email", + description: "Administrative action to update an account's email.", input: { encoding: 'application/json', schema: { @@ -1454,7 +1452,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: "Administrative action to update an account's handle", + description: "Administrative action to update an account's handle.", input: { encoding: 'application/json', schema: { @@ -1482,7 +1480,7 @@ export const schemaDict = { main: { type: 'procedure', description: - 'Update the service-specific admin status of a subject (account, record, or blob)', + 'Update the service-specific admin status of a subject (account, record, or blob).', input: { encoding: 'application/json', schema: { @@ -1568,7 +1566,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Updates the handle of the account', + description: 'Updates the handle of the account.', input: { encoding: 'application/json', schema: { @@ -1591,41 +1589,42 @@ export const schemaDict = { defs: { label: { type: 'object', - description: 'Metadata tag on an atproto resource (eg, repo or record)', + description: + 'Metadata tag on an atproto resource (eg, repo or record).', required: ['src', 'uri', 'val', 'cts'], properties: { src: { type: 'string', format: 'did', - description: 'DID of the actor who created this label', + description: 'DID of the actor who created this label.', }, uri: { type: 'string', format: 'uri', description: - 'AT URI of the record, repository (account), or other resource which this label applies to', + 'AT URI of the record, repository (account), or other resource that this label applies to.', }, cid: { type: 'string', format: 'cid', description: - "optionally, CID specifying the specific version of 'uri' resource this label applies to", + "Optionally, CID specifying the specific version of 'uri' resource this label applies to.", }, val: { type: 'string', maxLength: 128, description: - 'the short string name of the value or type of this label', + 'The short string name of the value or type of this label.', }, neg: { type: 'boolean', description: - 'if true, this is a negation label, overwriting a previous label', + 'If true, this is a negation label, overwriting a previous label.', }, cts: { type: 'string', format: 'datetime', - description: 'timestamp when this label was created', + description: 'Timestamp when this label was created.', }, }, }, @@ -1648,14 +1647,14 @@ export const schemaDict = { selfLabel: { type: 'object', description: - 'Metadata tag on an atproto record, published by the author within the record. Note -- schemas should use #selfLabels, not #selfLabel.', + 'Metadata tag on an atproto record, published by the author within the record. Note that schemas should use #selfLabels, not #selfLabel.', required: ['val'], properties: { val: { type: 'string', maxLength: 128, description: - 'the short string name of the value or type of this label', + 'The short string name of the value or type of this label.', }, }, }, @@ -1678,7 +1677,7 @@ export const schemaDict = { type: 'string', }, description: - "List of AT URI patterns to match (boolean 'OR'). Each may be a prefix (ending with '*'; will match inclusive of the string leading to '*'), or a full URI", + "List of AT URI patterns to match (boolean 'OR'). Each may be a prefix (ending with '*'; will match inclusive of the string leading to '*'), or a full URI.", }, sources: { type: 'array', @@ -1686,7 +1685,8 @@ export const schemaDict = { type: 'string', format: 'did', }, - description: 'Optional list of label sources (DIDs) to filter on', + description: + 'Optional list of label sources (DIDs) to filter on.', }, limit: { type: 'integer', @@ -1727,7 +1727,7 @@ export const schemaDict = { defs: { main: { type: 'subscription', - description: 'Subscribe to label updates', + description: 'Subscribe to label updates.', parameters: { type: 'params', properties: { @@ -1922,7 +1922,7 @@ export const schemaDict = { validate: { type: 'boolean', default: true, - description: 'Validate the records?', + description: 'Flag for validating the records.', }, writes: { type: 'array', @@ -2031,7 +2031,7 @@ export const schemaDict = { validate: { type: 'boolean', default: true, - description: 'Validate the record?', + description: 'Flag for validating the record.', }, record: { type: 'unknown', @@ -2041,7 +2041,7 @@ export const schemaDict = { type: 'string', format: 'cid', description: - 'Compare and swap with the previous commit by cid.', + 'Compare and swap with the previous commit by CID.', }, }, }, @@ -2102,13 +2102,13 @@ export const schemaDict = { type: 'string', format: 'cid', description: - 'Compare and swap with the previous record by cid.', + 'Compare and swap with the previous record by CID.', }, swapCommit: { type: 'string', format: 'cid', description: - 'Compare and swap with the previous commit by cid.', + 'Compare and swap with the previous commit by CID.', }, }, }, @@ -2278,7 +2278,7 @@ export const schemaDict = { }, reverse: { type: 'boolean', - description: 'Reverse the order of the returned records?', + description: 'Flag to reverse the order of the returned records.', }, }, }, @@ -2353,7 +2353,7 @@ export const schemaDict = { validate: { type: 'boolean', default: true, - description: 'Validate the record?', + description: 'Flag for validating the record.', }, record: { type: 'unknown', @@ -2363,13 +2363,13 @@ export const schemaDict = { type: 'string', format: 'cid', description: - 'Compare and swap with the previous record by cid.', + 'Compare and swap with the previous record by CID.', }, swapCommit: { type: 'string', format: 'cid', description: - 'Compare and swap with the previous commit by cid.', + 'Compare and swap with the previous commit by CID.', }, }, }, @@ -2583,7 +2583,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Create an app-specific password.', + description: 'Create an App Password.', input: { encoding: 'application/json', schema: { @@ -2671,7 +2671,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Create an invite code.', + description: 'Create invite codes.', input: { encoding: 'application/json', schema: { @@ -2859,7 +2859,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Delete a user account with a token and password.', + description: "Delete an actor's account with a token and password.", input: { encoding: 'application/json', schema: { @@ -2950,7 +2950,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Get all invite codes for a given account', + description: 'Get all invite codes for a given account.', parameters: { type: 'params', properties: { @@ -3030,7 +3030,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'List all app-specific passwords.', + description: 'List all App Passwords.', output: { encoding: 'application/json', schema: { @@ -3126,7 +3126,7 @@ export const schemaDict = { main: { type: 'procedure', description: - 'Request an email with a code to confirm ownership of email', + 'Request an email with a code to confirm ownership of email.', }, }, }, @@ -3248,7 +3248,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Revoke an app-specific password by name.', + description: 'Revoke an App Password by name.', input: { encoding: 'application/json', schema: { @@ -3337,7 +3337,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Gets blocks from a given repo.', + description: 'Get blocks from a given repo.', parameters: { type: 'params', required: ['did', 'cids'], @@ -3432,7 +3432,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Gets the current commit CID & revision of the repo.', + description: 'Get the current commit CID & revision of the repo.', parameters: { type: 'params', required: ['did'], @@ -3475,7 +3475,7 @@ export const schemaDict = { main: { type: 'query', description: - 'Gets blocks needed for existence or non-existence of record.', + 'Get blocks needed for existence or non-existence of record.', parameters: { type: 'params', required: ['did', 'collection', 'rkey'], @@ -3512,7 +3512,7 @@ export const schemaDict = { main: { type: 'query', description: - "Gets the did's repo, optionally catching up from a specific revision.", + "Gets the DID's repo, optionally catching up from a specific revision.", parameters: { type: 'params', required: ['did'], @@ -3540,7 +3540,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'List blob cids since some revision', + description: 'List blob CIDs since some revision.', parameters: { type: 'params', required: ['did'], @@ -3552,7 +3552,7 @@ export const schemaDict = { }, since: { type: 'string', - description: 'Optional revision of the repo to list blobs since', + description: 'Optional revision of the repo to list blobs since.', }, limit: { type: 'integer', @@ -3593,7 +3593,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'List dids and root cids of hosted repos', + description: 'List DIDs and root CIDs of hosted repos.', parameters: { type: 'params', properties: { @@ -3654,7 +3654,7 @@ export const schemaDict = { main: { type: 'procedure', description: - 'Notify a crawling service of a recent update. Often when a long break between updates causes the connection with the crawling service to break.', + 'Notify a crawling service of a recent update; often when a long break between updates causes the connection with the crawling service to break.', input: { encoding: 'application/json', schema: { @@ -3702,7 +3702,7 @@ export const schemaDict = { defs: { main: { type: 'subscription', - description: 'Subscribe to repo updates', + description: 'Subscribe to repo updates.', parameters: { type: 'params', properties: { @@ -3771,15 +3771,15 @@ export const schemaDict = { }, rev: { type: 'string', - description: 'The rev of the emitted commit', + description: 'The rev of the emitted commit.', }, since: { type: 'string', - description: 'The rev of the last emitted commit from this repo', + description: 'The rev of the last emitted commit from this repo.', }, blocks: { type: 'bytes', - description: 'CAR file containing relevant blocks', + description: 'CAR file containing relevant blocks.', maxLength: 1000000, }, ops: { @@ -3877,7 +3877,7 @@ export const schemaDict = { repoOp: { type: 'object', description: - "A repo operation, ie a write of a single record. For creates and updates, cid is the record's CID as of this operation. For deletes, it's null.", + "A repo operation, ie a write of a single record. For creates and updates, CID is the record's CID as of this operation. For deletes, it's null.", required: ['action', 'path', 'cid'], nullable: ['cid'], properties: { @@ -4164,7 +4164,7 @@ export const schemaDict = { birthDate: { type: 'string', format: 'datetime', - description: 'The birth date of the owner of the account.', + description: 'The birth date of account owner.', }, }, }, @@ -4206,7 +4206,7 @@ export const schemaDict = { properties: { sort: { type: 'string', - description: 'Sorting mode.', + description: 'Sorting mode for threads.', knownValues: ['oldest', 'newest', 'most-likes', 'random'], }, prioritizeFollowedUsers: { @@ -4250,6 +4250,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get detailed profile view of an actor.', parameters: { type: 'params', required: ['actor'], @@ -4276,6 +4277,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get detailed profile views of multiple actors.', parameters: { type: 'params', required: ['actors'], @@ -4315,8 +4317,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: - 'Get a list of actors suggested for following. Used in discovery UIs.', + description: 'Get a list of suggested actors, used for discovery.', parameters: { type: 'params', properties: { @@ -4359,6 +4360,7 @@ export const schemaDict = { defs: { main: { type: 'record', + description: 'A declaration of a profile.', key: 'literal:self', record: { type: 'object', @@ -4398,7 +4400,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Sets the private preferences attached to the account.', + description: 'Set the private preferences attached to the account.', input: { encoding: 'application/json', schema: { @@ -4427,12 +4429,12 @@ export const schemaDict = { properties: { term: { type: 'string', - description: "DEPRECATED: use 'q' instead", + description: "DEPRECATED: use 'q' instead.", }, q: { type: 'string', description: - 'search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended', + 'Search query string. Syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended.', }, limit: { type: 'integer', @@ -4473,17 +4475,17 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Find actor suggestions for a search term.', + description: 'Find actor suggestions for a prefix search term.', parameters: { type: 'params', properties: { term: { type: 'string', - description: "DEPRECATED: use 'q' instead", + description: "DEPRECATED: use 'q' instead.", }, q: { type: 'string', - description: 'search query prefix; not a full query string', + description: 'Search query prefix; not a full query string.', }, limit: { type: 'integer', @@ -4516,7 +4518,7 @@ export const schemaDict = { lexicon: 1, id: 'app.bsky.embed.external', description: - 'A representation of some externally linked content, embedded in another form of content', + 'A representation of some externally linked content, embedded in another form of content.', defs: { main: { type: 'object', @@ -4583,7 +4585,7 @@ export const schemaDict = { AppBskyEmbedImages: { lexicon: 1, id: 'app.bsky.embed.images', - description: 'A set of images embedded in some other form of content', + description: 'A set of images embedded in some other form of content.', defs: { main: { type: 'object', @@ -4672,7 +4674,7 @@ export const schemaDict = { lexicon: 1, id: 'app.bsky.embed.record', description: - 'A representation of a record embedded in another form of content', + 'A representation of a record embedded in another form of content.', defs: { main: { type: 'object', @@ -4782,7 +4784,7 @@ export const schemaDict = { lexicon: 1, id: 'app.bsky.embed.recordWithMedia', description: - 'A representation of a record embedded in another form of content, alongside other compatible embeds', + 'A representation of a record embedded in another form of content, alongside other compatible embeds.', defs: { main: { type: 'object', @@ -5150,7 +5152,7 @@ export const schemaDict = { main: { type: 'query', description: - 'Returns information about a given feed generator including TOS & offered feed URIs', + 'Get information about a feed generator, including policies and offered feed URIs.', output: { encoding: 'application/json', schema: { @@ -5205,7 +5207,7 @@ export const schemaDict = { defs: { main: { type: 'record', - description: 'A declaration of the existence of a feed generator', + description: 'A declaration of the existence of a feed generator.', key: 'any', record: { type: 'object', @@ -5256,7 +5258,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Retrieve a list of feeds created by a given actor', + description: 'Get a list of feeds created by the actor.', parameters: { type: 'params', required: ['actor'], @@ -5304,7 +5306,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'A view of the posts liked by an actor.', + description: 'Get a list of posts liked by an actor.', parameters: { type: 'params', required: ['actor'], @@ -5360,7 +5362,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: "A view of an actor's feed.", + description: "Get a view of an actor's feed.", parameters: { type: 'params', required: ['actor'], @@ -5426,7 +5428,7 @@ export const schemaDict = { main: { type: 'query', description: - "Compose and hydrate a feed from a user's selected feed generator", + "Get a hydrated feed from an actor's selected feed generator.", parameters: { type: 'params', required: ['feed'], @@ -5479,8 +5481,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: - 'Get information about a specific feed offered by a feed generator, such as its online status', + description: 'Get information about a feed generator.', parameters: { type: 'params', required: ['feed'], @@ -5519,7 +5520,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Get information about a list of feed generators', + description: 'Get information about a list of feed generators.', parameters: { type: 'params', required: ['feeds'], @@ -5558,7 +5559,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'A skeleton of a feed provided by a feed generator', + description: 'Get a skeleton of a feed provided by a feed generator.', parameters: { type: 'params', required: ['feed'], @@ -5611,6 +5612,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get the list of likes.', parameters: { type: 'params', required: ['uri'], @@ -5688,7 +5690,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'A view of a recent posts from actors in a list', + description: 'Get a view of a recent posts from actors in a list.', parameters: { type: 'params', required: ['list'], @@ -5741,6 +5743,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get posts in a thread.', parameters: { type: 'params', required: ['uri'], @@ -5794,7 +5797,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: "A view of an actor's feed.", + description: "Get a view of an actor's feed.", parameters: { type: 'params', required: ['uris'], @@ -5834,6 +5837,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get a list of reposts.', parameters: { type: 'params', required: ['uri'], @@ -5936,7 +5940,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: "A view of the user's home timeline.", + description: "Get a view of the actor's home timeline.", parameters: { type: 'params', properties: { @@ -5982,6 +5986,7 @@ export const schemaDict = { defs: { main: { type: 'record', + description: 'A declaration of a like.', key: 'tid', record: { type: 'object', @@ -6006,6 +6011,7 @@ export const schemaDict = { defs: { main: { type: 'record', + description: 'A declaration of a post.', key: 'tid', record: { type: 'object', @@ -6128,6 +6134,7 @@ export const schemaDict = { id: 'app.bsky.feed.repost', defs: { main: { + description: 'A declaration of a repost.', type: 'record', key: 'tid', record: { @@ -6153,7 +6160,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Find posts matching search criteria', + description: 'Find posts matching search criteria.', parameters: { type: 'params', required: ['q'], @@ -6161,7 +6168,7 @@ export const schemaDict = { q: { type: 'string', description: - 'search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended', + 'Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended.', }, limit: { type: 'integer', @@ -6172,7 +6179,7 @@ export const schemaDict = { cursor: { type: 'string', description: - 'optional pagination mechanism; may not necessarily allow scrolling through entire result set', + 'Optional pagination mechanism; may not necessarily allow scrolling through entire result set.', }, }, }, @@ -6188,7 +6195,7 @@ export const schemaDict = { hitsTotal: { type: 'integer', description: - 'count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits', + 'Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits.', }, posts: { type: 'array', @@ -6273,7 +6280,7 @@ export const schemaDict = { defs: { main: { type: 'record', - description: 'A block.', + description: 'A declaration of a block.', key: 'tid', record: { type: 'object', @@ -6400,12 +6407,12 @@ export const schemaDict = { modlist: { type: 'token', description: - 'A list of actors to apply an aggregate moderation action (mute/block) on', + 'A list of actors to apply an aggregate moderation action (mute/block) on.', }, curatelist: { type: 'token', description: - 'A list of actors used for curation purposes such as list feeds or interaction gating', + 'A list of actors used for curation purposes such as list feeds or interaction gating.', }, listViewerState: { type: 'object', @@ -6427,7 +6434,7 @@ export const schemaDict = { defs: { main: { type: 'record', - description: 'A social follow.', + description: 'A declaration of a social follow.', key: 'tid', record: { type: 'object', @@ -6452,7 +6459,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: "Who is the requester's account blocking?", + description: 'Get a list of who the actor is blocking.', parameters: { type: 'params', properties: { @@ -6495,7 +6502,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Who is following an actor?', + description: "Get a list of an actor's followers.", parameters: { type: 'params', required: ['actor'], @@ -6547,7 +6554,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Who is an actor following?', + description: 'Get a list of who the actor follows.', parameters: { type: 'params', required: ['actor'], @@ -6599,7 +6606,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Fetch a list of actors', + description: 'Get a list of actors.', parameters: { type: 'params', required: ['list'], @@ -6651,7 +6658,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: "Which lists is the requester's account blocking?", + description: 'Get lists that the actor is blocking.', parameters: { type: 'params', properties: { @@ -6694,7 +6701,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: "Which lists is the requester's account muting?", + description: 'Get lists that the actor is muting.', parameters: { type: 'params', properties: { @@ -6737,7 +6744,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Fetch a list of lists that belong to an actor', + description: 'Get a list of lists that belong to an actor.', parameters: { type: 'params', required: ['actor'], @@ -6785,7 +6792,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Who does the viewer mute?', + description: 'Get a list of who the actor mutes.', parameters: { type: 'params', properties: { @@ -6940,7 +6947,7 @@ export const schemaDict = { defs: { main: { type: 'record', - description: 'An item under a declared list of actors', + description: 'An item under a declared list of actors.', key: 'tid', record: { type: 'object', @@ -6969,7 +6976,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Mute an actor by did or handle.', + description: 'Mute an actor by DID or handle.', input: { encoding: 'application/json', schema: { @@ -7015,7 +7022,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Unmute an actor by did or handle.', + description: 'Unmute an actor by DID or handle.', input: { encoding: 'application/json', schema: { @@ -7061,6 +7068,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get the count of unread notifications.', parameters: { type: 'params', properties: { @@ -7091,6 +7099,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get a list of notifications.', parameters: { type: 'params', properties: { @@ -7197,7 +7206,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Register for push notifications with a service', + description: 'Register for push notifications with a service.', input: { encoding: 'application/json', schema: { @@ -7357,7 +7366,7 @@ export const schemaDict = { main: { type: 'query', description: - 'DEPRECATED: will be removed soon, please find a feed generator alternative', + 'DEPRECATED: will be removed soon. Use a feed generator alternative.', parameters: { type: 'params', properties: { @@ -7404,7 +7413,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'An unspecced view of globally popular feed generators', + description: 'An unspecced view of globally popular feed generators.', parameters: { type: 'params', properties: { @@ -7450,7 +7459,8 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'A skeleton of a timeline - UNSPECCED & WILL GO AWAY SOON', + description: + 'DEPRECATED: a skeleton of a timeline. Unspecced and will be unavailable soon.', parameters: { type: 'params', properties: { @@ -7498,7 +7508,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Backend Actors (profile) search, returning only skeleton', + description: 'Backend Actors (profile) search, returns only skeleton.', parameters: { type: 'params', required: ['q'], @@ -7506,11 +7516,11 @@ export const schemaDict = { q: { type: 'string', description: - 'search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. For typeahead search, only simple term match is supported, not full syntax', + 'Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. For typeahead search, only simple term match is supported, not full syntax.', }, typeahead: { type: 'boolean', - description: "if true, acts as fast/simple 'typeahead' query", + description: "If true, acts as fast/simple 'typeahead' query.", }, limit: { type: 'integer', @@ -7521,7 +7531,7 @@ export const schemaDict = { cursor: { type: 'string', description: - 'optional pagination mechanism; may not necessarily allow scrolling through entire result set', + 'Optional pagination mechanism; may not necessarily allow scrolling through entire result set.', }, }, }, @@ -7537,7 +7547,7 @@ export const schemaDict = { hitsTotal: { type: 'integer', description: - 'count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits', + 'Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits.', }, actors: { type: 'array', @@ -7563,7 +7573,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Backend Posts search, returning only skeleton', + description: 'Backend Posts search, returns only skeleton', parameters: { type: 'params', required: ['q'], @@ -7571,7 +7581,7 @@ export const schemaDict = { q: { type: 'string', description: - 'search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended', + 'Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended.', }, limit: { type: 'integer', @@ -7582,7 +7592,7 @@ export const schemaDict = { cursor: { type: 'string', description: - 'optional pagination mechanism; may not necessarily allow scrolling through entire result set', + 'Optional pagination mechanism; may not necessarily allow scrolling through entire result set.', }, }, }, @@ -7598,7 +7608,7 @@ export const schemaDict = { hitsTotal: { type: 'integer', description: - 'count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits', + 'Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits.', }, posts: { type: 'array', diff --git a/packages/bsky/src/lexicon/types/app/bsky/actor/defs.ts b/packages/bsky/src/lexicon/types/app/bsky/actor/defs.ts index 171f5c5ef48..c20177ca50e 100644 --- a/packages/bsky/src/lexicon/types/app/bsky/actor/defs.ts +++ b/packages/bsky/src/lexicon/types/app/bsky/actor/defs.ts @@ -169,7 +169,7 @@ export function validateSavedFeedsPref(v: unknown): ValidationResult { } export interface PersonalDetailsPref { - /** The birth date of the owner of the account. */ + /** The birth date of account owner. */ birthDate?: string [k: string]: unknown } @@ -215,7 +215,7 @@ export function validateFeedViewPref(v: unknown): ValidationResult { } export interface ThreadViewPref { - /** Sorting mode. */ + /** Sorting mode for threads. */ sort?: 'oldest' | 'newest' | 'most-likes' | 'random' | (string & {}) /** Show followed users at the top of all replies. */ prioritizeFollowedUsers?: boolean diff --git a/packages/bsky/src/lexicon/types/app/bsky/actor/searchActors.ts b/packages/bsky/src/lexicon/types/app/bsky/actor/searchActors.ts index 0222f3658da..f072b8a4d04 100644 --- a/packages/bsky/src/lexicon/types/app/bsky/actor/searchActors.ts +++ b/packages/bsky/src/lexicon/types/app/bsky/actor/searchActors.ts @@ -10,9 +10,9 @@ import { HandlerAuth } from '@atproto/xrpc-server' import * as AppBskyActorDefs from './defs' export interface QueryParams { - /** DEPRECATED: use 'q' instead */ + /** DEPRECATED: use 'q' instead. */ term?: string - /** search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended */ + /** Search query string. Syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. */ q?: string limit: number cursor?: string diff --git a/packages/bsky/src/lexicon/types/app/bsky/actor/searchActorsTypeahead.ts b/packages/bsky/src/lexicon/types/app/bsky/actor/searchActorsTypeahead.ts index ba0d62444ce..0cf56753db2 100644 --- a/packages/bsky/src/lexicon/types/app/bsky/actor/searchActorsTypeahead.ts +++ b/packages/bsky/src/lexicon/types/app/bsky/actor/searchActorsTypeahead.ts @@ -10,9 +10,9 @@ import { HandlerAuth } from '@atproto/xrpc-server' import * as AppBskyActorDefs from './defs' export interface QueryParams { - /** DEPRECATED: use 'q' instead */ + /** DEPRECATED: use 'q' instead. */ term?: string - /** search query prefix; not a full query string */ + /** Search query prefix; not a full query string. */ q?: string limit: number } diff --git a/packages/bsky/src/lexicon/types/app/bsky/feed/searchPosts.ts b/packages/bsky/src/lexicon/types/app/bsky/feed/searchPosts.ts index 6b5fe08e467..36ac7cbb67d 100644 --- a/packages/bsky/src/lexicon/types/app/bsky/feed/searchPosts.ts +++ b/packages/bsky/src/lexicon/types/app/bsky/feed/searchPosts.ts @@ -10,10 +10,10 @@ import { HandlerAuth } from '@atproto/xrpc-server' import * as AppBskyFeedDefs from './defs' export interface QueryParams { - /** search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended */ + /** Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. */ q: string limit: number - /** optional pagination mechanism; may not necessarily allow scrolling through entire result set */ + /** Optional pagination mechanism; may not necessarily allow scrolling through entire result set. */ cursor?: string } @@ -21,7 +21,7 @@ export type InputSchema = undefined export interface OutputSchema { cursor?: string - /** count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits */ + /** Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits. */ hitsTotal?: number posts: AppBskyFeedDefs.PostView[] [k: string]: unknown diff --git a/packages/bsky/src/lexicon/types/app/bsky/graph/defs.ts b/packages/bsky/src/lexicon/types/app/bsky/graph/defs.ts index 121d9db200a..f6c7cb7d77d 100644 --- a/packages/bsky/src/lexicon/types/app/bsky/graph/defs.ts +++ b/packages/bsky/src/lexicon/types/app/bsky/graph/defs.ts @@ -79,9 +79,9 @@ export type ListPurpose = | 'app.bsky.graph.defs#curatelist' | (string & {}) -/** A list of actors to apply an aggregate moderation action (mute/block) on */ +/** A list of actors to apply an aggregate moderation action (mute/block) on. */ export const MODLIST = 'app.bsky.graph.defs#modlist' -/** A list of actors used for curation purposes such as list feeds or interaction gating */ +/** A list of actors used for curation purposes such as list feeds or interaction gating. */ export const CURATELIST = 'app.bsky.graph.defs#curatelist' export interface ListViewerState { diff --git a/packages/bsky/src/lexicon/types/app/bsky/unspecced/searchActorsSkeleton.ts b/packages/bsky/src/lexicon/types/app/bsky/unspecced/searchActorsSkeleton.ts index 2cf59bf86a9..5c45b9fb622 100644 --- a/packages/bsky/src/lexicon/types/app/bsky/unspecced/searchActorsSkeleton.ts +++ b/packages/bsky/src/lexicon/types/app/bsky/unspecced/searchActorsSkeleton.ts @@ -10,12 +10,12 @@ import { HandlerAuth } from '@atproto/xrpc-server' import * as AppBskyUnspeccedDefs from './defs' export interface QueryParams { - /** search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. For typeahead search, only simple term match is supported, not full syntax */ + /** Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. For typeahead search, only simple term match is supported, not full syntax. */ q: string - /** if true, acts as fast/simple 'typeahead' query */ + /** If true, acts as fast/simple 'typeahead' query. */ typeahead?: boolean limit: number - /** optional pagination mechanism; may not necessarily allow scrolling through entire result set */ + /** Optional pagination mechanism; may not necessarily allow scrolling through entire result set. */ cursor?: string } @@ -23,7 +23,7 @@ export type InputSchema = undefined export interface OutputSchema { cursor?: string - /** count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits */ + /** Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits. */ hitsTotal?: number actors: AppBskyUnspeccedDefs.SkeletonSearchActor[] [k: string]: unknown diff --git a/packages/bsky/src/lexicon/types/app/bsky/unspecced/searchPostsSkeleton.ts b/packages/bsky/src/lexicon/types/app/bsky/unspecced/searchPostsSkeleton.ts index df990d2c5c6..15532087b82 100644 --- a/packages/bsky/src/lexicon/types/app/bsky/unspecced/searchPostsSkeleton.ts +++ b/packages/bsky/src/lexicon/types/app/bsky/unspecced/searchPostsSkeleton.ts @@ -10,10 +10,10 @@ import { HandlerAuth } from '@atproto/xrpc-server' import * as AppBskyUnspeccedDefs from './defs' export interface QueryParams { - /** search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended */ + /** Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. */ q: string limit: number - /** optional pagination mechanism; may not necessarily allow scrolling through entire result set */ + /** Optional pagination mechanism; may not necessarily allow scrolling through entire result set. */ cursor?: string } @@ -21,7 +21,7 @@ export type InputSchema = undefined export interface OutputSchema { cursor?: string - /** count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits */ + /** Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits. */ hitsTotal?: number posts: AppBskyUnspeccedDefs.SkeletonSearchPost[] [k: string]: unknown diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts index 7c74f6b8b98..8a21c42119e 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts @@ -31,7 +31,7 @@ export function validateStatusAttr(v: unknown): ValidationResult { export interface ActionView { id: number action: ActionType - /** Indicates how long this action was meant to be in effect before automatically expiring. */ + /** Indicates how long this action is meant to be in effect before automatically expiring. */ durationInHours?: number subject: | RepoRef @@ -63,7 +63,7 @@ export function validateActionView(v: unknown): ValidationResult { export interface ActionViewDetail { id: number action: ActionType - /** Indicates how long this action was meant to be in effect before automatically expiring. */ + /** Indicates how long this action is meant to be in effect before automatically expiring. */ durationInHours?: number subject: | RepoView @@ -97,7 +97,7 @@ export function validateActionViewDetail(v: unknown): ValidationResult { export interface ActionViewCurrent { id: number action: ActionType - /** Indicates how long this action was meant to be in effect before automatically expiring. */ + /** Indicates how long this action is meant to be in effect before automatically expiring. */ durationInHours?: number [k: string]: unknown } diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/disableAccountInvites.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/disableAccountInvites.ts index 051fabb65e1..62864923dfd 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/disableAccountInvites.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/disableAccountInvites.ts @@ -12,7 +12,7 @@ export interface QueryParams {} export interface InputSchema { account: string - /** Additionally add a note describing why the invites were disabled */ + /** Optional reason for disabled invites. */ note?: string [k: string]: unknown } diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/enableAccountInvites.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/enableAccountInvites.ts index 4a26d302333..fb3aa8b8375 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/enableAccountInvites.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/enableAccountInvites.ts @@ -12,7 +12,7 @@ export interface QueryParams {} export interface InputSchema { account: string - /** Additionally add a note describing why the invites were enabled */ + /** Optional reason for enabled invites. */ note?: string [k: string]: unknown } diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/getModerationReports.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/getModerationReports.ts index d50af44c757..b80811cf213 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/getModerationReports.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/getModerationReports.ts @@ -12,9 +12,9 @@ import * as ComAtprotoAdminDefs from './defs' export interface QueryParams { subject?: string ignoreSubjects?: string[] - /** Get all reports that were actioned by a specific moderator */ + /** Get all reports that were actioned by a specific moderator. */ actionedBy?: string - /** Filter reports made by one or more DIDs */ + /** Filter reports made by one or more DIDs. */ reporters?: string[] resolved?: boolean actionType?: @@ -25,7 +25,7 @@ export interface QueryParams { | (string & {}) limit: number cursor?: string - /** Reverse the order of the returned records? when true, returns reports in chronological order */ + /** Reverse the order of the returned records. When true, returns reports in chronological order. */ reverse?: boolean } diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/takeModerationAction.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/takeModerationAction.ts index fbbf14dff0f..33877d90d11 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/takeModerationAction.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/takeModerationAction.ts @@ -26,7 +26,7 @@ export interface InputSchema { createLabelVals?: string[] negateLabelVals?: string[] reason: string - /** Indicates how long this action was meant to be in effect before automatically expiring. */ + /** Indicates how long this action is meant to be in effect before automatically expiring. */ durationInHours?: number createdBy: string [k: string]: unknown diff --git a/packages/bsky/src/lexicon/types/com/atproto/label/defs.ts b/packages/bsky/src/lexicon/types/com/atproto/label/defs.ts index a01ad78e254..7268650129a 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/label/defs.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/label/defs.ts @@ -6,19 +6,19 @@ import { lexicons } from '../../../../lexicons' import { isObj, hasProp } from '../../../../util' import { CID } from 'multiformats/cid' -/** Metadata tag on an atproto resource (eg, repo or record) */ +/** Metadata tag on an atproto resource (eg, repo or record). */ export interface Label { - /** DID of the actor who created this label */ + /** DID of the actor who created this label. */ src: string - /** AT URI of the record, repository (account), or other resource which this label applies to */ + /** AT URI of the record, repository (account), or other resource that this label applies to. */ uri: string - /** optionally, CID specifying the specific version of 'uri' resource this label applies to */ + /** Optionally, CID specifying the specific version of 'uri' resource this label applies to. */ cid?: string - /** the short string name of the value or type of this label */ + /** The short string name of the value or type of this label. */ val: string - /** if true, this is a negation label, overwriting a previous label */ + /** If true, this is a negation label, overwriting a previous label. */ neg?: boolean - /** timestamp when this label was created */ + /** Timestamp when this label was created. */ cts: string [k: string]: unknown } @@ -53,9 +53,9 @@ export function validateSelfLabels(v: unknown): ValidationResult { return lexicons.validate('com.atproto.label.defs#selfLabels', v) } -/** Metadata tag on an atproto record, published by the author within the record. Note -- schemas should use #selfLabels, not #selfLabel. */ +/** Metadata tag on an atproto record, published by the author within the record. Note that schemas should use #selfLabels, not #selfLabel. */ export interface SelfLabel { - /** the short string name of the value or type of this label */ + /** The short string name of the value or type of this label. */ val: string [k: string]: unknown } diff --git a/packages/bsky/src/lexicon/types/com/atproto/label/queryLabels.ts b/packages/bsky/src/lexicon/types/com/atproto/label/queryLabels.ts index 72cf5c52be6..1d7f8a4def5 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/label/queryLabels.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/label/queryLabels.ts @@ -10,9 +10,9 @@ import { HandlerAuth } from '@atproto/xrpc-server' import * as ComAtprotoLabelDefs from './defs' export interface QueryParams { - /** List of AT URI patterns to match (boolean 'OR'). Each may be a prefix (ending with '*'; will match inclusive of the string leading to '*'), or a full URI */ + /** List of AT URI patterns to match (boolean 'OR'). Each may be a prefix (ending with '*'; will match inclusive of the string leading to '*'), or a full URI. */ uriPatterns: string[] - /** Optional list of label sources (DIDs) to filter on */ + /** Optional list of label sources (DIDs) to filter on. */ sources?: string[] limit: number cursor?: string diff --git a/packages/bsky/src/lexicon/types/com/atproto/repo/applyWrites.ts b/packages/bsky/src/lexicon/types/com/atproto/repo/applyWrites.ts index 53f2972e116..61d1e7c28e4 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/repo/applyWrites.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/repo/applyWrites.ts @@ -13,7 +13,7 @@ export interface QueryParams {} export interface InputSchema { /** The handle or DID of the repo. */ repo: string - /** Validate the records? */ + /** Flag for validating the records. */ validate: boolean writes: (Create | Update | Delete)[] swapCommit?: string diff --git a/packages/bsky/src/lexicon/types/com/atproto/repo/createRecord.ts b/packages/bsky/src/lexicon/types/com/atproto/repo/createRecord.ts index e069f8caf74..df8c5d9e600 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/repo/createRecord.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/repo/createRecord.ts @@ -17,11 +17,11 @@ export interface InputSchema { collection: string /** The key of the record. */ rkey?: string - /** Validate the record? */ + /** Flag for validating the record. */ validate: boolean /** The record to create. */ record: {} - /** Compare and swap with the previous commit by cid. */ + /** Compare and swap with the previous commit by CID. */ swapCommit?: string [k: string]: unknown } diff --git a/packages/bsky/src/lexicon/types/com/atproto/repo/deleteRecord.ts b/packages/bsky/src/lexicon/types/com/atproto/repo/deleteRecord.ts index 5ee016cbed1..f45118a3769 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/repo/deleteRecord.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/repo/deleteRecord.ts @@ -17,9 +17,9 @@ export interface InputSchema { collection: string /** The key of the record. */ rkey: string - /** Compare and swap with the previous record by cid. */ + /** Compare and swap with the previous record by CID. */ swapRecord?: string - /** Compare and swap with the previous commit by cid. */ + /** Compare and swap with the previous commit by CID. */ swapCommit?: string [k: string]: unknown } diff --git a/packages/bsky/src/lexicon/types/com/atproto/repo/listRecords.ts b/packages/bsky/src/lexicon/types/com/atproto/repo/listRecords.ts index e58d9714e33..a6cf6abd1f3 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/repo/listRecords.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/repo/listRecords.ts @@ -20,7 +20,7 @@ export interface QueryParams { rkeyStart?: string /** DEPRECATED: The highest sort-ordered rkey to stop at (exclusive) */ rkeyEnd?: string - /** Reverse the order of the returned records? */ + /** Flag to reverse the order of the returned records. */ reverse?: boolean } diff --git a/packages/bsky/src/lexicon/types/com/atproto/repo/putRecord.ts b/packages/bsky/src/lexicon/types/com/atproto/repo/putRecord.ts index 364eb59f6f1..f10f773c1c4 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/repo/putRecord.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/repo/putRecord.ts @@ -17,13 +17,13 @@ export interface InputSchema { collection: string /** The key of the record. */ rkey: string - /** Validate the record? */ + /** Flag for validating the record. */ validate: boolean /** The record to write. */ record: {} - /** Compare and swap with the previous record by cid. */ + /** Compare and swap with the previous record by CID. */ swapRecord?: string | null - /** Compare and swap with the previous commit by cid. */ + /** Compare and swap with the previous commit by CID. */ swapCommit?: string [k: string]: unknown } diff --git a/packages/bsky/src/lexicon/types/com/atproto/sync/listBlobs.ts b/packages/bsky/src/lexicon/types/com/atproto/sync/listBlobs.ts index 936b08a69f8..b397bb3b3df 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/sync/listBlobs.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/sync/listBlobs.ts @@ -11,7 +11,7 @@ import { HandlerAuth } from '@atproto/xrpc-server' export interface QueryParams { /** The DID of the repo. */ did: string - /** Optional revision of the repo to list blobs since */ + /** Optional revision of the repo to list blobs since. */ since?: string limit: number cursor?: string diff --git a/packages/bsky/src/lexicon/types/com/atproto/sync/subscribeRepos.ts b/packages/bsky/src/lexicon/types/com/atproto/sync/subscribeRepos.ts index ae9cf01f8f2..fb334778bf6 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/sync/subscribeRepos.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/sync/subscribeRepos.ts @@ -39,11 +39,11 @@ export interface Commit { repo: string commit: CID prev?: CID | null - /** The rev of the emitted commit */ + /** The rev of the emitted commit. */ rev: string - /** The rev of the last emitted commit from this repo */ + /** The rev of the last emitted commit from this repo. */ since: string | null - /** CAR file containing relevant blocks */ + /** CAR file containing relevant blocks. */ blocks: Uint8Array ops: RepoOp[] blobs: CID[] @@ -140,7 +140,7 @@ export function validateInfo(v: unknown): ValidationResult { return lexicons.validate('com.atproto.sync.subscribeRepos#info', v) } -/** A repo operation, ie a write of a single record. For creates and updates, cid is the record's CID as of this operation. For deletes, it's null. */ +/** A repo operation, ie a write of a single record. For creates and updates, CID is the record's CID as of this operation. For deletes, it's null. */ export interface RepoOp { action: 'create' | 'update' | 'delete' | (string & {}) path: string diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index 545d8e9d43d..14c1be9f477 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -43,7 +43,7 @@ export const schemaDict = { durationInHours: { type: 'integer', description: - 'Indicates how long this action was meant to be in effect before automatically expiring.', + 'Indicates how long this action is meant to be in effect before automatically expiring.', }, subject: { type: 'union', @@ -116,7 +116,7 @@ export const schemaDict = { durationInHours: { type: 'integer', description: - 'Indicates how long this action was meant to be in effect before automatically expiring.', + 'Indicates how long this action is meant to be in effect before automatically expiring.', }, subject: { type: 'union', @@ -184,7 +184,7 @@ export const schemaDict = { durationInHours: { type: 'integer', description: - 'Indicates how long this action was meant to be in effect before automatically expiring.', + 'Indicates how long this action is meant to be in effect before automatically expiring.', }, }, }, @@ -725,7 +725,7 @@ export const schemaDict = { main: { type: 'procedure', description: - 'Disable an account from receiving new invite codes, but does not invalidate existing codes', + 'Disable an account from receiving new invite codes, but does not invalidate existing codes.', input: { encoding: 'application/json', schema: { @@ -738,8 +738,7 @@ export const schemaDict = { }, note: { type: 'string', - description: - 'Additionally add a note describing why the invites were disabled', + description: 'Optional reason for disabled invites.', }, }, }, @@ -754,7 +753,7 @@ export const schemaDict = { main: { type: 'procedure', description: - 'Disable some set of codes and/or all codes associated with a set of users', + 'Disable some set of codes and/or all codes associated with a set of users.', input: { encoding: 'application/json', schema: { @@ -784,7 +783,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Re-enable an accounts ability to receive invite codes', + description: "Re-enable an account's ability to receive invite codes.", input: { encoding: 'application/json', schema: { @@ -797,8 +796,7 @@ export const schemaDict = { }, note: { type: 'string', - description: - 'Additionally add a note describing why the invites were enabled', + description: 'Optional reason for enabled invites.', }, }, }, @@ -812,7 +810,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'View details about an account.', + description: 'Get details about an account.', parameters: { type: 'params', required: ['did'], @@ -839,7 +837,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Admin view of invite codes', + description: 'Get an admin view of invite codes.', parameters: { type: 'params', properties: { @@ -887,7 +885,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'View details about a moderation action.', + description: 'Get details about a moderation action.', parameters: { type: 'params', required: ['id'], @@ -913,7 +911,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'List moderation actions related to a subject.', + description: 'Get a list of moderation actions related to a subject.', parameters: { type: 'params', properties: { @@ -959,7 +957,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'View details about a moderation report.', + description: 'Get details about a moderation report.', parameters: { type: 'params', required: ['id'], @@ -985,7 +983,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'List moderation reports related to a subject.', + description: 'Get moderation reports related to a subject.', parameters: { type: 'params', properties: { @@ -1002,14 +1000,14 @@ export const schemaDict = { type: 'string', format: 'did', description: - 'Get all reports that were actioned by a specific moderator', + 'Get all reports that were actioned by a specific moderator.', }, reporters: { type: 'array', items: { type: 'string', }, - description: 'Filter reports made by one or more DIDs', + description: 'Filter reports made by one or more DIDs.', }, resolved: { type: 'boolean', @@ -1035,7 +1033,7 @@ export const schemaDict = { reverse: { type: 'boolean', description: - 'Reverse the order of the returned records? when true, returns reports in chronological order', + 'Reverse the order of the returned records. When true, returns reports in chronological order.', }, }, }, @@ -1067,7 +1065,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'View details about a record.', + description: 'Get details about a record.', parameters: { type: 'params', required: ['uri'], @@ -1103,7 +1101,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'View details about a repository.', + description: 'Get details about a repository.', parameters: { type: 'params', required: ['did'], @@ -1136,7 +1134,7 @@ export const schemaDict = { main: { type: 'query', description: - 'Fetch the service-specific the admin status of a subject (account, record, or blob)', + 'Get the service-specific admin status of a subject (account, record, or blob).', parameters: { type: 'params', properties: { @@ -1309,7 +1307,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: "Send email to a user's primary email address", + description: "Send email to a user's account email address.", input: { encoding: 'application/json', schema: { @@ -1350,7 +1348,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Take a moderation action on a repo.', + description: 'Take a moderation action on an actor.', input: { encoding: 'application/json', schema: { @@ -1397,7 +1395,7 @@ export const schemaDict = { durationInHours: { type: 'integer', description: - 'Indicates how long this action was meant to be in effect before automatically expiring.', + 'Indicates how long this action is meant to be in effect before automatically expiring.', }, createdBy: { type: 'string', @@ -1427,7 +1425,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: "Administrative action to update an account's email", + description: "Administrative action to update an account's email.", input: { encoding: 'application/json', schema: { @@ -1454,7 +1452,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: "Administrative action to update an account's handle", + description: "Administrative action to update an account's handle.", input: { encoding: 'application/json', schema: { @@ -1482,7 +1480,7 @@ export const schemaDict = { main: { type: 'procedure', description: - 'Update the service-specific admin status of a subject (account, record, or blob)', + 'Update the service-specific admin status of a subject (account, record, or blob).', input: { encoding: 'application/json', schema: { @@ -1568,7 +1566,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Updates the handle of the account', + description: 'Updates the handle of the account.', input: { encoding: 'application/json', schema: { @@ -1591,41 +1589,42 @@ export const schemaDict = { defs: { label: { type: 'object', - description: 'Metadata tag on an atproto resource (eg, repo or record)', + description: + 'Metadata tag on an atproto resource (eg, repo or record).', required: ['src', 'uri', 'val', 'cts'], properties: { src: { type: 'string', format: 'did', - description: 'DID of the actor who created this label', + description: 'DID of the actor who created this label.', }, uri: { type: 'string', format: 'uri', description: - 'AT URI of the record, repository (account), or other resource which this label applies to', + 'AT URI of the record, repository (account), or other resource that this label applies to.', }, cid: { type: 'string', format: 'cid', description: - "optionally, CID specifying the specific version of 'uri' resource this label applies to", + "Optionally, CID specifying the specific version of 'uri' resource this label applies to.", }, val: { type: 'string', maxLength: 128, description: - 'the short string name of the value or type of this label', + 'The short string name of the value or type of this label.', }, neg: { type: 'boolean', description: - 'if true, this is a negation label, overwriting a previous label', + 'If true, this is a negation label, overwriting a previous label.', }, cts: { type: 'string', format: 'datetime', - description: 'timestamp when this label was created', + description: 'Timestamp when this label was created.', }, }, }, @@ -1648,14 +1647,14 @@ export const schemaDict = { selfLabel: { type: 'object', description: - 'Metadata tag on an atproto record, published by the author within the record. Note -- schemas should use #selfLabels, not #selfLabel.', + 'Metadata tag on an atproto record, published by the author within the record. Note that schemas should use #selfLabels, not #selfLabel.', required: ['val'], properties: { val: { type: 'string', maxLength: 128, description: - 'the short string name of the value or type of this label', + 'The short string name of the value or type of this label.', }, }, }, @@ -1678,7 +1677,7 @@ export const schemaDict = { type: 'string', }, description: - "List of AT URI patterns to match (boolean 'OR'). Each may be a prefix (ending with '*'; will match inclusive of the string leading to '*'), or a full URI", + "List of AT URI patterns to match (boolean 'OR'). Each may be a prefix (ending with '*'; will match inclusive of the string leading to '*'), or a full URI.", }, sources: { type: 'array', @@ -1686,7 +1685,8 @@ export const schemaDict = { type: 'string', format: 'did', }, - description: 'Optional list of label sources (DIDs) to filter on', + description: + 'Optional list of label sources (DIDs) to filter on.', }, limit: { type: 'integer', @@ -1727,7 +1727,7 @@ export const schemaDict = { defs: { main: { type: 'subscription', - description: 'Subscribe to label updates', + description: 'Subscribe to label updates.', parameters: { type: 'params', properties: { @@ -1922,7 +1922,7 @@ export const schemaDict = { validate: { type: 'boolean', default: true, - description: 'Validate the records?', + description: 'Flag for validating the records.', }, writes: { type: 'array', @@ -2031,7 +2031,7 @@ export const schemaDict = { validate: { type: 'boolean', default: true, - description: 'Validate the record?', + description: 'Flag for validating the record.', }, record: { type: 'unknown', @@ -2041,7 +2041,7 @@ export const schemaDict = { type: 'string', format: 'cid', description: - 'Compare and swap with the previous commit by cid.', + 'Compare and swap with the previous commit by CID.', }, }, }, @@ -2102,13 +2102,13 @@ export const schemaDict = { type: 'string', format: 'cid', description: - 'Compare and swap with the previous record by cid.', + 'Compare and swap with the previous record by CID.', }, swapCommit: { type: 'string', format: 'cid', description: - 'Compare and swap with the previous commit by cid.', + 'Compare and swap with the previous commit by CID.', }, }, }, @@ -2278,7 +2278,7 @@ export const schemaDict = { }, reverse: { type: 'boolean', - description: 'Reverse the order of the returned records?', + description: 'Flag to reverse the order of the returned records.', }, }, }, @@ -2353,7 +2353,7 @@ export const schemaDict = { validate: { type: 'boolean', default: true, - description: 'Validate the record?', + description: 'Flag for validating the record.', }, record: { type: 'unknown', @@ -2363,13 +2363,13 @@ export const schemaDict = { type: 'string', format: 'cid', description: - 'Compare and swap with the previous record by cid.', + 'Compare and swap with the previous record by CID.', }, swapCommit: { type: 'string', format: 'cid', description: - 'Compare and swap with the previous commit by cid.', + 'Compare and swap with the previous commit by CID.', }, }, }, @@ -2583,7 +2583,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Create an app-specific password.', + description: 'Create an App Password.', input: { encoding: 'application/json', schema: { @@ -2671,7 +2671,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Create an invite code.', + description: 'Create invite codes.', input: { encoding: 'application/json', schema: { @@ -2859,7 +2859,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Delete a user account with a token and password.', + description: "Delete an actor's account with a token and password.", input: { encoding: 'application/json', schema: { @@ -2950,7 +2950,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Get all invite codes for a given account', + description: 'Get all invite codes for a given account.', parameters: { type: 'params', properties: { @@ -3030,7 +3030,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'List all app-specific passwords.', + description: 'List all App Passwords.', output: { encoding: 'application/json', schema: { @@ -3126,7 +3126,7 @@ export const schemaDict = { main: { type: 'procedure', description: - 'Request an email with a code to confirm ownership of email', + 'Request an email with a code to confirm ownership of email.', }, }, }, @@ -3248,7 +3248,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Revoke an app-specific password by name.', + description: 'Revoke an App Password by name.', input: { encoding: 'application/json', schema: { @@ -3337,7 +3337,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Gets blocks from a given repo.', + description: 'Get blocks from a given repo.', parameters: { type: 'params', required: ['did', 'cids'], @@ -3432,7 +3432,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Gets the current commit CID & revision of the repo.', + description: 'Get the current commit CID & revision of the repo.', parameters: { type: 'params', required: ['did'], @@ -3475,7 +3475,7 @@ export const schemaDict = { main: { type: 'query', description: - 'Gets blocks needed for existence or non-existence of record.', + 'Get blocks needed for existence or non-existence of record.', parameters: { type: 'params', required: ['did', 'collection', 'rkey'], @@ -3512,7 +3512,7 @@ export const schemaDict = { main: { type: 'query', description: - "Gets the did's repo, optionally catching up from a specific revision.", + "Gets the DID's repo, optionally catching up from a specific revision.", parameters: { type: 'params', required: ['did'], @@ -3540,7 +3540,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'List blob cids since some revision', + description: 'List blob CIDs since some revision.', parameters: { type: 'params', required: ['did'], @@ -3552,7 +3552,7 @@ export const schemaDict = { }, since: { type: 'string', - description: 'Optional revision of the repo to list blobs since', + description: 'Optional revision of the repo to list blobs since.', }, limit: { type: 'integer', @@ -3593,7 +3593,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'List dids and root cids of hosted repos', + description: 'List DIDs and root CIDs of hosted repos.', parameters: { type: 'params', properties: { @@ -3654,7 +3654,7 @@ export const schemaDict = { main: { type: 'procedure', description: - 'Notify a crawling service of a recent update. Often when a long break between updates causes the connection with the crawling service to break.', + 'Notify a crawling service of a recent update; often when a long break between updates causes the connection with the crawling service to break.', input: { encoding: 'application/json', schema: { @@ -3702,7 +3702,7 @@ export const schemaDict = { defs: { main: { type: 'subscription', - description: 'Subscribe to repo updates', + description: 'Subscribe to repo updates.', parameters: { type: 'params', properties: { @@ -3771,15 +3771,15 @@ export const schemaDict = { }, rev: { type: 'string', - description: 'The rev of the emitted commit', + description: 'The rev of the emitted commit.', }, since: { type: 'string', - description: 'The rev of the last emitted commit from this repo', + description: 'The rev of the last emitted commit from this repo.', }, blocks: { type: 'bytes', - description: 'CAR file containing relevant blocks', + description: 'CAR file containing relevant blocks.', maxLength: 1000000, }, ops: { @@ -3877,7 +3877,7 @@ export const schemaDict = { repoOp: { type: 'object', description: - "A repo operation, ie a write of a single record. For creates and updates, cid is the record's CID as of this operation. For deletes, it's null.", + "A repo operation, ie a write of a single record. For creates and updates, CID is the record's CID as of this operation. For deletes, it's null.", required: ['action', 'path', 'cid'], nullable: ['cid'], properties: { @@ -4164,7 +4164,7 @@ export const schemaDict = { birthDate: { type: 'string', format: 'datetime', - description: 'The birth date of the owner of the account.', + description: 'The birth date of account owner.', }, }, }, @@ -4206,7 +4206,7 @@ export const schemaDict = { properties: { sort: { type: 'string', - description: 'Sorting mode.', + description: 'Sorting mode for threads.', knownValues: ['oldest', 'newest', 'most-likes', 'random'], }, prioritizeFollowedUsers: { @@ -4250,6 +4250,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get detailed profile view of an actor.', parameters: { type: 'params', required: ['actor'], @@ -4276,6 +4277,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get detailed profile views of multiple actors.', parameters: { type: 'params', required: ['actors'], @@ -4315,8 +4317,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: - 'Get a list of actors suggested for following. Used in discovery UIs.', + description: 'Get a list of suggested actors, used for discovery.', parameters: { type: 'params', properties: { @@ -4359,6 +4360,7 @@ export const schemaDict = { defs: { main: { type: 'record', + description: 'A declaration of a profile.', key: 'literal:self', record: { type: 'object', @@ -4398,7 +4400,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Sets the private preferences attached to the account.', + description: 'Set the private preferences attached to the account.', input: { encoding: 'application/json', schema: { @@ -4427,12 +4429,12 @@ export const schemaDict = { properties: { term: { type: 'string', - description: "DEPRECATED: use 'q' instead", + description: "DEPRECATED: use 'q' instead.", }, q: { type: 'string', description: - 'search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended', + 'Search query string. Syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended.', }, limit: { type: 'integer', @@ -4473,17 +4475,17 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Find actor suggestions for a search term.', + description: 'Find actor suggestions for a prefix search term.', parameters: { type: 'params', properties: { term: { type: 'string', - description: "DEPRECATED: use 'q' instead", + description: "DEPRECATED: use 'q' instead.", }, q: { type: 'string', - description: 'search query prefix; not a full query string', + description: 'Search query prefix; not a full query string.', }, limit: { type: 'integer', @@ -4516,7 +4518,7 @@ export const schemaDict = { lexicon: 1, id: 'app.bsky.embed.external', description: - 'A representation of some externally linked content, embedded in another form of content', + 'A representation of some externally linked content, embedded in another form of content.', defs: { main: { type: 'object', @@ -4583,7 +4585,7 @@ export const schemaDict = { AppBskyEmbedImages: { lexicon: 1, id: 'app.bsky.embed.images', - description: 'A set of images embedded in some other form of content', + description: 'A set of images embedded in some other form of content.', defs: { main: { type: 'object', @@ -4672,7 +4674,7 @@ export const schemaDict = { lexicon: 1, id: 'app.bsky.embed.record', description: - 'A representation of a record embedded in another form of content', + 'A representation of a record embedded in another form of content.', defs: { main: { type: 'object', @@ -4782,7 +4784,7 @@ export const schemaDict = { lexicon: 1, id: 'app.bsky.embed.recordWithMedia', description: - 'A representation of a record embedded in another form of content, alongside other compatible embeds', + 'A representation of a record embedded in another form of content, alongside other compatible embeds.', defs: { main: { type: 'object', @@ -5150,7 +5152,7 @@ export const schemaDict = { main: { type: 'query', description: - 'Returns information about a given feed generator including TOS & offered feed URIs', + 'Get information about a feed generator, including policies and offered feed URIs.', output: { encoding: 'application/json', schema: { @@ -5205,7 +5207,7 @@ export const schemaDict = { defs: { main: { type: 'record', - description: 'A declaration of the existence of a feed generator', + description: 'A declaration of the existence of a feed generator.', key: 'any', record: { type: 'object', @@ -5256,7 +5258,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Retrieve a list of feeds created by a given actor', + description: 'Get a list of feeds created by the actor.', parameters: { type: 'params', required: ['actor'], @@ -5304,7 +5306,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'A view of the posts liked by an actor.', + description: 'Get a list of posts liked by an actor.', parameters: { type: 'params', required: ['actor'], @@ -5360,7 +5362,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: "A view of an actor's feed.", + description: "Get a view of an actor's feed.", parameters: { type: 'params', required: ['actor'], @@ -5426,7 +5428,7 @@ export const schemaDict = { main: { type: 'query', description: - "Compose and hydrate a feed from a user's selected feed generator", + "Get a hydrated feed from an actor's selected feed generator.", parameters: { type: 'params', required: ['feed'], @@ -5479,8 +5481,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: - 'Get information about a specific feed offered by a feed generator, such as its online status', + description: 'Get information about a feed generator.', parameters: { type: 'params', required: ['feed'], @@ -5519,7 +5520,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Get information about a list of feed generators', + description: 'Get information about a list of feed generators.', parameters: { type: 'params', required: ['feeds'], @@ -5558,7 +5559,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'A skeleton of a feed provided by a feed generator', + description: 'Get a skeleton of a feed provided by a feed generator.', parameters: { type: 'params', required: ['feed'], @@ -5611,6 +5612,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get the list of likes.', parameters: { type: 'params', required: ['uri'], @@ -5688,7 +5690,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'A view of a recent posts from actors in a list', + description: 'Get a view of a recent posts from actors in a list.', parameters: { type: 'params', required: ['list'], @@ -5741,6 +5743,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get posts in a thread.', parameters: { type: 'params', required: ['uri'], @@ -5794,7 +5797,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: "A view of an actor's feed.", + description: "Get a view of an actor's feed.", parameters: { type: 'params', required: ['uris'], @@ -5834,6 +5837,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get a list of reposts.', parameters: { type: 'params', required: ['uri'], @@ -5936,7 +5940,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: "A view of the user's home timeline.", + description: "Get a view of the actor's home timeline.", parameters: { type: 'params', properties: { @@ -5982,6 +5986,7 @@ export const schemaDict = { defs: { main: { type: 'record', + description: 'A declaration of a like.', key: 'tid', record: { type: 'object', @@ -6006,6 +6011,7 @@ export const schemaDict = { defs: { main: { type: 'record', + description: 'A declaration of a post.', key: 'tid', record: { type: 'object', @@ -6128,6 +6134,7 @@ export const schemaDict = { id: 'app.bsky.feed.repost', defs: { main: { + description: 'A declaration of a repost.', type: 'record', key: 'tid', record: { @@ -6153,7 +6160,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Find posts matching search criteria', + description: 'Find posts matching search criteria.', parameters: { type: 'params', required: ['q'], @@ -6161,7 +6168,7 @@ export const schemaDict = { q: { type: 'string', description: - 'search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended', + 'Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended.', }, limit: { type: 'integer', @@ -6172,7 +6179,7 @@ export const schemaDict = { cursor: { type: 'string', description: - 'optional pagination mechanism; may not necessarily allow scrolling through entire result set', + 'Optional pagination mechanism; may not necessarily allow scrolling through entire result set.', }, }, }, @@ -6188,7 +6195,7 @@ export const schemaDict = { hitsTotal: { type: 'integer', description: - 'count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits', + 'Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits.', }, posts: { type: 'array', @@ -6273,7 +6280,7 @@ export const schemaDict = { defs: { main: { type: 'record', - description: 'A block.', + description: 'A declaration of a block.', key: 'tid', record: { type: 'object', @@ -6400,12 +6407,12 @@ export const schemaDict = { modlist: { type: 'token', description: - 'A list of actors to apply an aggregate moderation action (mute/block) on', + 'A list of actors to apply an aggregate moderation action (mute/block) on.', }, curatelist: { type: 'token', description: - 'A list of actors used for curation purposes such as list feeds or interaction gating', + 'A list of actors used for curation purposes such as list feeds or interaction gating.', }, listViewerState: { type: 'object', @@ -6427,7 +6434,7 @@ export const schemaDict = { defs: { main: { type: 'record', - description: 'A social follow.', + description: 'A declaration of a social follow.', key: 'tid', record: { type: 'object', @@ -6452,7 +6459,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: "Who is the requester's account blocking?", + description: 'Get a list of who the actor is blocking.', parameters: { type: 'params', properties: { @@ -6495,7 +6502,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Who is following an actor?', + description: "Get a list of an actor's followers.", parameters: { type: 'params', required: ['actor'], @@ -6547,7 +6554,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Who is an actor following?', + description: 'Get a list of who the actor follows.', parameters: { type: 'params', required: ['actor'], @@ -6599,7 +6606,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Fetch a list of actors', + description: 'Get a list of actors.', parameters: { type: 'params', required: ['list'], @@ -6651,7 +6658,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: "Which lists is the requester's account blocking?", + description: 'Get lists that the actor is blocking.', parameters: { type: 'params', properties: { @@ -6694,7 +6701,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: "Which lists is the requester's account muting?", + description: 'Get lists that the actor is muting.', parameters: { type: 'params', properties: { @@ -6737,7 +6744,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Fetch a list of lists that belong to an actor', + description: 'Get a list of lists that belong to an actor.', parameters: { type: 'params', required: ['actor'], @@ -6785,7 +6792,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Who does the viewer mute?', + description: 'Get a list of who the actor mutes.', parameters: { type: 'params', properties: { @@ -6940,7 +6947,7 @@ export const schemaDict = { defs: { main: { type: 'record', - description: 'An item under a declared list of actors', + description: 'An item under a declared list of actors.', key: 'tid', record: { type: 'object', @@ -6969,7 +6976,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Mute an actor by did or handle.', + description: 'Mute an actor by DID or handle.', input: { encoding: 'application/json', schema: { @@ -7015,7 +7022,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Unmute an actor by did or handle.', + description: 'Unmute an actor by DID or handle.', input: { encoding: 'application/json', schema: { @@ -7061,6 +7068,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get the count of unread notifications.', parameters: { type: 'params', properties: { @@ -7091,6 +7099,7 @@ export const schemaDict = { defs: { main: { type: 'query', + description: 'Get a list of notifications.', parameters: { type: 'params', properties: { @@ -7197,7 +7206,7 @@ export const schemaDict = { defs: { main: { type: 'procedure', - description: 'Register for push notifications with a service', + description: 'Register for push notifications with a service.', input: { encoding: 'application/json', schema: { @@ -7357,7 +7366,7 @@ export const schemaDict = { main: { type: 'query', description: - 'DEPRECATED: will be removed soon, please find a feed generator alternative', + 'DEPRECATED: will be removed soon. Use a feed generator alternative.', parameters: { type: 'params', properties: { @@ -7404,7 +7413,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'An unspecced view of globally popular feed generators', + description: 'An unspecced view of globally popular feed generators.', parameters: { type: 'params', properties: { @@ -7450,7 +7459,8 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'A skeleton of a timeline - UNSPECCED & WILL GO AWAY SOON', + description: + 'DEPRECATED: a skeleton of a timeline. Unspecced and will be unavailable soon.', parameters: { type: 'params', properties: { @@ -7498,7 +7508,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Backend Actors (profile) search, returning only skeleton', + description: 'Backend Actors (profile) search, returns only skeleton.', parameters: { type: 'params', required: ['q'], @@ -7506,11 +7516,11 @@ export const schemaDict = { q: { type: 'string', description: - 'search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. For typeahead search, only simple term match is supported, not full syntax', + 'Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. For typeahead search, only simple term match is supported, not full syntax.', }, typeahead: { type: 'boolean', - description: "if true, acts as fast/simple 'typeahead' query", + description: "If true, acts as fast/simple 'typeahead' query.", }, limit: { type: 'integer', @@ -7521,7 +7531,7 @@ export const schemaDict = { cursor: { type: 'string', description: - 'optional pagination mechanism; may not necessarily allow scrolling through entire result set', + 'Optional pagination mechanism; may not necessarily allow scrolling through entire result set.', }, }, }, @@ -7537,7 +7547,7 @@ export const schemaDict = { hitsTotal: { type: 'integer', description: - 'count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits', + 'Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits.', }, actors: { type: 'array', @@ -7563,7 +7573,7 @@ export const schemaDict = { defs: { main: { type: 'query', - description: 'Backend Posts search, returning only skeleton', + description: 'Backend Posts search, returns only skeleton', parameters: { type: 'params', required: ['q'], @@ -7571,7 +7581,7 @@ export const schemaDict = { q: { type: 'string', description: - 'search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended', + 'Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended.', }, limit: { type: 'integer', @@ -7582,7 +7592,7 @@ export const schemaDict = { cursor: { type: 'string', description: - 'optional pagination mechanism; may not necessarily allow scrolling through entire result set', + 'Optional pagination mechanism; may not necessarily allow scrolling through entire result set.', }, }, }, @@ -7598,7 +7608,7 @@ export const schemaDict = { hitsTotal: { type: 'integer', description: - 'count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits', + 'Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits.', }, posts: { type: 'array', diff --git a/packages/pds/src/lexicon/types/app/bsky/actor/defs.ts b/packages/pds/src/lexicon/types/app/bsky/actor/defs.ts index 171f5c5ef48..c20177ca50e 100644 --- a/packages/pds/src/lexicon/types/app/bsky/actor/defs.ts +++ b/packages/pds/src/lexicon/types/app/bsky/actor/defs.ts @@ -169,7 +169,7 @@ export function validateSavedFeedsPref(v: unknown): ValidationResult { } export interface PersonalDetailsPref { - /** The birth date of the owner of the account. */ + /** The birth date of account owner. */ birthDate?: string [k: string]: unknown } @@ -215,7 +215,7 @@ export function validateFeedViewPref(v: unknown): ValidationResult { } export interface ThreadViewPref { - /** Sorting mode. */ + /** Sorting mode for threads. */ sort?: 'oldest' | 'newest' | 'most-likes' | 'random' | (string & {}) /** Show followed users at the top of all replies. */ prioritizeFollowedUsers?: boolean diff --git a/packages/pds/src/lexicon/types/app/bsky/actor/searchActors.ts b/packages/pds/src/lexicon/types/app/bsky/actor/searchActors.ts index 0222f3658da..f072b8a4d04 100644 --- a/packages/pds/src/lexicon/types/app/bsky/actor/searchActors.ts +++ b/packages/pds/src/lexicon/types/app/bsky/actor/searchActors.ts @@ -10,9 +10,9 @@ import { HandlerAuth } from '@atproto/xrpc-server' import * as AppBskyActorDefs from './defs' export interface QueryParams { - /** DEPRECATED: use 'q' instead */ + /** DEPRECATED: use 'q' instead. */ term?: string - /** search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended */ + /** Search query string. Syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. */ q?: string limit: number cursor?: string diff --git a/packages/pds/src/lexicon/types/app/bsky/actor/searchActorsTypeahead.ts b/packages/pds/src/lexicon/types/app/bsky/actor/searchActorsTypeahead.ts index ba0d62444ce..0cf56753db2 100644 --- a/packages/pds/src/lexicon/types/app/bsky/actor/searchActorsTypeahead.ts +++ b/packages/pds/src/lexicon/types/app/bsky/actor/searchActorsTypeahead.ts @@ -10,9 +10,9 @@ import { HandlerAuth } from '@atproto/xrpc-server' import * as AppBskyActorDefs from './defs' export interface QueryParams { - /** DEPRECATED: use 'q' instead */ + /** DEPRECATED: use 'q' instead. */ term?: string - /** search query prefix; not a full query string */ + /** Search query prefix; not a full query string. */ q?: string limit: number } diff --git a/packages/pds/src/lexicon/types/app/bsky/feed/searchPosts.ts b/packages/pds/src/lexicon/types/app/bsky/feed/searchPosts.ts index 6b5fe08e467..36ac7cbb67d 100644 --- a/packages/pds/src/lexicon/types/app/bsky/feed/searchPosts.ts +++ b/packages/pds/src/lexicon/types/app/bsky/feed/searchPosts.ts @@ -10,10 +10,10 @@ import { HandlerAuth } from '@atproto/xrpc-server' import * as AppBskyFeedDefs from './defs' export interface QueryParams { - /** search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended */ + /** Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. */ q: string limit: number - /** optional pagination mechanism; may not necessarily allow scrolling through entire result set */ + /** Optional pagination mechanism; may not necessarily allow scrolling through entire result set. */ cursor?: string } @@ -21,7 +21,7 @@ export type InputSchema = undefined export interface OutputSchema { cursor?: string - /** count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits */ + /** Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits. */ hitsTotal?: number posts: AppBskyFeedDefs.PostView[] [k: string]: unknown diff --git a/packages/pds/src/lexicon/types/app/bsky/graph/defs.ts b/packages/pds/src/lexicon/types/app/bsky/graph/defs.ts index 121d9db200a..f6c7cb7d77d 100644 --- a/packages/pds/src/lexicon/types/app/bsky/graph/defs.ts +++ b/packages/pds/src/lexicon/types/app/bsky/graph/defs.ts @@ -79,9 +79,9 @@ export type ListPurpose = | 'app.bsky.graph.defs#curatelist' | (string & {}) -/** A list of actors to apply an aggregate moderation action (mute/block) on */ +/** A list of actors to apply an aggregate moderation action (mute/block) on. */ export const MODLIST = 'app.bsky.graph.defs#modlist' -/** A list of actors used for curation purposes such as list feeds or interaction gating */ +/** A list of actors used for curation purposes such as list feeds or interaction gating. */ export const CURATELIST = 'app.bsky.graph.defs#curatelist' export interface ListViewerState { diff --git a/packages/pds/src/lexicon/types/app/bsky/unspecced/searchActorsSkeleton.ts b/packages/pds/src/lexicon/types/app/bsky/unspecced/searchActorsSkeleton.ts index 2cf59bf86a9..5c45b9fb622 100644 --- a/packages/pds/src/lexicon/types/app/bsky/unspecced/searchActorsSkeleton.ts +++ b/packages/pds/src/lexicon/types/app/bsky/unspecced/searchActorsSkeleton.ts @@ -10,12 +10,12 @@ import { HandlerAuth } from '@atproto/xrpc-server' import * as AppBskyUnspeccedDefs from './defs' export interface QueryParams { - /** search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. For typeahead search, only simple term match is supported, not full syntax */ + /** Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. For typeahead search, only simple term match is supported, not full syntax. */ q: string - /** if true, acts as fast/simple 'typeahead' query */ + /** If true, acts as fast/simple 'typeahead' query. */ typeahead?: boolean limit: number - /** optional pagination mechanism; may not necessarily allow scrolling through entire result set */ + /** Optional pagination mechanism; may not necessarily allow scrolling through entire result set. */ cursor?: string } @@ -23,7 +23,7 @@ export type InputSchema = undefined export interface OutputSchema { cursor?: string - /** count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits */ + /** Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits. */ hitsTotal?: number actors: AppBskyUnspeccedDefs.SkeletonSearchActor[] [k: string]: unknown diff --git a/packages/pds/src/lexicon/types/app/bsky/unspecced/searchPostsSkeleton.ts b/packages/pds/src/lexicon/types/app/bsky/unspecced/searchPostsSkeleton.ts index df990d2c5c6..15532087b82 100644 --- a/packages/pds/src/lexicon/types/app/bsky/unspecced/searchPostsSkeleton.ts +++ b/packages/pds/src/lexicon/types/app/bsky/unspecced/searchPostsSkeleton.ts @@ -10,10 +10,10 @@ import { HandlerAuth } from '@atproto/xrpc-server' import * as AppBskyUnspeccedDefs from './defs' export interface QueryParams { - /** search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended */ + /** Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. */ q: string limit: number - /** optional pagination mechanism; may not necessarily allow scrolling through entire result set */ + /** Optional pagination mechanism; may not necessarily allow scrolling through entire result set. */ cursor?: string } @@ -21,7 +21,7 @@ export type InputSchema = undefined export interface OutputSchema { cursor?: string - /** count of search hits. optional, may be rounded/truncated, and may not be possible to paginate through all hits */ + /** Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits. */ hitsTotal?: number posts: AppBskyUnspeccedDefs.SkeletonSearchPost[] [k: string]: unknown diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts index 7c74f6b8b98..8a21c42119e 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts @@ -31,7 +31,7 @@ export function validateStatusAttr(v: unknown): ValidationResult { export interface ActionView { id: number action: ActionType - /** Indicates how long this action was meant to be in effect before automatically expiring. */ + /** Indicates how long this action is meant to be in effect before automatically expiring. */ durationInHours?: number subject: | RepoRef @@ -63,7 +63,7 @@ export function validateActionView(v: unknown): ValidationResult { export interface ActionViewDetail { id: number action: ActionType - /** Indicates how long this action was meant to be in effect before automatically expiring. */ + /** Indicates how long this action is meant to be in effect before automatically expiring. */ durationInHours?: number subject: | RepoView @@ -97,7 +97,7 @@ export function validateActionViewDetail(v: unknown): ValidationResult { export interface ActionViewCurrent { id: number action: ActionType - /** Indicates how long this action was meant to be in effect before automatically expiring. */ + /** Indicates how long this action is meant to be in effect before automatically expiring. */ durationInHours?: number [k: string]: unknown } diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/disableAccountInvites.ts b/packages/pds/src/lexicon/types/com/atproto/admin/disableAccountInvites.ts index 051fabb65e1..62864923dfd 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/disableAccountInvites.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/disableAccountInvites.ts @@ -12,7 +12,7 @@ export interface QueryParams {} export interface InputSchema { account: string - /** Additionally add a note describing why the invites were disabled */ + /** Optional reason for disabled invites. */ note?: string [k: string]: unknown } diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/enableAccountInvites.ts b/packages/pds/src/lexicon/types/com/atproto/admin/enableAccountInvites.ts index 4a26d302333..fb3aa8b8375 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/enableAccountInvites.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/enableAccountInvites.ts @@ -12,7 +12,7 @@ export interface QueryParams {} export interface InputSchema { account: string - /** Additionally add a note describing why the invites were enabled */ + /** Optional reason for enabled invites. */ note?: string [k: string]: unknown } diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/getModerationReports.ts b/packages/pds/src/lexicon/types/com/atproto/admin/getModerationReports.ts index d50af44c757..b80811cf213 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/getModerationReports.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/getModerationReports.ts @@ -12,9 +12,9 @@ import * as ComAtprotoAdminDefs from './defs' export interface QueryParams { subject?: string ignoreSubjects?: string[] - /** Get all reports that were actioned by a specific moderator */ + /** Get all reports that were actioned by a specific moderator. */ actionedBy?: string - /** Filter reports made by one or more DIDs */ + /** Filter reports made by one or more DIDs. */ reporters?: string[] resolved?: boolean actionType?: @@ -25,7 +25,7 @@ export interface QueryParams { | (string & {}) limit: number cursor?: string - /** Reverse the order of the returned records? when true, returns reports in chronological order */ + /** Reverse the order of the returned records. When true, returns reports in chronological order. */ reverse?: boolean } diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/takeModerationAction.ts b/packages/pds/src/lexicon/types/com/atproto/admin/takeModerationAction.ts index fbbf14dff0f..33877d90d11 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/takeModerationAction.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/takeModerationAction.ts @@ -26,7 +26,7 @@ export interface InputSchema { createLabelVals?: string[] negateLabelVals?: string[] reason: string - /** Indicates how long this action was meant to be in effect before automatically expiring. */ + /** Indicates how long this action is meant to be in effect before automatically expiring. */ durationInHours?: number createdBy: string [k: string]: unknown diff --git a/packages/pds/src/lexicon/types/com/atproto/label/defs.ts b/packages/pds/src/lexicon/types/com/atproto/label/defs.ts index a01ad78e254..7268650129a 100644 --- a/packages/pds/src/lexicon/types/com/atproto/label/defs.ts +++ b/packages/pds/src/lexicon/types/com/atproto/label/defs.ts @@ -6,19 +6,19 @@ import { lexicons } from '../../../../lexicons' import { isObj, hasProp } from '../../../../util' import { CID } from 'multiformats/cid' -/** Metadata tag on an atproto resource (eg, repo or record) */ +/** Metadata tag on an atproto resource (eg, repo or record). */ export interface Label { - /** DID of the actor who created this label */ + /** DID of the actor who created this label. */ src: string - /** AT URI of the record, repository (account), or other resource which this label applies to */ + /** AT URI of the record, repository (account), or other resource that this label applies to. */ uri: string - /** optionally, CID specifying the specific version of 'uri' resource this label applies to */ + /** Optionally, CID specifying the specific version of 'uri' resource this label applies to. */ cid?: string - /** the short string name of the value or type of this label */ + /** The short string name of the value or type of this label. */ val: string - /** if true, this is a negation label, overwriting a previous label */ + /** If true, this is a negation label, overwriting a previous label. */ neg?: boolean - /** timestamp when this label was created */ + /** Timestamp when this label was created. */ cts: string [k: string]: unknown } @@ -53,9 +53,9 @@ export function validateSelfLabels(v: unknown): ValidationResult { return lexicons.validate('com.atproto.label.defs#selfLabels', v) } -/** Metadata tag on an atproto record, published by the author within the record. Note -- schemas should use #selfLabels, not #selfLabel. */ +/** Metadata tag on an atproto record, published by the author within the record. Note that schemas should use #selfLabels, not #selfLabel. */ export interface SelfLabel { - /** the short string name of the value or type of this label */ + /** The short string name of the value or type of this label. */ val: string [k: string]: unknown } diff --git a/packages/pds/src/lexicon/types/com/atproto/label/queryLabels.ts b/packages/pds/src/lexicon/types/com/atproto/label/queryLabels.ts index 72cf5c52be6..1d7f8a4def5 100644 --- a/packages/pds/src/lexicon/types/com/atproto/label/queryLabels.ts +++ b/packages/pds/src/lexicon/types/com/atproto/label/queryLabels.ts @@ -10,9 +10,9 @@ import { HandlerAuth } from '@atproto/xrpc-server' import * as ComAtprotoLabelDefs from './defs' export interface QueryParams { - /** List of AT URI patterns to match (boolean 'OR'). Each may be a prefix (ending with '*'; will match inclusive of the string leading to '*'), or a full URI */ + /** List of AT URI patterns to match (boolean 'OR'). Each may be a prefix (ending with '*'; will match inclusive of the string leading to '*'), or a full URI. */ uriPatterns: string[] - /** Optional list of label sources (DIDs) to filter on */ + /** Optional list of label sources (DIDs) to filter on. */ sources?: string[] limit: number cursor?: string diff --git a/packages/pds/src/lexicon/types/com/atproto/repo/applyWrites.ts b/packages/pds/src/lexicon/types/com/atproto/repo/applyWrites.ts index 53f2972e116..61d1e7c28e4 100644 --- a/packages/pds/src/lexicon/types/com/atproto/repo/applyWrites.ts +++ b/packages/pds/src/lexicon/types/com/atproto/repo/applyWrites.ts @@ -13,7 +13,7 @@ export interface QueryParams {} export interface InputSchema { /** The handle or DID of the repo. */ repo: string - /** Validate the records? */ + /** Flag for validating the records. */ validate: boolean writes: (Create | Update | Delete)[] swapCommit?: string diff --git a/packages/pds/src/lexicon/types/com/atproto/repo/createRecord.ts b/packages/pds/src/lexicon/types/com/atproto/repo/createRecord.ts index e069f8caf74..df8c5d9e600 100644 --- a/packages/pds/src/lexicon/types/com/atproto/repo/createRecord.ts +++ b/packages/pds/src/lexicon/types/com/atproto/repo/createRecord.ts @@ -17,11 +17,11 @@ export interface InputSchema { collection: string /** The key of the record. */ rkey?: string - /** Validate the record? */ + /** Flag for validating the record. */ validate: boolean /** The record to create. */ record: {} - /** Compare and swap with the previous commit by cid. */ + /** Compare and swap with the previous commit by CID. */ swapCommit?: string [k: string]: unknown } diff --git a/packages/pds/src/lexicon/types/com/atproto/repo/deleteRecord.ts b/packages/pds/src/lexicon/types/com/atproto/repo/deleteRecord.ts index 5ee016cbed1..f45118a3769 100644 --- a/packages/pds/src/lexicon/types/com/atproto/repo/deleteRecord.ts +++ b/packages/pds/src/lexicon/types/com/atproto/repo/deleteRecord.ts @@ -17,9 +17,9 @@ export interface InputSchema { collection: string /** The key of the record. */ rkey: string - /** Compare and swap with the previous record by cid. */ + /** Compare and swap with the previous record by CID. */ swapRecord?: string - /** Compare and swap with the previous commit by cid. */ + /** Compare and swap with the previous commit by CID. */ swapCommit?: string [k: string]: unknown } diff --git a/packages/pds/src/lexicon/types/com/atproto/repo/listRecords.ts b/packages/pds/src/lexicon/types/com/atproto/repo/listRecords.ts index e58d9714e33..a6cf6abd1f3 100644 --- a/packages/pds/src/lexicon/types/com/atproto/repo/listRecords.ts +++ b/packages/pds/src/lexicon/types/com/atproto/repo/listRecords.ts @@ -20,7 +20,7 @@ export interface QueryParams { rkeyStart?: string /** DEPRECATED: The highest sort-ordered rkey to stop at (exclusive) */ rkeyEnd?: string - /** Reverse the order of the returned records? */ + /** Flag to reverse the order of the returned records. */ reverse?: boolean } diff --git a/packages/pds/src/lexicon/types/com/atproto/repo/putRecord.ts b/packages/pds/src/lexicon/types/com/atproto/repo/putRecord.ts index 364eb59f6f1..f10f773c1c4 100644 --- a/packages/pds/src/lexicon/types/com/atproto/repo/putRecord.ts +++ b/packages/pds/src/lexicon/types/com/atproto/repo/putRecord.ts @@ -17,13 +17,13 @@ export interface InputSchema { collection: string /** The key of the record. */ rkey: string - /** Validate the record? */ + /** Flag for validating the record. */ validate: boolean /** The record to write. */ record: {} - /** Compare and swap with the previous record by cid. */ + /** Compare and swap with the previous record by CID. */ swapRecord?: string | null - /** Compare and swap with the previous commit by cid. */ + /** Compare and swap with the previous commit by CID. */ swapCommit?: string [k: string]: unknown } diff --git a/packages/pds/src/lexicon/types/com/atproto/sync/listBlobs.ts b/packages/pds/src/lexicon/types/com/atproto/sync/listBlobs.ts index 936b08a69f8..b397bb3b3df 100644 --- a/packages/pds/src/lexicon/types/com/atproto/sync/listBlobs.ts +++ b/packages/pds/src/lexicon/types/com/atproto/sync/listBlobs.ts @@ -11,7 +11,7 @@ import { HandlerAuth } from '@atproto/xrpc-server' export interface QueryParams { /** The DID of the repo. */ did: string - /** Optional revision of the repo to list blobs since */ + /** Optional revision of the repo to list blobs since. */ since?: string limit: number cursor?: string diff --git a/packages/pds/src/lexicon/types/com/atproto/sync/subscribeRepos.ts b/packages/pds/src/lexicon/types/com/atproto/sync/subscribeRepos.ts index ae9cf01f8f2..fb334778bf6 100644 --- a/packages/pds/src/lexicon/types/com/atproto/sync/subscribeRepos.ts +++ b/packages/pds/src/lexicon/types/com/atproto/sync/subscribeRepos.ts @@ -39,11 +39,11 @@ export interface Commit { repo: string commit: CID prev?: CID | null - /** The rev of the emitted commit */ + /** The rev of the emitted commit. */ rev: string - /** The rev of the last emitted commit from this repo */ + /** The rev of the last emitted commit from this repo. */ since: string | null - /** CAR file containing relevant blocks */ + /** CAR file containing relevant blocks. */ blocks: Uint8Array ops: RepoOp[] blobs: CID[] @@ -140,7 +140,7 @@ export function validateInfo(v: unknown): ValidationResult { return lexicons.validate('com.atproto.sync.subscribeRepos#info', v) } -/** A repo operation, ie a write of a single record. For creates and updates, cid is the record's CID as of this operation. For deletes, it's null. */ +/** A repo operation, ie a write of a single record. For creates and updates, CID is the record's CID as of this operation. For deletes, it's null. */ export interface RepoOp { action: 'create' | 'update' | 'delete' | (string & {}) path: string From e1b5f2537a5ba4d8b951a741269b604856028ae5 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Tue, 21 Nov 2023 15:30:40 -0600 Subject: [PATCH 36/59] Prevent signature malleability (#1839) * prevent signature malleability * tidy & allow toggle * fix in xrpc-server * add changeset * add test vecotrs --- .changeset/modern-planets-change.md | 6 ++ .../crypto/signature-fixtures.json | 22 +++++ packages/crypto/src/p256/operations.ts | 18 +++- packages/crypto/src/secp256k1/operations.ts | 18 +++- packages/crypto/src/types.ts | 2 +- packages/crypto/tests/signatures.test.ts | 94 ++++++++++++++++++- packages/xrpc-server/src/auth.ts | 2 +- 7 files changed, 156 insertions(+), 6 deletions(-) create mode 100644 .changeset/modern-planets-change.md diff --git a/.changeset/modern-planets-change.md b/.changeset/modern-planets-change.md new file mode 100644 index 00000000000..d019686c266 --- /dev/null +++ b/.changeset/modern-planets-change.md @@ -0,0 +1,6 @@ +--- +'@atproto/crypto': minor +'@atproto/xrpc-server': patch +--- + +Prevent signature malleability through DER-encoded signatures diff --git a/interop-test-files/crypto/signature-fixtures.json b/interop-test-files/crypto/signature-fixtures.json index 7cdeb55ea75..2e41be58cc2 100644 --- a/interop-test-files/crypto/signature-fixtures.json +++ b/interop-test-files/crypto/signature-fixtures.json @@ -42,5 +42,27 @@ "signatureBase64": "5WpdIuEUUfVUYaozsi8G0B3cWO09cgZbIIwg1t2YKdXYA67MYxYiTMAVfdnkDCMN9S5B3vHosRe07aORmoshoQ", "validSignature": false, "tags": ["high-s"] + }, + { + "comment": "P-256 key and signature, with DER-encoded signature which is invalid in atproto", + "messageBase64": "oWVoZWxsb2V3b3JsZA", + "algorithm": "ES256", + "didDocSuite": "EcdsaSecp256r1VerificationKey2019", + "publicKeyDid": "did:key:zDnaeT6hL2RnTdUhAPLij1QBkhYZnmuKyM7puQLW1tkF4Zkt8", + "publicKeyMultibase": "ze8N2PPxnu19hmBQ58t5P3E9Yj6CqakJmTVCaKvf9Byq2", + "signatureBase64": "MEQCIFxYelWJ9lNcAVt+jK0y/T+DC/X4ohFZ+m8f9SEItkY1AiACX7eXz5sgtaRrz/SdPR8kprnbHMQVde0T2R8yOTBweA", + "validSignature": false, + "tags": ["der-encoded"] + }, + { + "comment": "K-256 key and signature, with DER-encoded signature which is invalid in atproto", + "messageBase64": "oWVoZWxsb2V3b3JsZA", + "algorithm": "ES256K", + "didDocSuite": "EcdsaSecp256k1VerificationKey2019", + "publicKeyDid": "did:key:zQ3shnriYMXc8wvkbJqfNWh5GXn2bVAeqTC92YuNbek4npqGF", + "publicKeyMultibase": "z22uZXWP8fdHXi4jyx8cCDiBf9qQTsAe6VcycoMQPfcMQX", + "signatureBase64": "MEUCIQCWumUqJqOCqInXF7AzhIRg2MhwRz2rWZcOEsOjPmNItgIgXJH7RnqfYY6M0eg33wU0sFYDlprwdOcpRn78Sz5ePgk", + "validSignature": false, + "tags": ["der-encoded"] } ] diff --git a/packages/crypto/src/p256/operations.ts b/packages/crypto/src/p256/operations.ts index 6f81b0371a9..e41c494ae55 100644 --- a/packages/crypto/src/p256/operations.ts +++ b/packages/crypto/src/p256/operations.ts @@ -1,5 +1,6 @@ import { p256 } from '@noble/curves/p256' import { sha256 } from '@noble/hashes/sha256' +import * as ui8 from 'uint8arrays' import { P256_JWT_ALG } from '../const' import { parseDidKey } from '../did' import { VerifyOptions } from '../types' @@ -23,8 +24,23 @@ export const verifySig = async ( sig: Uint8Array, opts?: VerifyOptions, ): Promise => { + const allowMalleable = opts?.allowMalleableSig ?? false const msgHash = await sha256(data) + // parse as compact sig to prevent signature malleability + // library supports sigs in 2 different formats: https://github.com/paulmillr/noble-curves/issues/99 + if (!allowMalleable && !isCompactFormat(sig)) { + return false + } return p256.verify(sig, msgHash, publicKey, { - lowS: opts?.lowS ?? true, + lowS: !allowMalleable, }) } + +export const isCompactFormat = (sig: Uint8Array) => { + try { + const parsed = p256.Signature.fromCompact(sig) + return ui8.equals(parsed.toCompactRawBytes(), sig) + } catch { + return false + } +} diff --git a/packages/crypto/src/secp256k1/operations.ts b/packages/crypto/src/secp256k1/operations.ts index f470c2da54c..bc2415feb47 100644 --- a/packages/crypto/src/secp256k1/operations.ts +++ b/packages/crypto/src/secp256k1/operations.ts @@ -1,5 +1,6 @@ import { secp256k1 as k256 } from '@noble/curves/secp256k1' import { sha256 } from '@noble/hashes/sha256' +import * as ui8 from 'uint8arrays' import { SECP256K1_JWT_ALG } from '../const' import { parseDidKey } from '../did' import { VerifyOptions } from '../types' @@ -23,8 +24,23 @@ export const verifySig = async ( sig: Uint8Array, opts?: VerifyOptions, ): Promise => { + const allowMalleable = opts?.allowMalleableSig ?? false const msgHash = await sha256(data) + // parse as compact sig to prevent signature malleability + // library supports sigs in 2 different formats: https://github.com/paulmillr/noble-curves/issues/99 + if (!allowMalleable && !isCompactFormat(sig)) { + return false + } return k256.verify(sig, msgHash, publicKey, { - lowS: opts?.lowS ?? true, + lowS: !allowMalleable, }) } + +export const isCompactFormat = (sig: Uint8Array) => { + try { + const parsed = k256.Signature.fromCompact(sig) + return ui8.equals(parsed.toCompactRawBytes(), sig) + } catch { + return false + } +} diff --git a/packages/crypto/src/types.ts b/packages/crypto/src/types.ts index a1089134f0a..be1dd4ec593 100644 --- a/packages/crypto/src/types.ts +++ b/packages/crypto/src/types.ts @@ -21,5 +21,5 @@ export type DidKeyPlugin = { } export type VerifyOptions = { - lowS?: boolean + allowMalleableSig?: boolean } diff --git a/packages/crypto/tests/signatures.test.ts b/packages/crypto/tests/signatures.test.ts index 83d2b6b72f0..8ccaf7d992c 100644 --- a/packages/crypto/tests/signatures.test.ts +++ b/packages/crypto/tests/signatures.test.ts @@ -78,7 +78,7 @@ describe('signatures', () => { keyBytes, messageBytes, signatureBytes, - { lowS: false }, + { allowMalleableSig: true }, ) expect(verified).toEqual(true) expect(vector.validSignature).toEqual(false) // otherwise would fail per low-s requirement @@ -87,7 +87,46 @@ describe('signatures', () => { keyBytes, messageBytes, signatureBytes, - { lowS: false }, + { allowMalleableSig: true }, + ) + expect(verified).toEqual(true) + expect(vector.validSignature).toEqual(false) // otherwise would fail per low-s requirement + } else { + throw new Error('Unsupported test vector') + } + } + }) + + it('verifies der-encoded signatures with explicit option', async () => { + const DERVectors = vectors.filter((vec) => vec.tags.includes('der-encoded')) + expect(DERVectors.length).toBeGreaterThanOrEqual(2) + for (const vector of DERVectors) { + const messageBytes = uint8arrays.fromString( + vector.messageBase64, + 'base64', + ) + const signatureBytes = uint8arrays.fromString( + vector.signatureBase64, + 'base64', + ) + const keyBytes = multibaseToBytes(vector.publicKeyMultibase) + const didKey = parseDidKey(vector.publicKeyDid) + expect(uint8arrays.equals(keyBytes, didKey.keyBytes)) + if (vector.algorithm === P256_JWT_ALG) { + const verified = await p256.verifySig( + keyBytes, + messageBytes, + signatureBytes, + { allowMalleableSig: true }, + ) + expect(verified).toEqual(true) + expect(vector.validSignature).toEqual(false) // otherwise would fail per low-s requirement + } else if (vector.algorithm === SECP256K1_JWT_ALG) { + const verified = await secp.verifySig( + keyBytes, + messageBytes, + signatureBytes, + { allowMalleableSig: true }, ) expect(verified).toEqual(true) expect(vector.validSignature).toEqual(false) // otherwise would fail per low-s requirement @@ -168,6 +207,39 @@ async function generateTestVectors(): Promise { validSignature: false, tags: ['high-s'], }, + // these vectors test to ensure we don't allow der-encoded signatures + { + messageBase64, + algorithm: P256_JWT_ALG, // "ES256" / ecdsa p-256 + publicKeyDid: p256Key.did(), + publicKeyMultibase: bytesToMultibase( + p256Key.publicKeyBytes(), + 'base58btc', + ), + signatureBase64: await makeDerEncodedSig( + messageBytes, + await p256Key.export(), + P256_JWT_ALG, + ), + validSignature: false, + tags: ['der-encoded'], + }, + { + messageBase64, + algorithm: SECP256K1_JWT_ALG, // "ES256K" / secp256k + publicKeyDid: secpKey.did(), + publicKeyMultibase: bytesToMultibase( + secpKey.publicKeyBytes(), + 'base58btc', + ), + signatureBase64: await makeDerEncodedSig( + messageBytes, + await secpKey.export(), + SECP256K1_JWT_ALG, + ), + validSignature: false, + tags: ['der-encoded'], + }, ] } @@ -195,6 +267,24 @@ async function makeHighSSig( return sig } +async function makeDerEncodedSig( + msgBytes: Uint8Array, + keyBytes: Uint8Array, + alg: string, +): Promise { + const hash = await sha256(msgBytes) + + let sig: string + if (alg === SECP256K1_JWT_ALG) { + const attempt = await nobleK256.sign(hash, keyBytes, { lowS: true }) + sig = uint8arrays.toString(attempt.toDERRawBytes(), 'base64') + } else { + const attempt = await nobleP256.sign(hash, keyBytes, { lowS: true }) + sig = uint8arrays.toString(attempt.toDERRawBytes(), 'base64') + } + return sig +} + type TestVector = { algorithm: string publicKeyDid: string diff --git a/packages/xrpc-server/src/auth.ts b/packages/xrpc-server/src/auth.ts index 0b0cbe03127..db6471aa23e 100644 --- a/packages/xrpc-server/src/auth.ts +++ b/packages/xrpc-server/src/auth.ts @@ -71,7 +71,7 @@ export const verifyJwt = async ( const sigBytes = ui8.fromString(sig, 'base64url') const verifySignatureWithKey = (key: string) => { return crypto.verifySignature(key, msgBytes, sigBytes, { - lowS: false, + allowMalleableSig: true, }) } From b532c502afb1211dea04dfcef6fb4f146e7fb7c4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 21 Nov 2023 15:47:43 -0600 Subject: [PATCH 37/59] Version packages (#1877) Co-authored-by: github-actions[bot] --- .changeset/modern-planets-change.md | 6 ------ packages/aws/CHANGELOG.md | 8 ++++++++ packages/aws/package.json | 2 +- packages/bsky/CHANGELOG.md | 11 +++++++++++ packages/bsky/package.json | 2 +- packages/crypto/CHANGELOG.md | 6 ++++++ packages/crypto/package.json | 2 +- packages/dev-env/CHANGELOG.md | 12 ++++++++++++ packages/dev-env/package.json | 2 +- packages/identity/CHANGELOG.md | 7 +++++++ packages/identity/package.json | 2 +- packages/pds/CHANGELOG.md | 12 ++++++++++++ packages/pds/package.json | 2 +- packages/repo/CHANGELOG.md | 8 ++++++++ packages/repo/package.json | 2 +- packages/xrpc-server/CHANGELOG.md | 9 +++++++++ packages/xrpc-server/package.json | 2 +- 17 files changed, 81 insertions(+), 14 deletions(-) delete mode 100644 .changeset/modern-planets-change.md diff --git a/.changeset/modern-planets-change.md b/.changeset/modern-planets-change.md deleted file mode 100644 index d019686c266..00000000000 --- a/.changeset/modern-planets-change.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@atproto/crypto': minor -'@atproto/xrpc-server': patch ---- - -Prevent signature malleability through DER-encoded signatures diff --git a/packages/aws/CHANGELOG.md b/packages/aws/CHANGELOG.md index 43ada7d9e0f..e49e47b3f5f 100644 --- a/packages/aws/CHANGELOG.md +++ b/packages/aws/CHANGELOG.md @@ -1,5 +1,13 @@ # @atproto/aws +## 0.1.5 + +### Patch Changes + +- Updated dependencies [[`e1b5f253`](https://github.com/bluesky-social/atproto/commit/e1b5f2537a5ba4d8b951a741269b604856028ae5)]: + - @atproto/crypto@0.3.0 + - @atproto/repo@0.3.5 + ## 0.1.4 ### Patch Changes diff --git a/packages/aws/package.json b/packages/aws/package.json index 40eaba51d83..e5f0b6c5507 100644 --- a/packages/aws/package.json +++ b/packages/aws/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/aws", - "version": "0.1.4", + "version": "0.1.5", "license": "MIT", "description": "Shared AWS cloud API helpers for atproto services", "keywords": [ diff --git a/packages/bsky/CHANGELOG.md b/packages/bsky/CHANGELOG.md index fb715a89591..7cbb585dfeb 100644 --- a/packages/bsky/CHANGELOG.md +++ b/packages/bsky/CHANGELOG.md @@ -1,5 +1,16 @@ # @atproto/bsky +## 0.0.15 + +### Patch Changes + +- Updated dependencies [[`e1b5f253`](https://github.com/bluesky-social/atproto/commit/e1b5f2537a5ba4d8b951a741269b604856028ae5)]: + - @atproto/crypto@0.3.0 + - @atproto/xrpc-server@0.4.1 + - @atproto/identity@0.3.2 + - @atproto/repo@0.3.5 + - @atproto/api@0.6.23 + ## 0.0.14 ### Patch Changes diff --git a/packages/bsky/package.json b/packages/bsky/package.json index 81e02626895..c713cd72227 100644 --- a/packages/bsky/package.json +++ b/packages/bsky/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/bsky", - "version": "0.0.14", + "version": "0.0.15", "license": "MIT", "description": "Reference implementation of app.bsky App View (Bluesky API)", "keywords": [ diff --git a/packages/crypto/CHANGELOG.md b/packages/crypto/CHANGELOG.md index c023f142e7a..7352ef08f5a 100644 --- a/packages/crypto/CHANGELOG.md +++ b/packages/crypto/CHANGELOG.md @@ -1,5 +1,11 @@ # @atproto/crypto +## 0.3.0 + +### Minor Changes + +- [#1839](https://github.com/bluesky-social/atproto/pull/1839) [`e1b5f253`](https://github.com/bluesky-social/atproto/commit/e1b5f2537a5ba4d8b951a741269b604856028ae5) Thanks [@dholms](https://github.com/dholms)! - Prevent signature malleability through DER-encoded signatures + ## 0.2.3 ### Patch Changes diff --git a/packages/crypto/package.json b/packages/crypto/package.json index b5b9773c077..20e5607e041 100644 --- a/packages/crypto/package.json +++ b/packages/crypto/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/crypto", - "version": "0.2.3", + "version": "0.3.0", "license": "MIT", "description": "Library for cryptographic keys and signing in atproto", "keywords": [ diff --git a/packages/dev-env/CHANGELOG.md b/packages/dev-env/CHANGELOG.md index 97b9eac682c..62f533f6b96 100644 --- a/packages/dev-env/CHANGELOG.md +++ b/packages/dev-env/CHANGELOG.md @@ -1,5 +1,17 @@ # @atproto/dev-env +## 0.2.15 + +### Patch Changes + +- Updated dependencies [[`e1b5f253`](https://github.com/bluesky-social/atproto/commit/e1b5f2537a5ba4d8b951a741269b604856028ae5)]: + - @atproto/crypto@0.3.0 + - @atproto/xrpc-server@0.4.1 + - @atproto/bsky@0.0.15 + - @atproto/identity@0.3.2 + - @atproto/pds@0.3.3 + - @atproto/api@0.6.23 + ## 0.2.14 ### Patch Changes diff --git a/packages/dev-env/package.json b/packages/dev-env/package.json index 90fd0824e9d..eb6f31d6986 100644 --- a/packages/dev-env/package.json +++ b/packages/dev-env/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/dev-env", - "version": "0.2.14", + "version": "0.2.15", "license": "MIT", "description": "Local development environment helper for atproto development", "keywords": [ diff --git a/packages/identity/CHANGELOG.md b/packages/identity/CHANGELOG.md index ac49fc13f1a..5b62503a3ce 100644 --- a/packages/identity/CHANGELOG.md +++ b/packages/identity/CHANGELOG.md @@ -1,5 +1,12 @@ # @atproto/identity +## 0.3.2 + +### Patch Changes + +- Updated dependencies [[`e1b5f253`](https://github.com/bluesky-social/atproto/commit/e1b5f2537a5ba4d8b951a741269b604856028ae5)]: + - @atproto/crypto@0.3.0 + ## 0.3.1 ### Patch Changes diff --git a/packages/identity/package.json b/packages/identity/package.json index e4f39c24ecd..1259ad290a5 100644 --- a/packages/identity/package.json +++ b/packages/identity/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/identity", - "version": "0.3.1", + "version": "0.3.2", "license": "MIT", "description": "Library for decentralized identities in atproto using DIDs and handles", "keywords": [ diff --git a/packages/pds/CHANGELOG.md b/packages/pds/CHANGELOG.md index aca0e381ad4..34cda77d01e 100644 --- a/packages/pds/CHANGELOG.md +++ b/packages/pds/CHANGELOG.md @@ -1,5 +1,17 @@ # @atproto/pds +## 0.3.3 + +### Patch Changes + +- Updated dependencies [[`e1b5f253`](https://github.com/bluesky-social/atproto/commit/e1b5f2537a5ba4d8b951a741269b604856028ae5)]: + - @atproto/crypto@0.3.0 + - @atproto/xrpc-server@0.4.1 + - @atproto/aws@0.1.5 + - @atproto/identity@0.3.2 + - @atproto/repo@0.3.5 + - @atproto/api@0.6.23 + ## 0.3.2 ### Patch Changes diff --git a/packages/pds/package.json b/packages/pds/package.json index c5bf32f9357..857fd912611 100644 --- a/packages/pds/package.json +++ b/packages/pds/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/pds", - "version": "0.3.2", + "version": "0.3.3", "license": "MIT", "description": "Reference implementation of atproto Personal Data Server (PDS)", "keywords": [ diff --git a/packages/repo/CHANGELOG.md b/packages/repo/CHANGELOG.md index 753b8f39811..35c6c3b9c44 100644 --- a/packages/repo/CHANGELOG.md +++ b/packages/repo/CHANGELOG.md @@ -1,5 +1,13 @@ # @atproto/repo +## 0.3.5 + +### Patch Changes + +- Updated dependencies [[`e1b5f253`](https://github.com/bluesky-social/atproto/commit/e1b5f2537a5ba4d8b951a741269b604856028ae5)]: + - @atproto/crypto@0.3.0 + - @atproto/identity@0.3.2 + ## 0.3.4 ### Patch Changes diff --git a/packages/repo/package.json b/packages/repo/package.json index 23022149653..c36cbcf668e 100644 --- a/packages/repo/package.json +++ b/packages/repo/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/repo", - "version": "0.3.4", + "version": "0.3.5", "license": "MIT", "description": "atproto repo and MST implementation", "keywords": [ diff --git a/packages/xrpc-server/CHANGELOG.md b/packages/xrpc-server/CHANGELOG.md index 1a0f2ab8279..29db2da0d61 100644 --- a/packages/xrpc-server/CHANGELOG.md +++ b/packages/xrpc-server/CHANGELOG.md @@ -1,5 +1,14 @@ # @atproto/xrpc-server +## 0.4.1 + +### Patch Changes + +- [#1839](https://github.com/bluesky-social/atproto/pull/1839) [`e1b5f253`](https://github.com/bluesky-social/atproto/commit/e1b5f2537a5ba4d8b951a741269b604856028ae5) Thanks [@dholms](https://github.com/dholms)! - Prevent signature malleability through DER-encoded signatures + +- Updated dependencies [[`e1b5f253`](https://github.com/bluesky-social/atproto/commit/e1b5f2537a5ba4d8b951a741269b604856028ae5)]: + - @atproto/crypto@0.3.0 + ## 0.4.0 ### Minor Changes diff --git a/packages/xrpc-server/package.json b/packages/xrpc-server/package.json index fae70972446..d3d2fa63c98 100644 --- a/packages/xrpc-server/package.json +++ b/packages/xrpc-server/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/xrpc-server", - "version": "0.4.0", + "version": "0.4.1", "license": "MIT", "description": "atproto HTTP API (XRPC) server library", "keywords": [ From f2323719344dfc22561c1acf2f2841247ac9d488 Mon Sep 17 00:00:00 2001 From: devin ivy Date: Tue, 21 Nov 2023 18:09:07 -0500 Subject: [PATCH 38/59] Provide proper error for updating to an email that's already in use (#1770) provide proper error for updating to an email already in use --- .../src/api/com/atproto/server/updateEmail.ts | 17 ++++++++++--- packages/pds/src/db/util.ts | 8 +++++++ packages/pds/src/services/account/index.ts | 24 ++++++++++++++----- packages/pds/tests/email-confirmation.test.ts | 13 ++++++++++ 4 files changed, 53 insertions(+), 9 deletions(-) diff --git a/packages/pds/src/api/com/atproto/server/updateEmail.ts b/packages/pds/src/api/com/atproto/server/updateEmail.ts index c10bb1c85f6..e0f9d9bc078 100644 --- a/packages/pds/src/api/com/atproto/server/updateEmail.ts +++ b/packages/pds/src/api/com/atproto/server/updateEmail.ts @@ -1,7 +1,8 @@ +import disposable from 'disposable-email' +import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { InvalidRequestError } from '@atproto/xrpc-server' -import disposable from 'disposable-email' +import { UserAlreadyExistsError } from '../../../../services/account' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.updateEmail({ @@ -38,7 +39,17 @@ export default function (server: Server, ctx: AppContext) { await accntSrvce.deleteEmailToken(did, 'update_email') } if (user.email !== email) { - await accntSrvce.updateEmail(did, email) + try { + await accntSrvce.updateEmail(did, email) + } catch (err) { + if (err instanceof UserAlreadyExistsError) { + throw new InvalidRequestError( + 'This email address is already in use, please use a different email.', + ) + } else { + throw err + } + } } }) }, diff --git a/packages/pds/src/db/util.ts b/packages/pds/src/db/util.ts index 6dd31c67898..09970e3be8b 100644 --- a/packages/pds/src/db/util.ts +++ b/packages/pds/src/db/util.ts @@ -57,3 +57,11 @@ export const dummyDialect = { export type DbRef = RawBuilder | ReturnType export type AnyQb = SelectQueryBuilder + +export const isErrUniqueViolation = (err: unknown) => { + const code = err?.['code'] + return ( + code === '23505' || // postgres, see https://www.postgresql.org/docs/current/errcodes-appendix.html + code === 'SQLITE_CONSTRAINT_UNIQUE' // sqlite, see https://www.sqlite.org/rescode.html#constraint_unique + ) +} diff --git a/packages/pds/src/services/account/index.ts b/packages/pds/src/services/account/index.ts index 3893be03209..2bcf651c2a2 100644 --- a/packages/pds/src/services/account/index.ts +++ b/packages/pds/src/services/account/index.ts @@ -8,7 +8,11 @@ import * as scrypt from '../../db/scrypt' import { UserAccountEntry } from '../../db/tables/user-account' import { DidHandle } from '../../db/tables/did-handle' import { RepoRoot } from '../../db/tables/repo-root' -import { countAll, notSoftDeletedClause } from '../../db/util' +import { + countAll, + isErrUniqueViolation, + notSoftDeletedClause, +} from '../../db/util' import { paginate, TimeCidKeyset } from '../../db/pagination' import * as sequencer from '../../sequencer' import { AppPassword } from '../../lexicon/types/com/atproto/server/createAppPassword' @@ -189,11 +193,19 @@ export class AccountService { } async updateEmail(did: string, email: string) { - await this.db.db - .updateTable('user_account') - .set({ email: email.toLowerCase(), emailConfirmedAt: null }) - .where('did', '=', did) - .executeTakeFirst() + try { + await this.db.db + .updateTable('user_account') + .set({ email: email.toLowerCase(), emailConfirmedAt: null }) + .where('did', '=', did) + .executeTakeFirst() + } catch (err) { + if (isErrUniqueViolation(err)) { + throw new UserAlreadyExistsError() + } else { + throw err + } + } } async updateUserPassword(did: string, password: string) { diff --git a/packages/pds/tests/email-confirmation.test.ts b/packages/pds/tests/email-confirmation.test.ts index cc95bfa424a..2f66b1939d2 100644 --- a/packages/pds/tests/email-confirmation.test.ts +++ b/packages/pds/tests/email-confirmation.test.ts @@ -194,6 +194,19 @@ describe('email confirmation', () => { ) }) + it('fails email update with in-use email', async () => { + const attempt = agent.api.com.atproto.server.updateEmail( + { + email: 'bob@test.com', + token: updateToken, + }, + { headers: sc.getHeaders(alice.did), encoding: 'application/json' }, + ) + await expect(attempt).rejects.toThrow( + 'This email address is already in use, please use a different email.', + ) + }) + it('updates email', async () => { await agent.api.com.atproto.server.updateEmail( { From 95d33f7b1187672ba97543aaaaa38726c1480328 Mon Sep 17 00:00:00 2001 From: devin ivy Date: Mon, 27 Nov 2023 15:30:09 -0500 Subject: [PATCH 39/59] Support unauthed usage of feeds (#1884) * update local feedgens to not require a viewer where possible * update getFeed to use optional auth * test feeds w/ optional auth --- .../bsky/src/api/app/bsky/feed/getFeed.ts | 14 +- packages/bsky/src/auth.ts | 12 +- packages/bsky/src/context.ts | 4 + packages/bsky/src/feed-gen/best-of-follows.ts | 9 +- packages/bsky/src/feed-gen/bsky-team.ts | 2 +- packages/bsky/src/feed-gen/hot-classic.ts | 2 +- packages/bsky/src/feed-gen/mutuals.ts | 8 +- packages/bsky/src/feed-gen/types.ts | 2 +- packages/bsky/src/feed-gen/whats-hot.ts | 2 +- packages/bsky/src/feed-gen/with-friends.ts | 10 +- .../feed-generation.test.ts.snap | 328 +++++++++++++++++- packages/bsky/tests/feed-generation.test.ts | 61 +++- 12 files changed, 420 insertions(+), 34 deletions(-) diff --git a/packages/bsky/src/api/app/bsky/feed/getFeed.ts b/packages/bsky/src/api/app/bsky/feed/getFeed.ts index 8af159decd3..bfae2caa2f5 100644 --- a/packages/bsky/src/api/app/bsky/feed/getFeed.ts +++ b/packages/bsky/src/api/app/bsky/feed/getFeed.ts @@ -33,7 +33,7 @@ export default function (server: Server, ctx: AppContext) { presentation, ) server.app.bsky.feed.getFeed({ - auth: ctx.authVerifierAnyAudience, + auth: ctx.authOptionalVerifierAnyAudience, handler: async ({ params, auth, req }) => { const db = ctx.db.getReplica() const feedService = ctx.services.feed(db) @@ -98,13 +98,15 @@ const hydration = async (state: SkeletonState, ctx: Context) => { const noBlocksOrMutes = (state: HydrationState) => { const { viewer } = state.params - state.feedItems = state.feedItems.filter( - (item) => + state.feedItems = state.feedItems.filter((item) => { + if (!viewer) return true + return ( !state.bam.block([viewer, item.postAuthorDid]) && !state.bam.block([viewer, item.originatorDid]) && !state.bam.mute([viewer, item.postAuthorDid]) && - !state.bam.mute([viewer, item.originatorDid]), - ) + !state.bam.mute([viewer, item.originatorDid]) + ) + }) return state } @@ -130,7 +132,7 @@ type Context = { authorization?: string } -type Params = GetFeedParams & { viewer: string } +type Params = GetFeedParams & { viewer: string | null } type SkeletonState = { params: Params diff --git a/packages/bsky/src/auth.ts b/packages/bsky/src/auth.ts index b19e6860e5c..220be08fc32 100644 --- a/packages/bsky/src/auth.ts +++ b/packages/bsky/src/auth.ts @@ -28,14 +28,18 @@ export const authVerifier = return { credentials: { did: payload.iss }, artifacts: { aud: opts.aud } } } -export const authOptionalVerifier = - (idResolver: IdResolver, opts: { aud: string | null }) => - async (reqCtx: { req: express.Request; res: express.Response }) => { +export const authOptionalVerifier = ( + idResolver: IdResolver, + opts: { aud: string | null }, +) => { + const verify = authVerifier(idResolver, opts) + return async (reqCtx: { req: express.Request; res: express.Response }) => { if (!reqCtx.req.headers.authorization) { return { credentials: { did: null } } } - return authVerifier(idResolver, opts)(reqCtx) + return verify(reqCtx) } +} export const authOptionalAccessOrRoleVerifier = ( idResolver: IdResolver, diff --git a/packages/bsky/src/context.ts b/packages/bsky/src/context.ts index 21c01c38fbd..3488c6a5c02 100644 --- a/packages/bsky/src/context.ts +++ b/packages/bsky/src/context.ts @@ -94,6 +94,10 @@ export class AppContext { return auth.authVerifier(this.idResolver, { aud: null }) } + get authOptionalVerifierAnyAudience() { + return auth.authOptionalVerifier(this.idResolver, { aud: null }) + } + get authOptionalVerifier() { return auth.authOptionalVerifier(this.idResolver, { aud: this.cfg.serverDid, diff --git a/packages/bsky/src/feed-gen/best-of-follows.ts b/packages/bsky/src/feed-gen/best-of-follows.ts index c1d4ee4d21b..33c70ea81a4 100644 --- a/packages/bsky/src/feed-gen/best-of-follows.ts +++ b/packages/bsky/src/feed-gen/best-of-follows.ts @@ -1,4 +1,4 @@ -import { InvalidRequestError } from '@atproto/xrpc-server' +import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton' import { AlgoHandler, AlgoResponse } from './types' import { GenericKeyset, paginate } from '../db/pagination' @@ -7,12 +7,15 @@ import AppContext from '../context' const handler: AlgoHandler = async ( ctx: AppContext, params: SkeletonParams, - viewer: string, + viewer: string | null, ): Promise => { + if (!viewer) { + throw new AuthRequiredError('This feed requires being logged-in') + } + const { limit, cursor } = params const db = ctx.db.getReplica('feed') const feedService = ctx.services.feed(db) - const { ref } = db.db.dynamic // candidates are ranked within a materialized view by like count, depreciated over time. diff --git a/packages/bsky/src/feed-gen/bsky-team.ts b/packages/bsky/src/feed-gen/bsky-team.ts index 3592dd42e26..feb9539345e 100644 --- a/packages/bsky/src/feed-gen/bsky-team.ts +++ b/packages/bsky/src/feed-gen/bsky-team.ts @@ -14,7 +14,7 @@ const BSKY_TEAM: NotEmptyArray = [ const handler: AlgoHandler = async ( ctx: AppContext, params: SkeletonParams, - _viewer: string, + _viewer: string | null, ): Promise => { const { limit = 50, cursor } = params const db = ctx.db.getReplica('feed') diff --git a/packages/bsky/src/feed-gen/hot-classic.ts b/packages/bsky/src/feed-gen/hot-classic.ts index c042cea7116..d1595105f27 100644 --- a/packages/bsky/src/feed-gen/hot-classic.ts +++ b/packages/bsky/src/feed-gen/hot-classic.ts @@ -11,7 +11,7 @@ const NO_WHATS_HOT_LABELS: NotEmptyArray = ['!no-promote'] const handler: AlgoHandler = async ( ctx: AppContext, params: SkeletonParams, - _viewer: string, + _viewer: string | null, ): Promise => { const { limit = 50, cursor } = params const db = ctx.db.getReplica('feed') diff --git a/packages/bsky/src/feed-gen/mutuals.ts b/packages/bsky/src/feed-gen/mutuals.ts index 65a3311a524..86583ebaa56 100644 --- a/packages/bsky/src/feed-gen/mutuals.ts +++ b/packages/bsky/src/feed-gen/mutuals.ts @@ -3,16 +3,20 @@ import AppContext from '../context' import { paginate } from '../db/pagination' import { AlgoHandler, AlgoResponse } from './types' import { FeedKeyset, getFeedDateThreshold } from '../api/app/bsky/util/feed' +import { AuthRequiredError } from '@atproto/xrpc-server' const handler: AlgoHandler = async ( ctx: AppContext, params: SkeletonParams, - viewer: string, + viewer: string | null, ): Promise => { + if (!viewer) { + throw new AuthRequiredError('This feed requires being logged-in') + } + const { limit = 50, cursor } = params const db = ctx.db.getReplica('feed') const feedService = ctx.services.feed(db) - const { ref } = db.db.dynamic const mutualsSubquery = db.db diff --git a/packages/bsky/src/feed-gen/types.ts b/packages/bsky/src/feed-gen/types.ts index 11ebf53fb39..4693d64d4dd 100644 --- a/packages/bsky/src/feed-gen/types.ts +++ b/packages/bsky/src/feed-gen/types.ts @@ -11,7 +11,7 @@ export type AlgoResponse = { export type AlgoHandler = ( ctx: AppContext, params: SkeletonParams, - requester: string, + viewer: string | null, ) => Promise export type MountedAlgos = Record diff --git a/packages/bsky/src/feed-gen/whats-hot.ts b/packages/bsky/src/feed-gen/whats-hot.ts index 511c767804e..2376b98f185 100644 --- a/packages/bsky/src/feed-gen/whats-hot.ts +++ b/packages/bsky/src/feed-gen/whats-hot.ts @@ -21,7 +21,7 @@ const NO_WHATS_HOT_LABELS: NotEmptyArray = [ const handler: AlgoHandler = async ( ctx: AppContext, params: SkeletonParams, - _viewer: string, + _viewer: string | null, ): Promise => { const { limit, cursor } = params const db = ctx.db.getReplica('feed') diff --git a/packages/bsky/src/feed-gen/with-friends.ts b/packages/bsky/src/feed-gen/with-friends.ts index 0fd8f31c48e..1e6d345ffcc 100644 --- a/packages/bsky/src/feed-gen/with-friends.ts +++ b/packages/bsky/src/feed-gen/with-friends.ts @@ -3,16 +3,20 @@ import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/ge import { paginate } from '../db/pagination' import { AlgoHandler, AlgoResponse } from './types' import { FeedKeyset, getFeedDateThreshold } from '../api/app/bsky/util/feed' +import { AuthRequiredError } from '@atproto/xrpc-server' const handler: AlgoHandler = async ( ctx: AppContext, params: SkeletonParams, - requester: string, + viewer: string | null, ): Promise => { + if (!viewer) { + throw new AuthRequiredError('This feed requires being logged-in') + } + const { cursor, limit = 50 } = params const db = ctx.db.getReplica('feed') const feedService = ctx.services.feed(db) - const { ref } = db.db.dynamic const keyset = new FeedKeyset(ref('post.sortAt'), ref('post.cid')) @@ -23,7 +27,7 @@ const handler: AlgoHandler = async ( .innerJoin('follow', 'follow.subjectDid', 'post.creator') .innerJoin('post_agg', 'post_agg.uri', 'post.uri') .where('post_agg.likeCount', '>=', 5) - .where('follow.creator', '=', requester) + .where('follow.creator', '=', viewer) .where('post.sortAt', '>', getFeedDateThreshold(sortFrom)) postsQb = paginate(postsQb, { limit, cursor, keyset, tryIndex: true }) diff --git a/packages/bsky/tests/__snapshots__/feed-generation.test.ts.snap b/packages/bsky/tests/__snapshots__/feed-generation.test.ts.snap index 1a5f8fc9281..ac9e0eee7a0 100644 --- a/packages/bsky/tests/__snapshots__/feed-generation.test.ts.snap +++ b/packages/bsky/tests/__snapshots__/feed-generation.test.ts.snap @@ -204,9 +204,9 @@ Array [ "muted": false, }, }, - "description": "Provides all feed candidates, blindly ignoring pagination limit", + "description": "Provides all feed candidates when authed", "did": "user(0)", - "displayName": "Bad Pagination", + "displayName": "Needs Auth", "indexedAt": "1970-01-01T00:00:00.000Z", "likeCount": 0, "uri": "record(4)", @@ -246,9 +246,9 @@ Array [ "muted": false, }, }, - "description": "Provides even-indexed feed candidates", + "description": "Provides all feed candidates, blindly ignoring pagination limit", "did": "user(0)", - "displayName": "Even", + "displayName": "Bad Pagination", "indexedAt": "1970-01-01T00:00:00.000Z", "likeCount": 0, "uri": "record(5)", @@ -288,14 +288,56 @@ Array [ "muted": false, }, }, + "description": "Provides even-indexed feed candidates", + "did": "user(0)", + "displayName": "Even", + "indexedAt": "1970-01-01T00:00:00.000Z", + "likeCount": 0, + "uri": "record(6)", + "viewer": Object {}, + }, + Object { + "cid": "cids(6)", + "creator": Object { + "avatar": "https://bsky.public.url/img/avatar/plain/user(2)/cids(1)@jpeg", + "description": "its me!", + "did": "user(1)", + "displayName": "ali", + "handle": "alice.test", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [ + Object { + "cid": "cids(2)", + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "user(1)", + "uri": "record(3)", + "val": "self-label-a", + }, + Object { + "cid": "cids(2)", + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "user(1)", + "uri": "record(3)", + "val": "self-label-b", + }, + ], + "viewer": Object { + "blockedBy": false, + "followedBy": "record(2)", + "following": "record(1)", + "muted": false, + }, + }, "description": "Provides all feed candidates", "did": "user(0)", "displayName": "All", "indexedAt": "1970-01-01T00:00:00.000Z", "likeCount": 2, - "uri": "record(6)", + "uri": "record(7)", "viewer": Object { - "like": "record(7)", + "like": "record(8)", }, }, ] @@ -822,6 +864,280 @@ Array [ ] `; +exports[`feed generation getFeed resolves basic feed contents without auth. 1`] = ` +Array [ + Object { + "post": Object { + "author": Object { + "avatar": "https://bsky.public.url/img/avatar/plain/user(1)/cids(1)@jpeg", + "did": "user(0)", + "displayName": "ali", + "handle": "alice.test", + "labels": Array [ + Object { + "cid": "cids(2)", + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "user(0)", + "uri": "record(1)", + "val": "self-label-a", + }, + Object { + "cid": "cids(2)", + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "user(0)", + "uri": "record(1)", + "val": "self-label-b", + }, + ], + }, + "cid": "cids(0)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [ + Object { + "cid": "cids(0)", + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "user(0)", + "uri": "record(0)", + "val": "self-label", + }, + ], + "likeCount": 0, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "labels": Object { + "$type": "com.atproto.label.defs#selfLabels", + "values": Array [ + Object { + "val": "self-label", + }, + ], + }, + "text": "hey there", + }, + "replyCount": 0, + "repostCount": 0, + "uri": "record(0)", + }, + }, + Object { + "post": Object { + "author": Object { + "did": "user(2)", + "handle": "carol.test", + "labels": Array [], + }, + "cid": "cids(3)", + "embed": Object { + "$type": "app.bsky.embed.recordWithMedia#view", + "media": Object { + "$type": "app.bsky.embed.images#view", + "images": Array [ + Object { + "alt": "tests/sample-img/key-landscape-small.jpg", + "fullsize": "https://bsky.public.url/img/feed_fullsize/plain/user(5)/cids(4)@jpeg", + "thumb": "https://bsky.public.url/img/feed_thumbnail/plain/user(5)/cids(4)@jpeg", + }, + Object { + "alt": "tests/sample-img/key-alt.jpg", + "fullsize": "https://bsky.public.url/img/feed_fullsize/plain/user(5)/cids(5)@jpeg", + "thumb": "https://bsky.public.url/img/feed_thumbnail/plain/user(5)/cids(5)@jpeg", + }, + ], + }, + "record": Object { + "record": Object { + "$type": "app.bsky.embed.record#viewRecord", + "author": Object { + "avatar": "https://bsky.public.url/img/avatar/plain/user(4)/cids(1)@jpeg", + "did": "user(3)", + "displayName": "bobby", + "handle": "bob.test", + "labels": Array [], + }, + "cid": "cids(6)", + "embeds": Array [], + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "uri": "record(3)", + "value": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "langs": Array [ + "en-US", + "i-klingon", + ], + "text": "bob back at it again!", + }, + }, + }, + }, + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 2, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "embed": Object { + "$type": "app.bsky.embed.recordWithMedia", + "media": Object { + "$type": "app.bsky.embed.images", + "images": Array [ + Object { + "alt": "tests/sample-img/key-landscape-small.jpg", + "image": Object { + "$type": "blob", + "mimeType": "image/jpeg", + "ref": Object { + "$link": "cids(4)", + }, + "size": 4114, + }, + }, + Object { + "alt": "tests/sample-img/key-alt.jpg", + "image": Object { + "$type": "blob", + "mimeType": "image/jpeg", + "ref": Object { + "$link": "cids(5)", + }, + "size": 12736, + }, + }, + ], + }, + "record": Object { + "record": Object { + "cid": "cids(6)", + "uri": "record(3)", + }, + }, + }, + "text": "hi im carol", + }, + "replyCount": 0, + "repostCount": 0, + "uri": "record(2)", + }, + }, + Object { + "post": Object { + "author": Object { + "did": "user(2)", + "handle": "carol.test", + "labels": Array [], + }, + "cid": "cids(7)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 0, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "reply": Object { + "parent": Object { + "cid": "cids(8)", + "uri": "record(5)", + }, + "root": Object { + "cid": "cids(8)", + "uri": "record(5)", + }, + }, + "text": "of course", + }, + "replyCount": 0, + "repostCount": 0, + "uri": "record(4)", + }, + "reply": Object { + "parent": Object { + "$type": "app.bsky.feed.defs#postView", + "author": Object { + "avatar": "https://bsky.public.url/img/avatar/plain/user(1)/cids(1)@jpeg", + "did": "user(0)", + "displayName": "ali", + "handle": "alice.test", + "labels": Array [ + Object { + "cid": "cids(2)", + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "user(0)", + "uri": "record(1)", + "val": "self-label-a", + }, + Object { + "cid": "cids(2)", + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "user(0)", + "uri": "record(1)", + "val": "self-label-b", + }, + ], + }, + "cid": "cids(8)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 3, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000000Z", + "text": "again", + }, + "replyCount": 2, + "repostCount": 1, + "uri": "record(5)", + }, + "root": Object { + "$type": "app.bsky.feed.defs#postView", + "author": Object { + "avatar": "https://bsky.public.url/img/avatar/plain/user(1)/cids(1)@jpeg", + "did": "user(0)", + "displayName": "ali", + "handle": "alice.test", + "labels": Array [ + Object { + "cid": "cids(2)", + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "user(0)", + "uri": "record(1)", + "val": "self-label-a", + }, + Object { + "cid": "cids(2)", + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "user(0)", + "uri": "record(1)", + "val": "self-label-b", + }, + ], + }, + "cid": "cids(8)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 3, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000000Z", + "text": "again", + }, + "replyCount": 2, + "repostCount": 1, + "uri": "record(5)", + }, + }, + }, +] +`; + exports[`feed generation getFeed resolves basic feed contents. 1`] = ` Array [ Object { diff --git a/packages/bsky/tests/feed-generation.test.ts b/packages/bsky/tests/feed-generation.test.ts index 09dfd92acc8..4970c13b31c 100644 --- a/packages/bsky/tests/feed-generation.test.ts +++ b/packages/bsky/tests/feed-generation.test.ts @@ -17,6 +17,9 @@ import { } from '../src/lexicon/types/app/bsky/feed/defs' import basicSeed from './seeds/basic' import { forSnapshot, paginateAll } from './_util' +import { AuthRequiredError } from '@atproto/xrpc-server' +import assert from 'assert' +import { XRPCError } from '@atproto/xrpc' describe('feed generation', () => { let network: TestNetwork @@ -33,6 +36,7 @@ describe('feed generation', () => { let feedUriBadPagination: string let feedUriPrime: string // Taken-down let feedUriPrimeRef: RecordRef + let feedUriNeedsAuth: string beforeAll(async () => { network = await TestNetwork.create({ @@ -52,11 +56,17 @@ describe('feed generation', () => { ) const evenUri = AtUri.make(alice, 'app.bsky.feed.generator', 'even') const primeUri = AtUri.make(alice, 'app.bsky.feed.generator', 'prime') + const needsAuthUri = AtUri.make( + alice, + 'app.bsky.feed.generator', + 'needs-auth', + ) gen = await network.createFeedGen({ [allUri.toString()]: feedGenHandler('all'), [evenUri.toString()]: feedGenHandler('even'), [feedUriBadPagination.toString()]: feedGenHandler('bad-pagination'), [primeUri.toString()]: feedGenHandler('prime'), + [needsAuthUri.toString()]: feedGenHandler('needs-auth'), }) const feedSuggestions = [ @@ -137,6 +147,16 @@ describe('feed generation', () => { }, sc.getHeaders(alice), ) + const needsAuth = await pdsAgent.api.app.bsky.feed.generator.create( + { repo: alice, rkey: 'needs-auth' }, + { + did: gen.did, + displayName: 'Needs Auth', + description: 'Provides all feed candidates when authed', + createdAt: new Date().toISOString(), + }, + sc.getHeaders(alice), + ) await network.processAll() await agent.api.com.atproto.admin.takeModerationAction( { @@ -161,6 +181,7 @@ describe('feed generation', () => { feedUriBadPagination = badPagination.uri feedUriPrime = prime.uri feedUriPrimeRef = new RecordRef(prime.uri, prime.cid) + feedUriNeedsAuth = needsAuth.uri }) it('feed gen records can be updated', async () => { @@ -198,11 +219,12 @@ describe('feed generation', () => { const paginatedAll: GeneratorView[] = results(await paginateAll(paginator)) - expect(paginatedAll.length).toEqual(4) + expect(paginatedAll.length).toEqual(5) expect(paginatedAll[0].uri).toEqual(feedUriOdd) - expect(paginatedAll[1].uri).toEqual(feedUriBadPagination) - expect(paginatedAll[2].uri).toEqual(feedUriEven) - expect(paginatedAll[3].uri).toEqual(feedUriAll) + expect(paginatedAll[1].uri).toEqual(feedUriNeedsAuth) + expect(paginatedAll[2].uri).toEqual(feedUriBadPagination) + expect(paginatedAll[3].uri).toEqual(feedUriEven) + expect(paginatedAll[4].uri).toEqual(feedUriAll) expect(paginatedAll.map((fg) => fg.uri)).not.toContain(feedUriPrime) // taken-down expect(forSnapshot(paginatedAll)).toMatchSnapshot() }) @@ -348,7 +370,9 @@ describe('feed generation', () => { {}, { headers: await network.serviceHeaders(sc.dids.bob) }, ) - expect(resEven.data.feeds.map((f) => f.likeCount)).toEqual([2, 0, 0, 0]) + expect(resEven.data.feeds.map((f) => f.likeCount)).toEqual([ + 2, 0, 0, 0, 0, + ]) expect(resEven.data.feeds.map((f) => f.uri)).not.toContain(feedUriPrime) // taken-down }) @@ -389,6 +413,16 @@ describe('feed generation', () => { expect(forSnapshot(feed.data.feed)).toMatchSnapshot() }) + it('resolves basic feed contents without auth.', async () => { + const feed = await agent.api.app.bsky.feed.getFeed({ feed: feedUriEven }) + expect(feed.data.feed.map((item) => item.post.uri)).toEqual([ + sc.posts[sc.dids.alice][0].ref.uriStr, + sc.posts[sc.dids.carol][0].ref.uriStr, + sc.replies[sc.dids.carol][0].ref.uriStr, + ]) + expect(forSnapshot(feed.data.feed)).toMatchSnapshot() + }) + it('paginates, handling replies and reposts.', async () => { const results = (results) => results.flatMap((res) => res.feed) const paginator = async (cursor?: string) => { @@ -461,6 +495,16 @@ describe('feed generation', () => { expect(feed.data['$auth']?.['iss']).toEqual(alice) }) + it('passes through auth error from feed.', async () => { + const tryGetFeed = agent.api.app.bsky.feed.getFeed({ + feed: feedUriNeedsAuth, + }) + const err = await tryGetFeed.catch((err) => err) + assert(err instanceof XRPCError) + expect(err.status).toBe(401) + expect(err.message).toBe('This feed requires auth') + }) + it('provides timing info in server-timing header.', async () => { const result = await agent.api.app.bsky.feed.getFeed( { feed: feedUriEven }, @@ -482,8 +526,13 @@ describe('feed generation', () => { }) const feedGenHandler = - (feedName: 'even' | 'all' | 'prime' | 'bad-pagination'): SkeletonHandler => + ( + feedName: 'even' | 'all' | 'prime' | 'bad-pagination' | 'needs-auth', + ): SkeletonHandler => async ({ req, params }) => { + if (feedName === 'needs-auth' && !req.headers.authorization) { + throw new AuthRequiredError('This feed requires auth') + } const { limit, cursor } = params const candidates: SkeletonFeedPost[] = [ { post: sc.posts[sc.dids.alice][0].ref.uriStr }, From 7edad62c12038de09452f8e005be5a1a75e18873 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Mon, 27 Nov 2023 20:14:20 -0600 Subject: [PATCH 40/59] Put canReply state on post viewer state instead of thread viewer state (#1882) * switch canReply from thread to post view * tweaks & fix up tests * update snaps * fix more snaps * hydrate feed items for getPosts & searchPosts * fix another snapshot * getPosts test * canReply -> blockedByGate & DRY up code * blockedByGate -> excludedByGate * replyDisabled --- lexicons/app/bsky/feed/defs.json | 12 +--- packages/api/src/client/lexicons.ts | 15 +--- .../src/client/types/app/bsky/feed/defs.ts | 19 +---- .../src/api/app/bsky/feed/getActorLikes.ts | 4 +- .../src/api/app/bsky/feed/getAuthorFeed.ts | 4 +- .../bsky/src/api/app/bsky/feed/getFeed.ts | 4 +- .../bsky/src/api/app/bsky/feed/getListFeed.ts | 4 +- .../src/api/app/bsky/feed/getPostThread.ts | 69 ++++-------------- .../bsky/src/api/app/bsky/feed/getPosts.ts | 39 ++++++----- .../bsky/src/api/app/bsky/feed/getTimeline.ts | 4 +- .../bsky/src/api/app/bsky/feed/searchPosts.ts | 37 +++++----- .../bsky/src/api/app/bsky/graph/getList.ts | 2 +- .../src/api/app/bsky/graph/getListBlocks.ts | 2 +- packages/bsky/src/lexicon/lexicons.ts | 15 +--- .../src/lexicon/types/app/bsky/feed/defs.ts | 19 +---- packages/bsky/src/services/actor/views.ts | 19 ++--- packages/bsky/src/services/feed/index.ts | 9 ++- packages/bsky/src/services/feed/util.ts | 66 ++++++++++++----- packages/bsky/src/services/feed/views.ts | 67 +++++++++++++++++- packages/bsky/src/services/graph/index.ts | 6 ++ packages/bsky/src/services/graph/types.ts | 1 + .../src/services/indexing/plugins/post.ts | 2 +- .../tests/__snapshots__/indexing.test.ts.snap | 6 -- .../__snapshots__/block-lists.test.ts.snap | 9 --- .../views/__snapshots__/blocks.test.ts.snap | 9 --- .../__snapshots__/mute-lists.test.ts.snap | 3 - .../views/__snapshots__/mutes.test.ts.snap | 3 - .../views/__snapshots__/thread.test.ts.snap | 30 -------- .../bsky/tests/views/threadgating.test.ts | 70 ++++++++++++++----- packages/pds/src/lexicon/lexicons.ts | 15 +--- .../src/lexicon/types/app/bsky/feed/defs.ts | 19 +---- 31 files changed, 261 insertions(+), 322 deletions(-) diff --git a/lexicons/app/bsky/feed/defs.json b/lexicons/app/bsky/feed/defs.json index 10f2812ce24..15a7cb7a719 100644 --- a/lexicons/app/bsky/feed/defs.json +++ b/lexicons/app/bsky/feed/defs.json @@ -38,7 +38,8 @@ "type": "object", "properties": { "repost": { "type": "string", "format": "at-uri" }, - "like": { "type": "string", "format": "at-uri" } + "like": { "type": "string", "format": "at-uri" }, + "replyDisabled": { "type": "boolean" } } }, "feedViewPost": { @@ -87,8 +88,7 @@ "type": "union", "refs": ["#threadViewPost", "#notFoundPost", "#blockedPost"] } - }, - "viewer": { "type": "ref", "ref": "#viewerThreadState" } + } } }, "notFoundPost": { @@ -116,12 +116,6 @@ "viewer": { "type": "ref", "ref": "app.bsky.actor.defs#viewerState" } } }, - "viewerThreadState": { - "type": "object", - "properties": { - "canReply": { "type": "boolean" } - } - }, "generatorView": { "type": "object", "required": ["uri", "cid", "did", "creator", "displayName", "indexedAt"], diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index 14c1be9f477..cc3f09f0c4e 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -4892,6 +4892,9 @@ export const schemaDict = { type: 'string', format: 'at-uri', }, + replyDisabled: { + type: 'boolean', + }, }, }, feedViewPost: { @@ -4975,10 +4978,6 @@ export const schemaDict = { ], }, }, - viewer: { - type: 'ref', - ref: 'lex:app.bsky.feed.defs#viewerThreadState', - }, }, }, notFoundPost: { @@ -5027,14 +5026,6 @@ export const schemaDict = { }, }, }, - viewerThreadState: { - type: 'object', - properties: { - canReply: { - type: 'boolean', - }, - }, - }, generatorView: { type: 'object', required: ['uri', 'cid', 'did', 'creator', 'displayName', 'indexedAt'], diff --git a/packages/api/src/client/types/app/bsky/feed/defs.ts b/packages/api/src/client/types/app/bsky/feed/defs.ts index 944fd34b072..82cbfd9951a 100644 --- a/packages/api/src/client/types/app/bsky/feed/defs.ts +++ b/packages/api/src/client/types/app/bsky/feed/defs.ts @@ -48,6 +48,7 @@ export function validatePostView(v: unknown): ValidationResult { export interface ViewerState { repost?: string like?: string + replyDisabled?: boolean [k: string]: unknown } @@ -137,7 +138,6 @@ export interface ThreadViewPost { | BlockedPost | { $type: string; [k: string]: unknown } )[] - viewer?: ViewerThreadState [k: string]: unknown } @@ -208,23 +208,6 @@ export function validateBlockedAuthor(v: unknown): ValidationResult { return lexicons.validate('app.bsky.feed.defs#blockedAuthor', v) } -export interface ViewerThreadState { - canReply?: boolean - [k: string]: unknown -} - -export function isViewerThreadState(v: unknown): v is ViewerThreadState { - return ( - isObj(v) && - hasProp(v, '$type') && - v.$type === 'app.bsky.feed.defs#viewerThreadState' - ) -} - -export function validateViewerThreadState(v: unknown): ValidationResult { - return lexicons.validate('app.bsky.feed.defs#viewerThreadState', v) -} - export interface GeneratorView { uri: string cid: string diff --git a/packages/bsky/src/api/app/bsky/feed/getActorLikes.ts b/packages/bsky/src/api/app/bsky/feed/getActorLikes.ts index 3da1b0d042a..36e36b0100b 100644 --- a/packages/bsky/src/api/app/bsky/feed/getActorLikes.ts +++ b/packages/bsky/src/api/app/bsky/feed/getActorLikes.ts @@ -107,9 +107,7 @@ const noPostBlocks = (state: HydrationState) => { const presentation = (state: HydrationState, ctx: Context) => { const { feedService } = ctx const { feedItems, cursor, params } = state - const feed = feedService.views.formatFeed(feedItems, state, { - viewer: params.viewer, - }) + const feed = feedService.views.formatFeed(feedItems, state, params.viewer) return { feed, cursor } } diff --git a/packages/bsky/src/api/app/bsky/feed/getAuthorFeed.ts b/packages/bsky/src/api/app/bsky/feed/getAuthorFeed.ts index c71ddd45791..26b945f3ecd 100644 --- a/packages/bsky/src/api/app/bsky/feed/getAuthorFeed.ts +++ b/packages/bsky/src/api/app/bsky/feed/getAuthorFeed.ts @@ -147,9 +147,7 @@ const noBlocksOrMutedReposts = (state: HydrationState) => { const presentation = (state: HydrationState, ctx: Context) => { const { feedService } = ctx const { feedItems, cursor, params } = state - const feed = feedService.views.formatFeed(feedItems, state, { - viewer: params.viewer, - }) + const feed = feedService.views.formatFeed(feedItems, state, params.viewer) return { feed, cursor } } diff --git a/packages/bsky/src/api/app/bsky/feed/getFeed.ts b/packages/bsky/src/api/app/bsky/feed/getFeed.ts index bfae2caa2f5..a09258c3163 100644 --- a/packages/bsky/src/api/app/bsky/feed/getFeed.ts +++ b/packages/bsky/src/api/app/bsky/feed/getFeed.ts @@ -113,9 +113,7 @@ const noBlocksOrMutes = (state: HydrationState) => { const presentation = (state: HydrationState, ctx: Context) => { const { feedService } = ctx const { feedItems, cursor, passthrough, params } = state - const feed = feedService.views.formatFeed(feedItems, state, { - viewer: params.viewer, - }) + const feed = feedService.views.formatFeed(feedItems, state, params.viewer) return { feed, cursor, diff --git a/packages/bsky/src/api/app/bsky/feed/getListFeed.ts b/packages/bsky/src/api/app/bsky/feed/getListFeed.ts index f166c8abb99..fd3f0360ef3 100644 --- a/packages/bsky/src/api/app/bsky/feed/getListFeed.ts +++ b/packages/bsky/src/api/app/bsky/feed/getListFeed.ts @@ -105,9 +105,7 @@ const noBlocksOrMutes = (state: HydrationState) => { const presentation = (state: HydrationState, ctx: Context) => { const { feedService } = ctx const { feedItems, cursor, params } = state - const feed = feedService.views.formatFeed(feedItems, state, { - viewer: params.viewer, - }) + const feed = feedService.views.formatFeed(feedItems, state, params.viewer) return { feed, cursor } } diff --git a/packages/bsky/src/api/app/bsky/feed/getPostThread.ts b/packages/bsky/src/api/app/bsky/feed/getPostThread.ts index 0e31107d052..873dd311ba0 100644 --- a/packages/bsky/src/api/app/bsky/feed/getPostThread.ts +++ b/packages/bsky/src/api/app/bsky/feed/getPostThread.ts @@ -6,27 +6,21 @@ import { NotFoundPost, ThreadViewPost, isNotFoundPost, - isThreadViewPost, } from '../../../../lexicon/types/app/bsky/feed/defs' -import { Record as PostRecord } from '../../../../lexicon/types/app/bsky/feed/post' -import { Record as ThreadgateRecord } from '../../../../lexicon/types/app/bsky/feed/threadgate' import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getPostThread' import AppContext from '../../../../context' import { FeedService, FeedRow, FeedHydrationState, - PostInfo, } from '../../../../services/feed' import { getAncestorsAndSelfQb, getDescendentsQb, } from '../../../../services/util/post' import { Database } from '../../../../db' -import DatabaseSchema from '../../../../db/database-schema' import { setRepoRev } from '../../../util' import { ActorInfoMap, ActorService } from '../../../../services/actor' -import { violatesThreadGate } from '../../../../services/feed/util' import { createPipeline, noRules } from '../../../../pipeline' export default function (server: Server, ctx: AppContext) { @@ -80,21 +74,7 @@ const hydration = async (state: SkeletonState, ctx: Context) => { } = state const relevant = getRelevantIds(threadData) const hydrated = await feedService.feedHydration({ ...relevant, viewer }) - // check root reply interaction rules - const anchorPostUri = threadData.post.postUri - const rootUri = threadData.post.replyRoot || anchorPostUri - const anchor = hydrated.posts[anchorPostUri] - const root = hydrated.posts[rootUri] - const gate = hydrated.threadgates[rootUri]?.record - const viewerCanReply = await checkViewerCanReply( - ctx.db.db, - anchor ?? null, - viewer, - new AtUri(rootUri).host, - (root?.record ?? null) as PostRecord | null, - gate ?? null, - ) - return { ...state, ...hydrated, viewerCanReply } + return { ...state, ...hydrated } } const presentation = (state: HydrationState, ctx: Context) => { @@ -103,16 +83,19 @@ const presentation = (state: HydrationState, ctx: Context) => { const actors = actorService.views.profileBasicPresentation( Object.keys(profiles), state, - { viewer: params.viewer }, + params.viewer, + ) + const thread = composeThread( + state.threadData, + actors, + state, + ctx, + params.viewer, ) - const thread = composeThread(state.threadData, actors, state, ctx) if (isNotFoundPost(thread)) { // @TODO technically this could be returned as a NotFoundPost based on lexicon throw new InvalidRequestError(`Post not found: ${params.uri}`, 'NotFound') } - if (isThreadViewPost(thread) && params.viewer) { - thread.viewer = { canReply: state.viewerCanReply } - } return { thread } } @@ -121,6 +104,7 @@ const composeThread = ( actors: ActorInfoMap, state: HydrationState, ctx: Context, + viewer: string | null, ) => { const { feedService } = ctx const { posts, threadgates, embeds, blocks, labels, lists } = state @@ -133,6 +117,7 @@ const composeThread = ( embeds, labels, lists, + viewer, ) // replies that are invalid due to reply-gating: @@ -179,14 +164,14 @@ const composeThread = ( notFound: true, } } else { - parent = composeThread(threadData.parent, actors, state, ctx) + parent = composeThread(threadData.parent, actors, state, ctx, viewer) } } let replies: (ThreadViewPost | NotFoundPost | BlockedPost)[] | undefined if (threadData.replies && !badReply) { replies = threadData.replies.flatMap((reply) => { - const thread = composeThread(reply, actors, state, ctx) + const thread = composeThread(reply, actors, state, ctx, viewer) // e.g. don't bother including #postNotFound reply placeholders for takedowns. either way matches api contract. const skip = [] return isNotFoundPost(thread) ? skip : thread @@ -223,6 +208,7 @@ const getRelevantIds = ( if (thread.post.replyRoot) { // ensure root is included for checking interactions uris.add(thread.post.replyRoot) + dids.add(new AtUri(thread.post.replyRoot).hostname) } return { dids, uris } } @@ -317,28 +303,6 @@ const getChildrenData = ( })) } -const checkViewerCanReply = async ( - db: DatabaseSchema, - anchor: PostInfo | null, - viewer: string | null, - owner: string, - root: PostRecord | null, - threadgate: ThreadgateRecord | null, -) => { - if (!viewer) return false - // @TODO re-enable invalidReplyRoot check - // if (anchor?.invalidReplyRoot || anchor?.violatesThreadGate) return false - if (anchor?.violatesThreadGate) return false - const viewerViolatesThreadGate = await violatesThreadGate( - db, - viewer, - owner, - root, - threadgate, - ) - return !viewerViolatesThreadGate -} - class ParentNotFoundError extends Error { constructor(public uri: string) { super(`Parent not found: ${uri}`) @@ -364,7 +328,4 @@ type SkeletonState = { threadData: PostThread } -type HydrationState = SkeletonState & - FeedHydrationState & { - viewerCanReply: boolean - } +type HydrationState = SkeletonState & FeedHydrationState diff --git a/packages/bsky/src/api/app/bsky/feed/getPosts.ts b/packages/bsky/src/api/app/bsky/feed/getPosts.ts index 90268e5f161..5ec4807accb 100644 --- a/packages/bsky/src/api/app/bsky/feed/getPosts.ts +++ b/packages/bsky/src/api/app/bsky/feed/getPosts.ts @@ -1,10 +1,13 @@ import { dedupeStrs } from '@atproto/common' -import { AtUri } from '@atproto/syntax' import { Server } from '../../../../lexicon' import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getPosts' import AppContext from '../../../../context' import { Database } from '../../../../db' -import { FeedHydrationState, FeedService } from '../../../../services/feed' +import { + FeedHydrationState, + FeedRow, + FeedService, +} from '../../../../services/feed' import { createPipeline } from '../../../../pipeline' import { ActorService } from '../../../../services/actor' @@ -31,18 +34,18 @@ export default function (server: Server, ctx: AppContext) { }) } -const skeleton = async (params: Params) => { - return { params, postUris: dedupeStrs(params.uris) } +const skeleton = async (params: Params, ctx: Context) => { + const deduped = dedupeStrs(params.uris) + const feedItems = await ctx.feedService.postUrisToFeedItems(deduped) + return { params, feedItems } } const hydration = async (state: SkeletonState, ctx: Context) => { const { feedService } = ctx - const { params, postUris } = state - const uris = new Set(postUris) - const dids = new Set(postUris.map((uri) => new AtUri(uri).hostname)) + const { params, feedItems } = state + const refs = feedService.feedItemRefs(feedItems) const hydrated = await feedService.feedHydration({ - uris, - dids, + ...refs, viewer: params.viewer, }) return { ...state, ...hydrated } @@ -50,32 +53,32 @@ const hydration = async (state: SkeletonState, ctx: Context) => { const noBlocks = (state: HydrationState) => { const { viewer } = state.params - state.postUris = state.postUris.filter((uri) => { - const post = state.posts[uri] - if (!viewer || !post) return true - return !state.bam.block([viewer, post.creator]) + state.feedItems = state.feedItems.filter((item) => { + if (!viewer) return true + return !state.bam.block([viewer, item.postAuthorDid]) }) return state } const presentation = (state: HydrationState, ctx: Context) => { const { feedService, actorService } = ctx - const { postUris, profiles, params } = state + const { feedItems, profiles, params } = state const SKIP = [] const actors = actorService.views.profileBasicPresentation( Object.keys(profiles), state, - { viewer: params.viewer }, + params.viewer, ) - const postViews = postUris.flatMap((uri) => { + const postViews = feedItems.flatMap((item) => { const postView = feedService.views.formatPostView( - uri, + item.postUri, actors, state.posts, state.threadgates, state.embeds, state.labels, state.lists, + params.viewer, ) return postView ?? SKIP }) @@ -92,7 +95,7 @@ type Params = QueryParams & { viewer: string | null } type SkeletonState = { params: Params - postUris: string[] + feedItems: FeedRow[] } type HydrationState = SkeletonState & FeedHydrationState diff --git a/packages/bsky/src/api/app/bsky/feed/getTimeline.ts b/packages/bsky/src/api/app/bsky/feed/getTimeline.ts index 9609ed6db42..18cc5c2629a 100644 --- a/packages/bsky/src/api/app/bsky/feed/getTimeline.ts +++ b/packages/bsky/src/api/app/bsky/feed/getTimeline.ts @@ -146,9 +146,7 @@ const noBlocksOrMutes = (state: HydrationState): HydrationState => { const presentation = (state: HydrationState, ctx: Context) => { const { feedService } = ctx const { feedItems, cursor, params } = state - const feed = feedService.views.formatFeed(feedItems, state, { - viewer: params.viewer, - }) + const feed = feedService.views.formatFeed(feedItems, state, params.viewer) return { feed, cursor } } diff --git a/packages/bsky/src/api/app/bsky/feed/searchPosts.ts b/packages/bsky/src/api/app/bsky/feed/searchPosts.ts index 718bfa7afa4..db143fc5b8c 100644 --- a/packages/bsky/src/api/app/bsky/feed/searchPosts.ts +++ b/packages/bsky/src/api/app/bsky/feed/searchPosts.ts @@ -2,11 +2,14 @@ import AppContext from '../../../../context' import { Server } from '../../../../lexicon' import { InvalidRequestError } from '@atproto/xrpc-server' import AtpAgent from '@atproto/api' -import { AtUri } from '@atproto/syntax' import { mapDefined } from '@atproto/common' import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/searchPosts' import { Database } from '../../../../db' -import { FeedHydrationState, FeedService } from '../../../../services/feed' +import { + FeedHydrationState, + FeedRow, + FeedService, +} from '../../../../services/feed' import { ActorService } from '../../../../services/actor' import { createPipeline } from '../../../../pipeline' @@ -51,9 +54,11 @@ const skeleton = async ( cursor: params.cursor, limit: params.limit, }) + const postUris = res.data.posts.map((a) => a.uri) + const feedItems = await ctx.feedService.postUrisToFeedItems(postUris) return { params, - postUris: res.data.posts.map((a) => a.uri), + feedItems, cursor: res.data.cursor, hitsTotal: res.data.hitsTotal, } @@ -64,12 +69,10 @@ const hydration = async ( ctx: Context, ): Promise => { const { feedService } = ctx - const { params, postUris } = state - const uris = new Set(postUris) - const dids = new Set(postUris.map((uri) => new AtUri(uri).hostname)) + const { params, feedItems } = state + const refs = feedService.feedItemRefs(feedItems) const hydrated = await feedService.feedHydration({ - uris, - dids, + ...refs, viewer: params.viewer, }) return { ...state, ...hydrated } @@ -77,32 +80,32 @@ const hydration = async ( const noBlocks = (state: HydrationState): HydrationState => { const { viewer } = state.params - state.postUris = state.postUris.filter((uri) => { - const post = state.posts[uri] - if (!viewer || !post) return true - return !state.bam.block([viewer, post.creator]) + state.feedItems = state.feedItems.filter((item) => { + if (!viewer) return true + return !state.bam.block([viewer, item.postAuthorDid]) }) return state } const presentation = (state: HydrationState, ctx: Context) => { const { feedService, actorService } = ctx - const { postUris, profiles, params } = state + const { feedItems, profiles, params } = state const actors = actorService.views.profileBasicPresentation( Object.keys(profiles), state, - { viewer: params.viewer }, + params.viewer, ) - const postViews = mapDefined(postUris, (uri) => + const postViews = mapDefined(feedItems, (item) => feedService.views.formatPostView( - uri, + item.postUri, actors, state.posts, state.threadgates, state.embeds, state.labels, state.lists, + params.viewer, ), ) return { posts: postViews, cursor: state.cursor, hitsTotal: state.hitsTotal } @@ -119,7 +122,7 @@ type Params = QueryParams & { viewer: string | null } type SkeletonState = { params: Params - postUris: string[] + feedItems: FeedRow[] hitsTotal?: number cursor?: string } diff --git a/packages/bsky/src/api/app/bsky/graph/getList.ts b/packages/bsky/src/api/app/bsky/graph/getList.ts index 1e6775d01cb..82963183a74 100644 --- a/packages/bsky/src/api/app/bsky/graph/getList.ts +++ b/packages/bsky/src/api/app/bsky/graph/getList.ts @@ -91,7 +91,7 @@ const presentation = (state: HydrationState, ctx: Context) => { const actors = actorService.views.profilePresentation( Object.keys(profileState.profiles), profileState, - { viewer: params.viewer }, + params.viewer, ) const creator = actors[list.creator] if (!creator) { diff --git a/packages/bsky/src/api/app/bsky/graph/getListBlocks.ts b/packages/bsky/src/api/app/bsky/graph/getListBlocks.ts index 0884005b244..a41c952508b 100644 --- a/packages/bsky/src/api/app/bsky/graph/getListBlocks.ts +++ b/packages/bsky/src/api/app/bsky/graph/getListBlocks.ts @@ -87,7 +87,7 @@ const presentation = (state: HydrationState, ctx: Context) => { const actors = actorService.views.profilePresentation( Object.keys(profileState.profiles), profileState, - { viewer: params.viewer }, + params.viewer, ) const lists = listInfos.map((list) => graphService.formatListView(list, actors), diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index 14c1be9f477..cc3f09f0c4e 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -4892,6 +4892,9 @@ export const schemaDict = { type: 'string', format: 'at-uri', }, + replyDisabled: { + type: 'boolean', + }, }, }, feedViewPost: { @@ -4975,10 +4978,6 @@ export const schemaDict = { ], }, }, - viewer: { - type: 'ref', - ref: 'lex:app.bsky.feed.defs#viewerThreadState', - }, }, }, notFoundPost: { @@ -5027,14 +5026,6 @@ export const schemaDict = { }, }, }, - viewerThreadState: { - type: 'object', - properties: { - canReply: { - type: 'boolean', - }, - }, - }, generatorView: { type: 'object', required: ['uri', 'cid', 'did', 'creator', 'displayName', 'indexedAt'], diff --git a/packages/bsky/src/lexicon/types/app/bsky/feed/defs.ts b/packages/bsky/src/lexicon/types/app/bsky/feed/defs.ts index 08d34d88ebb..382d3f58ecf 100644 --- a/packages/bsky/src/lexicon/types/app/bsky/feed/defs.ts +++ b/packages/bsky/src/lexicon/types/app/bsky/feed/defs.ts @@ -48,6 +48,7 @@ export function validatePostView(v: unknown): ValidationResult { export interface ViewerState { repost?: string like?: string + replyDisabled?: boolean [k: string]: unknown } @@ -137,7 +138,6 @@ export interface ThreadViewPost { | BlockedPost | { $type: string; [k: string]: unknown } )[] - viewer?: ViewerThreadState [k: string]: unknown } @@ -208,23 +208,6 @@ export function validateBlockedAuthor(v: unknown): ValidationResult { return lexicons.validate('app.bsky.feed.defs#blockedAuthor', v) } -export interface ViewerThreadState { - canReply?: boolean - [k: string]: unknown -} - -export function isViewerThreadState(v: unknown): v is ViewerThreadState { - return ( - isObj(v) && - hasProp(v, '$type') && - v.$type === 'app.bsky.feed.defs#viewerThreadState' - ) -} - -export function validateViewerThreadState(v: unknown): ValidationResult { - return lexicons.validate('app.bsky.feed.defs#viewerThreadState', v) -} - export interface GeneratorView { uri: string cid: string diff --git a/packages/bsky/src/services/actor/views.ts b/packages/bsky/src/services/actor/views.ts index 7118671bd04..5c40eac308b 100644 --- a/packages/bsky/src/services/actor/views.ts +++ b/packages/bsky/src/services/actor/views.ts @@ -45,10 +45,7 @@ export class ActorViews { viewer, ...opts, }) - return this.profilePresentation(dids, hydrated, { - viewer, - ...opts, - }) + return this.profilePresentation(dids, hydrated, viewer) } async profilesBasic( @@ -62,10 +59,7 @@ export class ActorViews { viewer, includeSoftDeleted: opts?.includeSoftDeleted, }) - return this.profileBasicPresentation(dids, hydrated, { - viewer, - omitLabels: opts?.omitLabels, - }) + return this.profileBasicPresentation(dids, hydrated, viewer, opts) } async profilesList( @@ -293,11 +287,8 @@ export class ActorViews { labels: Labels bam: BlockAndMuteState }, - opts?: { - viewer?: string | null - }, + viewer: string | null, ): ProfileViewMap { - const { viewer } = opts ?? {} const { profiles, lists, labels, bam } = state return dids.reduce((acc, did) => { const prof = profiles[did] @@ -357,12 +348,12 @@ export class ActorViews { profileBasicPresentation( dids: string[], state: ProfileHydrationState, + viewer: string | null, opts?: { - viewer?: string | null omitLabels?: boolean }, ): ProfileViewMap { - const result = this.profilePresentation(dids, state, opts) + const result = this.profilePresentation(dids, state, viewer) return Object.values(result).reduce((acc, prof) => { const profileBasic = { did: prof.did, diff --git a/packages/bsky/src/services/feed/index.ts b/packages/bsky/src/services/feed/index.ts index dab9673d9db..f7fe7b2d817 100644 --- a/packages/bsky/src/services/feed/index.ts +++ b/packages/bsky/src/services/feed/index.ts @@ -44,6 +44,7 @@ import { import { FeedViews } from './views' import { LabelCache } from '../../label-cache' import { threadgateToPostUri, postToThreadgateUri } from './util' +import { mapDefined } from '@atproto/common' export * from './types' @@ -205,6 +206,11 @@ export class FeedService { }, {} as Record) } + async postUrisToFeedItems(uris: string[]): Promise { + const feedItems = await this.getFeedItems(uris) + return mapDefined(uris, (uri) => feedItems[uri]) + } + feedItemRefs(items: FeedRow[]) { const actorDids = new Set() const postUris = new Set() @@ -399,7 +405,7 @@ export class FeedService { const actorInfos = this.services.actor.views.profileBasicPresentation( [...nestedDids], feedState, - { viewer }, + viewer, ) const recordEmbedViews: RecordEmbedViewRecordMap = {} for (const uri of nestedUris) { @@ -423,6 +429,7 @@ export class FeedService { feedState.embeds, feedState.labels, feedState.lists, + viewer, ) recordEmbedViews[uri] = this.views.getRecordEmbedView( uri, diff --git a/packages/bsky/src/services/feed/util.ts b/packages/bsky/src/services/feed/util.ts index b2e2ce8d92d..83b5e59d705 100644 --- a/packages/bsky/src/services/feed/util.ts +++ b/packages/bsky/src/services/feed/util.ts @@ -36,43 +36,72 @@ export const invalidReplyRoot = ( return parent.record.reply?.root.uri !== replyRoot } -export const violatesThreadGate = async ( - db: DatabaseSchema, - did: string, - owner: string, - root: PostRecord | null, +type ParsedThreadGate = { + canReply?: boolean + allowMentions?: boolean + allowFollowing?: boolean + allowListUris?: string[] +} + +export const parseThreadGate = ( + replierDid: string, + ownerDid: string, + rootPost: PostRecord | null, gate: GateRecord | null, -) => { - if (did === owner) return false - if (!gate?.allow) return false +): ParsedThreadGate => { + if (replierDid === ownerDid) { + return { canReply: true } + } + // if gate.allow is unset then *any* reply is allowed, if it is an empty array then *no* reply is allowed + if (!gate || !gate.allow) { + return { canReply: true } + } - const allowMentions = gate.allow.find(isMentionRule) - const allowFollowing = gate.allow.find(isFollowingRule) + const allowMentions = !!gate.allow.find(isMentionRule) + const allowFollowing = !!gate.allow.find(isFollowingRule) const allowListUris = gate.allow?.filter(isListRule).map((item) => item.list) // check mentions first since it's quick and synchronous if (allowMentions) { - const isMentioned = root?.facets?.some((facet) => { - return facet.features.some((item) => isMention(item) && item.did === did) + const isMentioned = rootPost?.facets?.some((facet) => { + return facet.features.some( + (item) => isMention(item) && item.did === replierDid, + ) }) if (isMentioned) { - return false + return { canReply: true, allowMentions, allowFollowing, allowListUris } } } + return { allowMentions, allowFollowing, allowListUris } +} - // check follows and list containment - if (!allowFollowing && !allowListUris.length) { +export const violatesThreadGate = async ( + db: DatabaseSchema, + replierDid: string, + ownerDid: string, + rootPost: PostRecord | null, + gate: GateRecord | null, +) => { + const { + canReply, + allowFollowing, + allowListUris = [], + } = parseThreadGate(replierDid, ownerDid, rootPost, gate) + if (canReply) { + return false + } + if (!allowFollowing && !allowListUris?.length) { return true } const { ref } = db.dynamic const nullResult = sql`${null}` const check = await db - .selectFrom(valuesList([did]).as(sql`subject (did)`)) + .selectFrom(valuesList([replierDid]).as(sql`subject (did)`)) .select([ allowFollowing ? db .selectFrom('follow') - .where('creator', '=', owner) + .where('creator', '=', ownerDid) .whereRef('subjectDid', '=', ref('subject.did')) .select('creator') .as('isFollowed') @@ -91,8 +120,7 @@ export const violatesThreadGate = async ( if (allowFollowing && check?.isFollowed) { return false - } - if (allowListUris.length && check?.isInList) { + } else if (allowListUris.length && check?.isInList) { return false } diff --git a/packages/bsky/src/services/feed/views.ts b/packages/bsky/src/services/feed/views.ts index dc5878db6cd..19dada6dfb7 100644 --- a/packages/bsky/src/services/feed/views.ts +++ b/packages/bsky/src/services/feed/views.ts @@ -5,7 +5,6 @@ import { GeneratorView, PostView, } from '../../lexicon/types/app/bsky/feed/defs' -import { isListRule } from '../../lexicon/types/app/bsky/feed/threadgate' import { Main as EmbedImages, isMain as isEmbedImages, @@ -22,6 +21,8 @@ import { ViewNotFound, ViewRecord, } from '../../lexicon/types/app/bsky/embed/record' +import { Record as PostRecord } from '../../lexicon/types/app/bsky/feed/post' +import { isListRule } from '../../lexicon/types/app/bsky/feed/threadgate' import { PostEmbedViews, FeedGenInfo, @@ -39,6 +40,8 @@ import { ImageUriBuilder } from '../../image/uri' import { LabelCache } from '../../label-cache' import { ActorInfoMap, ActorService } from '../actor' import { ListInfoMap, GraphService } from '../graph' +import { AtUri } from '@atproto/syntax' +import { parseThreadGate } from './util' export class FeedViews { constructor( @@ -91,8 +94,8 @@ export class FeedViews { formatFeed( items: FeedRow[], state: FeedHydrationState, + viewer: string | null, opts?: { - viewer?: string | null usePostViewUnion?: boolean }, ): FeedViewPost[] { @@ -101,7 +104,7 @@ export class FeedViews { const actors = this.services.actor.views.profileBasicPresentation( Object.keys(profiles), state, - opts, + viewer, ) const feed: FeedViewPost[] = [] for (const item of items) { @@ -114,6 +117,7 @@ export class FeedViews { embeds, labels, lists, + viewer, ) // skip over not found & blocked posts if (!post || blocks[post.uri]?.reply) { @@ -149,6 +153,7 @@ export class FeedViews { labels, lists, blocks, + viewer, opts, ) const replyRoot = this.formatMaybePostView( @@ -160,6 +165,7 @@ export class FeedViews { labels, lists, blocks, + viewer, opts, ) if (replyRoot && replyParent) { @@ -182,6 +188,7 @@ export class FeedViews { embeds: PostEmbedViews, labels: Labels, lists: ListInfoMap, + viewer: string | null, ): PostView | undefined { const post = posts[uri] const gate = threadgates[uri] @@ -207,6 +214,14 @@ export class FeedViews { ? { repost: post.requesterRepost ?? undefined, like: post.requesterLike ?? undefined, + replyDisabled: this.userReplyDisabled( + uri, + actors, + posts, + threadgates, + lists, + viewer, + ), } : undefined, labels: [...postLabels, ...postSelfLabels], @@ -217,6 +232,50 @@ export class FeedViews { } } + userReplyDisabled( + uri: string, + actors: ActorInfoMap, + posts: PostInfoMap, + threadgates: ThreadgateInfoMap, + lists: ListInfoMap, + viewer: string | null, + ): boolean | undefined { + if (viewer === null) { + return undefined + } else if (posts[uri]?.violatesThreadGate) { + return true + } + + const rootUriStr: string = + posts[uri]?.record?.['reply']?.['root']?.['uri'] ?? uri + const gate = threadgates[rootUriStr]?.record + if (!gate) { + return undefined + } + const rootPost = posts[rootUriStr]?.record as PostRecord | undefined + const ownerDid = new AtUri(rootUriStr).hostname + + const { + canReply, + allowFollowing, + allowListUris = [], + } = parseThreadGate(viewer, ownerDid, rootPost ?? null, gate ?? null) + + if (canReply) { + return false + } + if (allowFollowing && actors[ownerDid]?.viewer?.followedBy) { + return false + } + for (const listUri of allowListUris) { + const list = lists[listUri] + if (list?.viewerInList) { + return false + } + } + return true + } + formatMaybePostView( uri: string, actors: ActorInfoMap, @@ -226,6 +285,7 @@ export class FeedViews { labels: Labels, lists: ListInfoMap, blocks: PostBlocksMap, + viewer: string | null, opts?: { usePostViewUnion?: boolean }, @@ -238,6 +298,7 @@ export class FeedViews { embeds, labels, lists, + viewer, ) if (!post) { if (!opts?.usePostViewUnion) return diff --git a/packages/bsky/src/services/graph/index.ts b/packages/bsky/src/services/graph/index.ts index eadf035db1a..4d3a117b0f4 100644 --- a/packages/bsky/src/services/graph/index.ts +++ b/packages/bsky/src/services/graph/index.ts @@ -91,6 +91,12 @@ export class GraphService { .whereRef('list_block.subjectUri', '=', ref('list.uri')) .select('list_block.uri') .as('viewerListBlockUri'), + this.db.db + .selectFrom('list_item') + .whereRef('list_item.listUri', '=', ref('list.uri')) + .where('list_item.subjectDid', '=', viewer ?? '') + .select('list_item.uri') + .as('viewerInList'), ]) } diff --git a/packages/bsky/src/services/graph/types.ts b/packages/bsky/src/services/graph/types.ts index f5ee0c13026..5ff254dc383 100644 --- a/packages/bsky/src/services/graph/types.ts +++ b/packages/bsky/src/services/graph/types.ts @@ -4,6 +4,7 @@ import { List } from '../../db/tables/list' export type ListInfo = Selectable & { viewerMuted: string | null viewerListBlockUri: string | null + viewerInList: string | null } export type ListInfoMap = Record diff --git a/packages/bsky/src/services/indexing/plugins/post.ts b/packages/bsky/src/services/indexing/plugins/post.ts index 7173c04a991..396544b8f26 100644 --- a/packages/bsky/src/services/indexing/plugins/post.ts +++ b/packages/bsky/src/services/indexing/plugins/post.ts @@ -420,7 +420,7 @@ async function validateReply( const violatesThreadGate = await feedutil.violatesThreadGate( db, creator, - new AtUri(reply.root.uri).host, + new AtUri(reply.root.uri).hostname, replyRefs.root?.record ?? null, replyRefs.gate?.record ?? null, ) diff --git a/packages/bsky/tests/__snapshots__/indexing.test.ts.snap b/packages/bsky/tests/__snapshots__/indexing.test.ts.snap index 0ed4aeb4d02..88c02c6e3e0 100644 --- a/packages/bsky/tests/__snapshots__/indexing.test.ts.snap +++ b/packages/bsky/tests/__snapshots__/indexing.test.ts.snap @@ -518,9 +518,6 @@ Object { "viewer": Object {}, }, "replies": Array [], - "viewer": Object { - "canReply": true, - }, }, } `; @@ -587,9 +584,6 @@ Object { "viewer": Object {}, }, "replies": Array [], - "viewer": Object { - "canReply": true, - }, }, } `; diff --git a/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap b/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap index 7843adb6cc8..86fe23283c4 100644 --- a/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap +++ b/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap @@ -126,9 +126,6 @@ Object { "uri": "record(0)", "viewer": Object {}, }, - "viewer": Object { - "canReply": true, - }, }, } `; @@ -204,9 +201,6 @@ Object { "viewer": Object {}, }, "replies": Array [], - "viewer": Object { - "canReply": true, - }, }, } `; @@ -288,9 +282,6 @@ Object { "uri": "record(7)", }, ], - "viewer": Object { - "canReply": true, - }, }, } `; diff --git a/packages/bsky/tests/views/__snapshots__/blocks.test.ts.snap b/packages/bsky/tests/views/__snapshots__/blocks.test.ts.snap index ba5c00182de..2a27fcf4955 100644 --- a/packages/bsky/tests/views/__snapshots__/blocks.test.ts.snap +++ b/packages/bsky/tests/views/__snapshots__/blocks.test.ts.snap @@ -126,9 +126,6 @@ Object { "uri": "record(0)", "viewer": Object {}, }, - "viewer": Object { - "canReply": true, - }, }, } `; @@ -204,9 +201,6 @@ Object { "viewer": Object {}, }, "replies": Array [], - "viewer": Object { - "canReply": true, - }, }, } `; @@ -359,9 +353,6 @@ Object { }, }, ], - "viewer": Object { - "canReply": true, - }, }, } `; diff --git a/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap b/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap index b58e7a3734f..438b48b4fdd 100644 --- a/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap +++ b/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap @@ -292,9 +292,6 @@ Object { }, }, ], - "viewer": Object { - "canReply": true, - }, } `; diff --git a/packages/bsky/tests/views/__snapshots__/mutes.test.ts.snap b/packages/bsky/tests/views/__snapshots__/mutes.test.ts.snap index ca8b664ec91..0e1c14c2696 100644 --- a/packages/bsky/tests/views/__snapshots__/mutes.test.ts.snap +++ b/packages/bsky/tests/views/__snapshots__/mutes.test.ts.snap @@ -269,8 +269,5 @@ Object { }, }, ], - "viewer": Object { - "canReply": true, - }, } `; diff --git a/packages/bsky/tests/views/__snapshots__/thread.test.ts.snap b/packages/bsky/tests/views/__snapshots__/thread.test.ts.snap index 6bc84753951..fb0fd6a3224 100644 --- a/packages/bsky/tests/views/__snapshots__/thread.test.ts.snap +++ b/packages/bsky/tests/views/__snapshots__/thread.test.ts.snap @@ -195,9 +195,6 @@ Object { }, }, "replies": Array [], - "viewer": Object { - "canReply": true, - }, } `; @@ -437,9 +434,6 @@ Object { ], }, ], - "viewer": Object { - "canReply": true, - }, } `; @@ -615,9 +609,6 @@ Object { }, }, ], - "viewer": Object { - "canReply": true, - }, } `; @@ -771,9 +762,6 @@ Object { ], }, ], - "viewer": Object { - "canReply": true, - }, } `; @@ -826,9 +814,6 @@ Object { "viewer": Object {}, }, "replies": Array [], - "viewer": Object { - "canReply": true, - }, } `; @@ -896,9 +881,6 @@ Object { "viewer": Object {}, }, "replies": Array [], - "viewer": Object { - "canReply": true, - }, } `; @@ -968,9 +950,6 @@ Object { }, }, "replies": Array [], - "viewer": Object { - "canReply": true, - }, } `; @@ -1040,9 +1019,6 @@ Object { }, }, "replies": Array [], - "viewer": Object { - "canReply": true, - }, } `; @@ -1243,9 +1219,6 @@ Object { ], }, ], - "viewer": Object { - "canReply": true, - }, } `; @@ -1384,8 +1357,5 @@ Object { "replies": Array [], }, ], - "viewer": Object { - "canReply": true, - }, } `; diff --git a/packages/bsky/tests/views/threadgating.test.ts b/packages/bsky/tests/views/threadgating.test.ts index 8cfaedba44e..53e5961b595 100644 --- a/packages/bsky/tests/views/threadgating.test.ts +++ b/packages/bsky/tests/views/threadgating.test.ts @@ -29,6 +29,19 @@ describe('views with thread gating', () => { await network.close() }) + // check that replyDisabled state is applied correctly in a simple method like getPosts + const checkReplyDisabled = async ( + uri: string, + user: string, + blocked: boolean | undefined, + ) => { + const res = await agent.api.app.bsky.feed.getPosts( + { uris: [uri] }, + { headers: await network.serviceHeaders(user) }, + ) + expect(res.data.posts[0].viewer?.replyDisabled).toBe(blocked) + } + it('applies gate for empty rules.', async () => { const post = await sc.post(sc.dids.carol, 'empty rules') await pdsAgent.api.app.bsky.feed.threadgate.create( @@ -46,8 +59,9 @@ describe('views with thread gating', () => { ) assert(isThreadViewPost(thread)) expect(forSnapshot(thread.post.threadgate)).toMatchSnapshot() - expect(thread.viewer).toEqual({ canReply: false }) + expect(thread.post.viewer).toEqual({ replyDisabled: true }) expect(thread.replies?.length).toEqual(0) + await checkReplyDisabled(post.ref.uriStr, sc.dids.alice, true) }) it('applies gate for mention rule.', async () => { @@ -98,7 +112,8 @@ describe('views with thread gating', () => { { headers: await network.serviceHeaders(sc.dids.alice) }, ) assert(isThreadViewPost(aliceThread)) - expect(aliceThread.viewer).toEqual({ canReply: false }) + expect(aliceThread.post.viewer).toEqual({ replyDisabled: true }) + await checkReplyDisabled(post.ref.uriStr, sc.dids.alice, true) const { data: { thread: danThread }, } = await agent.api.app.bsky.feed.getPostThread( @@ -107,7 +122,8 @@ describe('views with thread gating', () => { ) assert(isThreadViewPost(danThread)) expect(forSnapshot(danThread.post.threadgate)).toMatchSnapshot() - expect(danThread.viewer).toEqual({ canReply: true }) + expect(danThread.post.viewer).toEqual({ replyDisabled: false }) + await checkReplyDisabled(post.ref.uriStr, sc.dids.dan, false) const [reply, ...otherReplies] = danThread.replies ?? [] assert(isThreadViewPost(reply)) expect(otherReplies.length).toEqual(0) @@ -146,7 +162,8 @@ describe('views with thread gating', () => { { headers: await network.serviceHeaders(sc.dids.dan) }, ) assert(isThreadViewPost(danThread)) - expect(danThread.viewer).toEqual({ canReply: false }) + expect(danThread.post.viewer).toEqual({ replyDisabled: true }) + await checkReplyDisabled(post.ref.uriStr, sc.dids.dan, true) const { data: { thread: aliceThread }, } = await agent.api.app.bsky.feed.getPostThread( @@ -155,7 +172,8 @@ describe('views with thread gating', () => { ) assert(isThreadViewPost(aliceThread)) expect(forSnapshot(aliceThread.post.threadgate)).toMatchSnapshot() - expect(aliceThread.viewer).toEqual({ canReply: true }) + expect(aliceThread.post.viewer).toEqual({ replyDisabled: false }) + await checkReplyDisabled(post.ref.uriStr, sc.dids.alice, false) const [reply, ...otherReplies] = aliceThread.replies ?? [] assert(isThreadViewPost(reply)) expect(otherReplies.length).toEqual(0) @@ -235,7 +253,8 @@ describe('views with thread gating', () => { { headers: await network.serviceHeaders(sc.dids.bob) }, ) assert(isThreadViewPost(bobThread)) - expect(bobThread.viewer).toEqual({ canReply: false }) + expect(bobThread.post.viewer).toEqual({ replyDisabled: true }) + await checkReplyDisabled(post.ref.uriStr, sc.dids.bob, true) const { data: { thread: aliceThread }, } = await agent.api.app.bsky.feed.getPostThread( @@ -243,7 +262,8 @@ describe('views with thread gating', () => { { headers: await network.serviceHeaders(sc.dids.alice) }, ) assert(isThreadViewPost(aliceThread)) - expect(aliceThread.viewer).toEqual({ canReply: true }) + expect(aliceThread.post.viewer).toEqual({ replyDisabled: false }) + await checkReplyDisabled(post.ref.uriStr, sc.dids.alice, false) const { data: { thread: danThread }, } = await agent.api.app.bsky.feed.getPostThread( @@ -252,7 +272,8 @@ describe('views with thread gating', () => { ) assert(isThreadViewPost(danThread)) expect(forSnapshot(danThread.post.threadgate)).toMatchSnapshot() - expect(danThread.viewer).toEqual({ canReply: true }) + expect(danThread.post.viewer).toEqual({ replyDisabled: false }) + await checkReplyDisabled(post.ref.uriStr, sc.dids.dan, false) const [reply1, reply2, ...otherReplies] = aliceThread.replies ?? [] assert(isThreadViewPost(reply1)) assert(isThreadViewPost(reply2)) @@ -292,8 +313,9 @@ describe('views with thread gating', () => { ) assert(isThreadViewPost(thread)) expect(forSnapshot(thread.post.threadgate)).toMatchSnapshot() - expect(thread.viewer).toEqual({ canReply: false }) + expect(thread.post.viewer).toEqual({ replyDisabled: true }) expect(thread.replies?.length).toEqual(0) + await checkReplyDisabled(post.ref.uriStr, sc.dids.alice, true) }) it('applies gate for multiple rules.', async () => { @@ -339,7 +361,8 @@ describe('views with thread gating', () => { { headers: await network.serviceHeaders(sc.dids.bob) }, ) assert(isThreadViewPost(bobThread)) - expect(bobThread.viewer).toEqual({ canReply: false }) + expect(bobThread.post.viewer).toEqual({ replyDisabled: true }) + await checkReplyDisabled(post.ref.uriStr, sc.dids.bob, true) const { data: { thread: aliceThread }, } = await agent.api.app.bsky.feed.getPostThread( @@ -347,7 +370,8 @@ describe('views with thread gating', () => { { headers: await network.serviceHeaders(sc.dids.alice) }, ) assert(isThreadViewPost(aliceThread)) - expect(aliceThread.viewer).toEqual({ canReply: true }) + expect(aliceThread.post.viewer).toEqual({ replyDisabled: false }) + await checkReplyDisabled(post.ref.uriStr, sc.dids.alice, false) const { data: { thread: danThread }, } = await agent.api.app.bsky.feed.getPostThread( @@ -356,7 +380,8 @@ describe('views with thread gating', () => { ) assert(isThreadViewPost(danThread)) expect(forSnapshot(danThread.post.threadgate)).toMatchSnapshot() - expect(danThread.viewer).toEqual({ canReply: true }) + expect(danThread.post.viewer).toEqual({ replyDisabled: false }) + await checkReplyDisabled(post.ref.uriStr, sc.dids.dan, false) const [reply1, reply2, ...otherReplies] = aliceThread.replies ?? [] assert(isThreadViewPost(reply1)) assert(isThreadViewPost(reply2)) @@ -387,7 +412,8 @@ describe('views with thread gating', () => { ) assert(isThreadViewPost(thread)) expect(forSnapshot(thread.post.threadgate)).toMatchSnapshot() - expect(thread.viewer).toEqual({ canReply: true }) + expect(thread.post.viewer).toEqual({ replyDisabled: false }) + await checkReplyDisabled(post.ref.uriStr, sc.dids.alice, false) const [reply, ...otherReplies] = thread.replies ?? [] assert(isThreadViewPost(reply)) expect(otherReplies.length).toEqual(0) @@ -438,7 +464,8 @@ describe('views with thread gating', () => { { headers: await network.serviceHeaders(sc.dids.dan) }, ) assert(isThreadViewPost(danThread)) - expect(danThread.viewer).toEqual({ canReply: false }) + expect(danThread.post.viewer).toEqual({ replyDisabled: true }) + await checkReplyDisabled(orphanedReply.ref.uriStr, sc.dids.dan, true) const { data: { thread: aliceThread }, } = await agent.api.app.bsky.feed.getPostThread( @@ -451,7 +478,8 @@ describe('views with thread gating', () => { aliceThread.parent.uri === post.ref.uriStr, ) expect(aliceThread.post.threadgate).toMatchSnapshot() - expect(aliceThread.viewer).toEqual({ canReply: true }) + expect(aliceThread.post.viewer).toEqual({ replyDisabled: false }) + await checkReplyDisabled(orphanedReply.ref.uriStr, sc.dids.alice, false) const [reply, ...otherReplies] = aliceThread.replies ?? [] assert(isThreadViewPost(reply)) expect(otherReplies.length).toEqual(0) @@ -480,7 +508,8 @@ describe('views with thread gating', () => { ) assert(isThreadViewPost(thread)) expect(forSnapshot(thread.post.threadgate)).toMatchSnapshot() - expect(thread.viewer).toEqual({ canReply: true }) + expect(thread.post.viewer).toEqual({ replyDisabled: false }) + await checkReplyDisabled(post.ref.uriStr, sc.dids.carol, false) const [reply, ...otherReplies] = thread.replies ?? [] assert(isThreadViewPost(reply)) expect(otherReplies.length).toEqual(0) @@ -516,10 +545,11 @@ describe('views with thread gating', () => { { headers: await network.serviceHeaders(sc.dids.alice) }, ) assert(isThreadViewPost(thread)) - expect(thread.viewer).toEqual({ canReply: false }) // nobody can reply to this, not even alice. + expect(thread.post.viewer).toEqual({ replyDisabled: true }) // nobody can reply to this, not even alice. expect(thread.replies).toBeUndefined() expect(thread.parent).toBeUndefined() expect(thread.post.threadgate).toBeUndefined() + await checkReplyDisabled(badReply.ref.uriStr, sc.dids.alice, true) // check feed view const { data: { feed }, @@ -552,8 +582,9 @@ describe('views with thread gating', () => { ) assert(isThreadViewPost(threadA)) expect(threadA.post.threadgate).toBeUndefined() - expect(threadA.viewer).toEqual({ canReply: true }) + expect(threadA.post.viewer).toEqual({}) expect(threadA.replies?.length).toEqual(1) + await checkReplyDisabled(postA.ref.uriStr, sc.dids.alice, undefined) const { data: { thread: threadB }, } = await agent.api.app.bsky.feed.getPostThread( @@ -562,7 +593,8 @@ describe('views with thread gating', () => { ) assert(isThreadViewPost(threadB)) expect(threadB.post.threadgate).toBeUndefined() - expect(threadB.viewer).toEqual({ canReply: true }) + expect(threadB.post.viewer).toEqual({}) + await checkReplyDisabled(postB.ref.uriStr, sc.dids.alice, undefined) expect(threadB.replies?.length).toEqual(1) }) }) diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index 14c1be9f477..cc3f09f0c4e 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -4892,6 +4892,9 @@ export const schemaDict = { type: 'string', format: 'at-uri', }, + replyDisabled: { + type: 'boolean', + }, }, }, feedViewPost: { @@ -4975,10 +4978,6 @@ export const schemaDict = { ], }, }, - viewer: { - type: 'ref', - ref: 'lex:app.bsky.feed.defs#viewerThreadState', - }, }, }, notFoundPost: { @@ -5027,14 +5026,6 @@ export const schemaDict = { }, }, }, - viewerThreadState: { - type: 'object', - properties: { - canReply: { - type: 'boolean', - }, - }, - }, generatorView: { type: 'object', required: ['uri', 'cid', 'did', 'creator', 'displayName', 'indexedAt'], diff --git a/packages/pds/src/lexicon/types/app/bsky/feed/defs.ts b/packages/pds/src/lexicon/types/app/bsky/feed/defs.ts index 08d34d88ebb..382d3f58ecf 100644 --- a/packages/pds/src/lexicon/types/app/bsky/feed/defs.ts +++ b/packages/pds/src/lexicon/types/app/bsky/feed/defs.ts @@ -48,6 +48,7 @@ export function validatePostView(v: unknown): ValidationResult { export interface ViewerState { repost?: string like?: string + replyDisabled?: boolean [k: string]: unknown } @@ -137,7 +138,6 @@ export interface ThreadViewPost { | BlockedPost | { $type: string; [k: string]: unknown } )[] - viewer?: ViewerThreadState [k: string]: unknown } @@ -208,23 +208,6 @@ export function validateBlockedAuthor(v: unknown): ValidationResult { return lexicons.validate('app.bsky.feed.defs#blockedAuthor', v) } -export interface ViewerThreadState { - canReply?: boolean - [k: string]: unknown -} - -export function isViewerThreadState(v: unknown): v is ViewerThreadState { - return ( - isObj(v) && - hasProp(v, '$type') && - v.$type === 'app.bsky.feed.defs#viewerThreadState' - ) -} - -export function validateViewerThreadState(v: unknown): ValidationResult { - return lexicons.validate('app.bsky.feed.defs#viewerThreadState', v) -} - export interface GeneratorView { uri: string cid: string From 1f9040a44dc46c14237792c0b1dd84b22c81b0b9 Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Thu, 30 Nov 2023 17:53:56 +0100 Subject: [PATCH 41/59] Evented architecture for moderation system (#1617) * :construction: WIP with proposed lexicons for event based mod architecture * :construction: Remove unnecessary moderation action lexicon * :construction: Working on event based actions * :sparkles: Add escalated subject status * :bug: Alright, fixed the error in lexicon * :construction: Working through reversal * :sparkles: Cleanup build errors * :sparkles: Add subject status endpoint * :sparkles: Add handler * :sparkles: get reports from mod actions table * :rightwards_twisted_arrows: Merge with upstream * :construction: Builds but test network doesnt start * :sparkles: Tests passing on event based status change * :sparkles: Rename index * :recycle: Rename takeModerationAction->emitModerationEvent * :sparkles: Implement label reversal * :white_check_mark: Auto-revert test working * :recycle: :white_check_mark: Refactored to event types and tests are passing * :sparkles: Add takedown event sequence validation * :sparkles: Adds support for blobCid status * :broom: Cleanup unnecessary method: * :sparkles: Hydrate handles with status and events * :sparkles: Re-implement auto reversal * :sparkles: Add takendown and mute filters * :sparkles: Allow filtering events by type * :sparkles: Allow filtering events by creator did * :sparkles: Add subjectStatus to record and repoview * :sparkles: Add persistent note feature * :sparkles: Log send email event * :bug: Fix logging send email event * :sparkles: Better type * :sparkles: Adjust migration to create separate moderation_event table * :broom: Cleanup types * :white_check_mark: Adjust tests with mod event emitter * :sparkles: Fix more tests around takedowns * :white_check_mark: Get test suite to pass * :white_check_mark: Get test suite to pass for pds * :white_check_mark: Get test suite to pass for pds * :white_check_mark: Update snapshot for feedgen * :white_check_mark: Why are more snapshots updating? * :recycle: Rename getModerationEvents -> queryModerationEvents * :recycle: Rename getModerationStatuses -> queryModerationStatuses * :recycle: Rename persistNote->sticky * :bug: Rename subject * :recycle: Cleanup expiresAt for scheduled actions * :sparkles: Add more tests, allow fetching mod history for all content by a user * :white_check_mark: Fix repo and record tests * :sparkles: Migrate reports and actions to events * :bug: Fix escalated status overwrite * :sparkles: Implement direct sql query to create events from actions and reports * :construction: Adding keyset pagination for subject statuses * :sparkles: Add migration for lastReportedAt * :sparkles: Migrate blob cids * :sparkles: Fix pagination on mod subject list endpoint * :bug: Fix blob actions * :white_check_mark: All tests passing on bsky package * :white_check_mark: Bring back snapshots * :white_check_mark: Skipping timeline test temporarily * :white_check_mark: Skipping some more tests to isolate failing ones * :white_check_mark: Bring back list-feed test * :white_check_mark: Bring back timeline test * :white_check_mark: Fix label action in seeding * :white_check_mark: Enable timeline proxied test * :white_check_mark: Enable search actor proxied test * :white_check_mark: Enable feedgen tests * :white_check_mark: Fix test for admin/get-record * :sparkles: Move note to comment for subject status * :sparkles: Accept comments in mute event * :sparkles: Remap flag event to ack event * :bug: Add legacyRef in report union selection * @atproto/api 0.6.24-next.0 * @atproto/api 0.6.24-next.1 * :sparkles: Adjust migration export and add index for blobCids column * :sparkles: Maintin action ids when migrating * :sparkles: Paginate events using createdAt timestamp * :white_check_mark: Update snapshot for pds test with events cursor update * :white_check_mark: Use only events for snapshot testing * :white_check_mark: Use only events for snapshot in the remaining test * relative paths to lexicons for build * fix bsky periodic event reversal in service entrypoint * :sparkles: Allow comments in takedown and label * :sparkles: Only import reports on consecutive run of the migration script * :sparkles: Adjust moderation property of blob entries * determine latest reports to migrate * :sparkles: Process new reports for subject status * :sparkles: Process unresolved reports on first migration run * fix transaction error, process just unresolved reports, make reported-at updates safe for reruns * tidy --------- Co-authored-by: Devin Ivy --- lexicons/com/atproto/admin/defs.json | 319 +++- ...onAction.json => emitModerationEvent.json} | 38 +- .../atproto/admin/getModerationActions.json | 40 - ...ionAction.json => getModerationEvent.json} | 6 +- .../atproto/admin/getModerationReport.json | 24 - .../atproto/admin/getModerationReports.json | 65 - .../atproto/admin/queryModerationEvents.json | 60 + .../admin/queryModerationStatuses.json | 95 ++ .../admin/resolveModerationReports.json | 29 - .../admin/reverseModerationAction.json | 29 - lexicons/com/atproto/admin/sendEmail.json | 5 +- packages/api/package.json | 2 +- packages/api/src/client/index.ts | 125 +- packages/api/src/client/lexicons.ts | 1024 ++++++------ .../client/types/com/atproto/admin/defs.ts | 360 ++++- ...rationAction.ts => emitModerationEvent.ts} | 24 +- ...erationAction.ts => getModerationEvent.ts} | 2 +- .../com/atproto/admin/getModerationReport.ts | 32 - .../com/atproto/admin/getModerationReports.ts | 53 - ...ionActions.ts => queryModerationEvents.ts} | 9 +- .../atproto/admin/queryModerationStatuses.ts | 60 + .../atproto/admin/resolveModerationReports.ts | 38 - .../atproto/admin/reverseModerationAction.ts | 38 - .../types/com/atproto/admin/sendEmail.ts | 1 + packages/bsky/src/api/blob-resolver.ts | 17 +- .../com/atproto/admin/emitModerationEvent.ts | 220 +++ .../com/atproto/admin/getModerationAction.ts | 44 - ...rationActions.ts => getModerationEvent.ts} | 16 +- .../com/atproto/admin/getModerationReport.ts | 43 - .../src/api/com/atproto/admin/getRecord.ts | 1 + ...ionReports.ts => queryModerationEvents.ts} | 29 +- .../atproto/admin/queryModerationStatuses.ts | 55 + .../atproto/admin/resolveModerationReports.ts | 24 - .../atproto/admin/reverseModerationAction.ts | 115 -- .../com/atproto/admin/takeModerationAction.ts | 156 -- .../com/atproto/moderation/createReport.ts | 16 +- .../src/api/com/atproto/moderation/util.ts | 58 +- packages/bsky/src/api/index.ts | 22 +- packages/bsky/src/auto-moderator/index.ts | 45 +- ...33377Z-create-moderation-subject-status.ts | 123 ++ packages/bsky/src/db/migrations/index.ts | 1 + packages/bsky/src/db/pagination.ts | 29 +- ... => periodic-moderation-event-reversal.ts} | 96 +- packages/bsky/src/db/tables/moderation.ts | 87 +- packages/bsky/src/index.ts | 3 +- packages/bsky/src/lexicon/index.ts | 97 +- packages/bsky/src/lexicon/lexicons.ts | 1024 ++++++------ .../lexicon/types/com/atproto/admin/defs.ts | 360 ++++- .../com/atproto/admin/emitModerationEvent.ts} | 24 +- ...erationAction.ts => getModerationEvent.ts} | 2 +- .../com/atproto/admin/getModerationReport.ts | 41 - ...ionActions.ts => queryModerationEvents.ts} | 9 +- ...nReports.ts => queryModerationStatuses.ts} | 35 +- .../atproto/admin/resolveModerationReports.ts | 49 - .../atproto/admin/reverseModerationAction.ts | 49 - .../types/com/atproto/admin/sendEmail.ts | 1 + packages/bsky/src/migrate-moderation-data.ts | 414 +++++ .../bsky/src/services/moderation/index.ts | 775 +++++----- .../src/services/moderation/pagination.ts | 96 ++ .../bsky/src/services/moderation/status.ts | 244 +++ .../bsky/src/services/moderation/types.ts | 49 + .../bsky/src/services/moderation/views.ts | 584 +++---- .../get-moderation-action.test.ts.snap | 172 --- .../get-moderation-actions.test.ts.snap | 178 --- .../get-moderation-report.test.ts.snap | 177 --- .../get-moderation-reports.test.ts.snap | 307 ---- .../__snapshots__/get-record.test.ts.snap | 162 +- .../admin/__snapshots__/get-repo.test.ts.snap | 74 +- .../moderation-events.test.ts.snap | 146 ++ .../moderation-statuses.test.ts.snap | 64 + .../__snapshots__/moderation.test.ts.snap | 125 -- .../tests/admin/get-moderation-action.test.ts | 100 -- .../admin/get-moderation-actions.test.ts | 164 -- .../tests/admin/get-moderation-report.test.ts | 100 -- .../admin/get-moderation-reports.test.ts | 332 ---- packages/bsky/tests/admin/get-record.test.ts | 14 +- packages/bsky/tests/admin/get-repo.test.ts | 14 +- .../tests/admin/moderation-events.test.ts | 221 +++ .../tests/admin/moderation-statuses.test.ts | 145 ++ packages/bsky/tests/admin/moderation.test.ts | 1372 ++++++----------- packages/bsky/tests/admin/repo-search.test.ts | 5 +- .../auto-moderator/fuzzy-matcher.test.ts | 3 +- .../tests/auto-moderator/takedowns.test.ts | 63 +- packages/bsky/tests/feed-generation.test.ts | 5 +- .../bsky/tests/views/actor-search.test.ts | 5 +- packages/bsky/tests/views/author-feed.test.ts | 78 +- packages/bsky/tests/views/follows.test.ts | 75 +- packages/bsky/tests/views/list-feed.test.ts | 26 +- .../bsky/tests/views/notifications.test.ts | 22 +- packages/bsky/tests/views/profile.test.ts | 38 +- packages/bsky/tests/views/thread.test.ts | 80 +- packages/bsky/tests/views/timeline.test.ts | 34 +- packages/dev-env/src/seed-client.ts | 53 +- ...rationAction.ts => emitModerationEvent.ts} | 4 +- .../com/atproto/admin/getModerationAction.ts | 20 - ...rationActions.ts => getModerationEvent.ts} | 8 +- .../pds/src/api/com/atproto/admin/index.ts | 22 +- ...ionReports.ts => queryModerationEvents.ts} | 4 +- ...onReport.ts => queryModerationStatuses.ts} | 8 +- .../atproto/admin/resolveModerationReports.ts | 20 - .../atproto/admin/reverseModerationAction.ts | 21 - .../src/api/com/atproto/admin/sendEmail.ts | 18 +- .../src/api/com/atproto/moderation/util.ts | 21 +- packages/pds/src/db/tables/moderation.ts | 22 +- packages/pds/src/lexicon/index.ts | 97 +- packages/pds/src/lexicon/lexicons.ts | 1024 ++++++------ .../lexicon/types/com/atproto/admin/defs.ts | 360 ++++- .../com/atproto/admin/emitModerationEvent.ts} | 24 +- .../com/atproto/admin/getModerationAction.ts | 41 - ...erationReport.ts => getModerationEvent.ts} | 2 +- ...ionActions.ts => queryModerationEvents.ts} | 9 +- ...nReports.ts => queryModerationStatuses.ts} | 35 +- .../atproto/admin/resolveModerationReports.ts | 49 - .../atproto/admin/reverseModerationAction.ts | 49 - .../types/com/atproto/admin/sendEmail.ts | 1 + .../proxied/__snapshots__/admin.test.ts.snap | 403 ++--- packages/pds/tests/proxied/admin.test.ts | 170 +- packages/pds/tests/seeds/basic.ts | 11 +- services/bsky/api.js | 14 +- 119 files changed, 6855 insertions(+), 7487 deletions(-) rename lexicons/com/atproto/admin/{takeModerationAction.json => emitModerationEvent.json} (50%) delete mode 100644 lexicons/com/atproto/admin/getModerationActions.json rename lexicons/com/atproto/admin/{getModerationAction.json => getModerationEvent.json} (67%) delete mode 100644 lexicons/com/atproto/admin/getModerationReport.json delete mode 100644 lexicons/com/atproto/admin/getModerationReports.json create mode 100644 lexicons/com/atproto/admin/queryModerationEvents.json create mode 100644 lexicons/com/atproto/admin/queryModerationStatuses.json delete mode 100644 lexicons/com/atproto/admin/resolveModerationReports.json delete mode 100644 lexicons/com/atproto/admin/reverseModerationAction.json rename packages/api/src/client/types/com/atproto/admin/{takeModerationAction.ts => emitModerationEvent.ts} (68%) rename packages/api/src/client/types/com/atproto/admin/{getModerationAction.ts => getModerationEvent.ts} (90%) delete mode 100644 packages/api/src/client/types/com/atproto/admin/getModerationReport.ts delete mode 100644 packages/api/src/client/types/com/atproto/admin/getModerationReports.ts rename packages/api/src/client/types/com/atproto/admin/{getModerationActions.ts => queryModerationEvents.ts} (60%) create mode 100644 packages/api/src/client/types/com/atproto/admin/queryModerationStatuses.ts delete mode 100644 packages/api/src/client/types/com/atproto/admin/resolveModerationReports.ts delete mode 100644 packages/api/src/client/types/com/atproto/admin/reverseModerationAction.ts create mode 100644 packages/bsky/src/api/com/atproto/admin/emitModerationEvent.ts delete mode 100644 packages/bsky/src/api/com/atproto/admin/getModerationAction.ts rename packages/bsky/src/api/com/atproto/admin/{getModerationActions.ts => getModerationEvent.ts} (50%) delete mode 100644 packages/bsky/src/api/com/atproto/admin/getModerationReport.ts rename packages/bsky/src/api/com/atproto/admin/{getModerationReports.ts => queryModerationEvents.ts} (52%) create mode 100644 packages/bsky/src/api/com/atproto/admin/queryModerationStatuses.ts delete mode 100644 packages/bsky/src/api/com/atproto/admin/resolveModerationReports.ts delete mode 100644 packages/bsky/src/api/com/atproto/admin/reverseModerationAction.ts delete mode 100644 packages/bsky/src/api/com/atproto/admin/takeModerationAction.ts create mode 100644 packages/bsky/src/db/migrations/20231003T202833377Z-create-moderation-subject-status.ts rename packages/bsky/src/db/{periodic-moderation-action-reversal.ts => periodic-moderation-event-reversal.ts} (54%) rename packages/{pds/src/lexicon/types/com/atproto/admin/takeModerationAction.ts => bsky/src/lexicon/types/com/atproto/admin/emitModerationEvent.ts} (71%) rename packages/bsky/src/lexicon/types/com/atproto/admin/{getModerationAction.ts => getModerationEvent.ts} (94%) delete mode 100644 packages/bsky/src/lexicon/types/com/atproto/admin/getModerationReport.ts rename packages/bsky/src/lexicon/types/com/atproto/admin/{getModerationActions.ts => queryModerationEvents.ts} (69%) rename packages/bsky/src/lexicon/types/com/atproto/admin/{getModerationReports.ts => queryModerationStatuses.ts} (56%) delete mode 100644 packages/bsky/src/lexicon/types/com/atproto/admin/resolveModerationReports.ts delete mode 100644 packages/bsky/src/lexicon/types/com/atproto/admin/reverseModerationAction.ts create mode 100644 packages/bsky/src/migrate-moderation-data.ts create mode 100644 packages/bsky/src/services/moderation/pagination.ts create mode 100644 packages/bsky/src/services/moderation/status.ts create mode 100644 packages/bsky/src/services/moderation/types.ts delete mode 100644 packages/bsky/tests/admin/__snapshots__/get-moderation-action.test.ts.snap delete mode 100644 packages/bsky/tests/admin/__snapshots__/get-moderation-actions.test.ts.snap delete mode 100644 packages/bsky/tests/admin/__snapshots__/get-moderation-report.test.ts.snap delete mode 100644 packages/bsky/tests/admin/__snapshots__/get-moderation-reports.test.ts.snap create mode 100644 packages/bsky/tests/admin/__snapshots__/moderation-events.test.ts.snap create mode 100644 packages/bsky/tests/admin/__snapshots__/moderation-statuses.test.ts.snap delete mode 100644 packages/bsky/tests/admin/get-moderation-action.test.ts delete mode 100644 packages/bsky/tests/admin/get-moderation-actions.test.ts delete mode 100644 packages/bsky/tests/admin/get-moderation-report.test.ts delete mode 100644 packages/bsky/tests/admin/get-moderation-reports.test.ts create mode 100644 packages/bsky/tests/admin/moderation-events.test.ts create mode 100644 packages/bsky/tests/admin/moderation-statuses.test.ts rename packages/pds/src/api/com/atproto/admin/{takeModerationAction.ts => emitModerationEvent.ts} (79%) delete mode 100644 packages/pds/src/api/com/atproto/admin/getModerationAction.ts rename packages/pds/src/api/com/atproto/admin/{getModerationActions.ts => getModerationEvent.ts} (69%) rename packages/pds/src/api/com/atproto/admin/{getModerationReports.ts => queryModerationEvents.ts} (78%) rename packages/pds/src/api/com/atproto/admin/{getModerationReport.ts => queryModerationStatuses.ts} (68%) delete mode 100644 packages/pds/src/api/com/atproto/admin/resolveModerationReports.ts delete mode 100644 packages/pds/src/api/com/atproto/admin/reverseModerationAction.ts rename packages/{bsky/src/lexicon/types/com/atproto/admin/takeModerationAction.ts => pds/src/lexicon/types/com/atproto/admin/emitModerationEvent.ts} (71%) delete mode 100644 packages/pds/src/lexicon/types/com/atproto/admin/getModerationAction.ts rename packages/pds/src/lexicon/types/com/atproto/admin/{getModerationReport.ts => getModerationEvent.ts} (94%) rename packages/pds/src/lexicon/types/com/atproto/admin/{getModerationActions.ts => queryModerationEvents.ts} (69%) rename packages/pds/src/lexicon/types/com/atproto/admin/{getModerationReports.ts => queryModerationStatuses.ts} (56%) delete mode 100644 packages/pds/src/lexicon/types/com/atproto/admin/resolveModerationReports.ts delete mode 100644 packages/pds/src/lexicon/types/com/atproto/admin/reverseModerationAction.ts diff --git a/lexicons/com/atproto/admin/defs.json b/lexicons/com/atproto/admin/defs.json index 42dc7165423..dcded1387d3 100644 --- a/lexicons/com/atproto/admin/defs.json +++ b/lexicons/com/atproto/admin/defs.json @@ -10,57 +10,67 @@ "ref": { "type": "string" } } }, - "actionView": { + "modEventView": { "type": "object", "required": [ "id", - "action", + "event", "subject", "subjectBlobCids", - "reason", "createdBy", - "createdAt", - "resolvedReportIds" + "createdAt" ], "properties": { "id": { "type": "integer" }, - "action": { "type": "ref", "ref": "#actionType" }, - "durationInHours": { - "type": "integer", - "description": "Indicates how long this action is meant to be in effect before automatically expiring." + "event": { + "type": "union", + "refs": [ + "#modEventTakedown", + "#modEventReverseTakedown", + "#modEventComment", + "#modEventReport", + "#modEventLabel", + "#modEventAcknowledge", + "#modEventEscalate", + "#modEventMute", + "#modEventEmail" + ] }, "subject": { "type": "union", "refs": ["#repoRef", "com.atproto.repo.strongRef"] }, "subjectBlobCids": { "type": "array", "items": { "type": "string" } }, - "createLabelVals": { "type": "array", "items": { "type": "string" } }, - "negateLabelVals": { "type": "array", "items": { "type": "string" } }, - "reason": { "type": "string" }, "createdBy": { "type": "string", "format": "did" }, "createdAt": { "type": "string", "format": "datetime" }, - "reversal": { "type": "ref", "ref": "#actionReversal" }, - "resolvedReportIds": { "type": "array", "items": { "type": "integer" } } + "creatorHandle": { "type": "string" }, + "subjectHandle": { "type": "string" } } }, - "actionViewDetail": { + "modEventViewDetail": { "type": "object", "required": [ "id", - "action", + "event", "subject", "subjectBlobs", - "reason", "createdBy", - "createdAt", - "resolvedReports" + "createdAt" ], "properties": { "id": { "type": "integer" }, - "action": { "type": "ref", "ref": "#actionType" }, - "durationInHours": { - "type": "integer", - "description": "Indicates how long this action is meant to be in effect before automatically expiring." + "event": { + "type": "union", + "refs": [ + "#modEventTakedown", + "#modEventReverseTakedown", + "#modEventComment", + "#modEventReport", + "#modEventLabel", + "#modEventAcknowledge", + "#modEventEscalate", + "#modEventMute" + ] }, "subject": { "type": "union", @@ -75,59 +85,10 @@ "type": "array", "items": { "type": "ref", "ref": "#blobView" } }, - "createLabelVals": { "type": "array", "items": { "type": "string" } }, - "negateLabelVals": { "type": "array", "items": { "type": "string" } }, - "reason": { "type": "string" }, - "createdBy": { "type": "string", "format": "did" }, - "createdAt": { "type": "string", "format": "datetime" }, - "reversal": { "type": "ref", "ref": "#actionReversal" }, - "resolvedReports": { - "type": "array", - "items": { "type": "ref", "ref": "#reportView" } - } - } - }, - "actionViewCurrent": { - "type": "object", - "required": ["id", "action"], - "properties": { - "id": { "type": "integer" }, - "action": { "type": "ref", "ref": "#actionType" }, - "durationInHours": { - "type": "integer", - "description": "Indicates how long this action is meant to be in effect before automatically expiring." - } - } - }, - "actionReversal": { - "type": "object", - "required": ["reason", "createdBy", "createdAt"], - "properties": { - "reason": { "type": "string" }, "createdBy": { "type": "string", "format": "did" }, "createdAt": { "type": "string", "format": "datetime" } } }, - "actionType": { - "type": "string", - "knownValues": ["#takedown", "#flag", "#acknowledge", "#escalate"] - }, - "takedown": { - "type": "token", - "description": "Moderation action type: Takedown. Indicates that content should not be served by the PDS." - }, - "flag": { - "type": "token", - "description": "Moderation action type: Flag. Indicates that the content was reviewed and considered to violate PDS rules, but may still be served." - }, - "acknowledge": { - "type": "token", - "description": "Moderation action type: Acknowledge. Indicates that the content was reviewed and not considered to violate PDS rules." - }, - "escalate": { - "type": "token", - "description": "Moderation action type: Escalate. Indicates that the content has been flagged for additional review." - }, "reportView": { "type": "object", "required": [ @@ -144,7 +105,7 @@ "type": "ref", "ref": "com.atproto.moderation.defs#reasonType" }, - "reason": { "type": "string" }, + "comment": { "type": "string" }, "subjectRepoHandle": { "type": "string" }, "subject": { "type": "union", @@ -158,6 +119,63 @@ } } }, + "subjectStatusView": { + "type": "object", + "required": ["id", "subject", "createdAt", "updatedAt", "reviewState"], + "properties": { + "id": { "type": "integer" }, + "subject": { + "type": "union", + "refs": ["#repoRef", "com.atproto.repo.strongRef"] + }, + "subjectBlobCids": { + "type": "array", + "items": { "type": "string", "format": "cid" } + }, + "subjectRepoHandle": { "type": "string" }, + "updatedAt": { + "type": "string", + "format": "datetime", + "description": "Timestamp referencing when the last update was made to the moderation status of the subject" + }, + "createdAt": { + "type": "string", + "format": "datetime", + "description": "Timestamp referencing the first moderation status impacting event was emitted on the subject" + }, + "reviewState": { + "type": "ref", + "ref": "#subjectReviewState" + }, + "comment": { + "type": "string", + "description": "Sticky comment on the subject." + }, + "muteUntil": { + "type": "string", + "format": "datetime" + }, + "lastReviewedBy": { + "type": "string", + "format": "did" + }, + "lastReviewedAt": { + "type": "string", + "format": "datetime" + }, + "lastReportedAt": { + "type": "string", + "format": "datetime" + }, + "takendown": { + "type": "boolean" + }, + "suspendUntil": { + "type": "string", + "format": "datetime" + } + } + }, "reportViewDetail": { "type": "object", "required": [ @@ -174,7 +192,7 @@ "type": "ref", "ref": "com.atproto.moderation.defs#reasonType" }, - "reason": { "type": "string" }, + "comment": { "type": "string" }, "subject": { "type": "union", "refs": [ @@ -184,11 +202,18 @@ "#recordViewNotFound" ] }, + "subjectStatus": { + "type": "ref", + "ref": "com.atproto.admin.defs#subjectStatusView" + }, "reportedBy": { "type": "string", "format": "did" }, "createdAt": { "type": "string", "format": "datetime" }, "resolvedByActions": { "type": "array", - "items": { "type": "ref", "ref": "com.atproto.admin.defs#actionView" } + "items": { + "type": "ref", + "ref": "com.atproto.admin.defs#modEventView" + } } } }, @@ -361,21 +386,15 @@ "moderation": { "type": "object", "properties": { - "currentAction": { "type": "ref", "ref": "#actionViewCurrent" } + "subjectStatus": { "type": "ref", "ref": "#subjectStatusView" } } }, "moderationDetail": { "type": "object", - "required": ["actions", "reports"], "properties": { - "currentAction": { "type": "ref", "ref": "#actionViewCurrent" }, - "actions": { - "type": "array", - "items": { "type": "ref", "ref": "#actionView" } - }, - "reports": { - "type": "array", - "items": { "type": "ref", "ref": "#reportView" } + "subjectStatus": { + "type": "ref", + "ref": "#subjectStatusView" } } }, @@ -410,6 +429,136 @@ "height": { "type": "integer" }, "length": { "type": "integer" } } + }, + "subjectReviewState": { + "type": "string", + "knownValues": ["#reviewOpen", "#reviewEscalated", "#reviewClosed"] + }, + "reviewOpen": { + "type": "token", + "description": "Moderator review status of a subject: Open. Indicates that the subject needs to be reviewed by a moderator" + }, + "reviewEscalated": { + "type": "token", + "description": "Moderator review status of a subject: Escalated. Indicates that the subject was escalated for review by a moderator" + }, + "reviewClosed": { + "type": "token", + "description": "Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator" + }, + "modEventTakedown": { + "type": "object", + "description": "Take down a subject permanently or temporarily", + "properties": { + "comment": { + "type": "string" + }, + "durationInHours": { + "type": "integer", + "description": "Indicates how long the takedown should be in effect before automatically expiring." + } + } + }, + "modEventReverseTakedown": { + "type": "object", + "description": "Revert take down action on a subject", + "properties": { + "comment": { + "type": "string", + "description": "Describe reasoning behind the reversal." + } + } + }, + "modEventComment": { + "type": "object", + "description": "Add a comment to a subject", + "required": ["comment"], + "properties": { + "comment": { + "type": "string" + }, + "sticky": { + "type": "boolean", + "description": "Make the comment persistent on the subject" + } + } + }, + "modEventReport": { + "type": "object", + "description": "Report a subject", + "required": ["reportType"], + "properties": { + "comment": { + "type": "string" + }, + "reportType": { + "type": "ref", + "ref": "com.atproto.moderation.defs#reasonType" + } + } + }, + "modEventLabel": { + "type": "object", + "description": "Apply/Negate labels on a subject", + "required": ["createLabelVals", "negateLabelVals"], + "properties": { + "comment": { + "type": "string" + }, + "createLabelVals": { + "type": "array", + "items": { "type": "string" } + }, + "negateLabelVals": { + "type": "array", + "items": { "type": "string" } + } + } + }, + "modEventAcknowledge": { + "type": "object", + "properties": { + "comment": { "type": "string" } + } + }, + "modEventEscalate": { + "type": "object", + "properties": { + "comment": { "type": "string" } + } + }, + "modEventMute": { + "type": "object", + "description": "Mute incoming reports on a subject", + "required": ["durationInHours"], + "properties": { + "comment": { "type": "string" }, + "durationInHours": { + "type": "integer", + "description": "Indicates how long the subject should remain muted." + } + } + }, + "modEventUnmute": { + "type": "object", + "description": "Unmute action on a subject", + "properties": { + "comment": { + "type": "string", + "description": "Describe reasoning behind the reversal." + } + } + }, + "modEventEmail": { + "type": "object", + "description": "Keep a log of outgoing email to a user", + "required": ["subjectLine"], + "properties": { + "subjectLine": { + "type": "string", + "description": "The subject line of the email sent to the user." + } + } } } } diff --git a/lexicons/com/atproto/admin/takeModerationAction.json b/lexicons/com/atproto/admin/emitModerationEvent.json similarity index 50% rename from lexicons/com/atproto/admin/takeModerationAction.json rename to lexicons/com/atproto/admin/emitModerationEvent.json index 70b650aa4b1..f32ad18461c 100644 --- a/lexicons/com/atproto/admin/takeModerationAction.json +++ b/lexicons/com/atproto/admin/emitModerationEvent.json @@ -1,6 +1,6 @@ { "lexicon": 1, - "id": "com.atproto.admin.takeModerationAction", + "id": "com.atproto.admin.emitModerationEvent", "defs": { "main": { "type": "procedure", @@ -9,14 +9,21 @@ "encoding": "application/json", "schema": { "type": "object", - "required": ["action", "subject", "reason", "createdBy"], + "required": ["event", "subject", "createdBy"], "properties": { - "action": { - "type": "string", - "knownValues": [ - "com.atproto.admin.defs#takedown", - "com.atproto.admin.defs#flag", - "com.atproto.admin.defs#acknowledge" + "event": { + "type": "union", + "refs": [ + "com.atproto.admin.defs#modEventTakedown", + "com.atproto.admin.defs#modEventAcknowledge", + "com.atproto.admin.defs#modEventEscalate", + "com.atproto.admin.defs#modEventComment", + "com.atproto.admin.defs#modEventLabel", + "com.atproto.admin.defs#modEventReport", + "com.atproto.admin.defs#modEventMute", + "com.atproto.admin.defs#modEventReverseTakedown", + "com.atproto.admin.defs#modEventUnmute", + "com.atproto.admin.defs#modEventEmail" ] }, "subject": { @@ -30,19 +37,6 @@ "type": "array", "items": { "type": "string", "format": "cid" } }, - "createLabelVals": { - "type": "array", - "items": { "type": "string" } - }, - "negateLabelVals": { - "type": "array", - "items": { "type": "string" } - }, - "reason": { "type": "string" }, - "durationInHours": { - "type": "integer", - "description": "Indicates how long this action is meant to be in effect before automatically expiring." - }, "createdBy": { "type": "string", "format": "did" } } } @@ -51,7 +45,7 @@ "encoding": "application/json", "schema": { "type": "ref", - "ref": "com.atproto.admin.defs#actionView" + "ref": "com.atproto.admin.defs#modEventView" } }, "errors": [{ "name": "SubjectHasAction" }] diff --git a/lexicons/com/atproto/admin/getModerationActions.json b/lexicons/com/atproto/admin/getModerationActions.json deleted file mode 100644 index 370ba7d2f72..00000000000 --- a/lexicons/com/atproto/admin/getModerationActions.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "lexicon": 1, - "id": "com.atproto.admin.getModerationActions", - "defs": { - "main": { - "type": "query", - "description": "Get a list of moderation actions related to a subject.", - "parameters": { - "type": "params", - "properties": { - "subject": { "type": "string" }, - "limit": { - "type": "integer", - "minimum": 1, - "maximum": 100, - "default": 50 - }, - "cursor": { "type": "string" } - } - }, - "output": { - "encoding": "application/json", - "schema": { - "type": "object", - "required": ["actions"], - "properties": { - "cursor": { "type": "string" }, - "actions": { - "type": "array", - "items": { - "type": "ref", - "ref": "com.atproto.admin.defs#actionView" - } - } - } - } - } - } - } -} diff --git a/lexicons/com/atproto/admin/getModerationAction.json b/lexicons/com/atproto/admin/getModerationEvent.json similarity index 67% rename from lexicons/com/atproto/admin/getModerationAction.json rename to lexicons/com/atproto/admin/getModerationEvent.json index eae0736bb3d..71499b94d9a 100644 --- a/lexicons/com/atproto/admin/getModerationAction.json +++ b/lexicons/com/atproto/admin/getModerationEvent.json @@ -1,10 +1,10 @@ { "lexicon": 1, - "id": "com.atproto.admin.getModerationAction", + "id": "com.atproto.admin.getModerationEvent", "defs": { "main": { "type": "query", - "description": "Get details about a moderation action.", + "description": "Get details about a moderation event.", "parameters": { "type": "params", "required": ["id"], @@ -16,7 +16,7 @@ "encoding": "application/json", "schema": { "type": "ref", - "ref": "com.atproto.admin.defs#actionViewDetail" + "ref": "com.atproto.admin.defs#modEventViewDetail" } } } diff --git a/lexicons/com/atproto/admin/getModerationReport.json b/lexicons/com/atproto/admin/getModerationReport.json deleted file mode 100644 index 0e7efc16fde..00000000000 --- a/lexicons/com/atproto/admin/getModerationReport.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "lexicon": 1, - "id": "com.atproto.admin.getModerationReport", - "defs": { - "main": { - "type": "query", - "description": "Get details about a moderation report.", - "parameters": { - "type": "params", - "required": ["id"], - "properties": { - "id": { "type": "integer" } - } - }, - "output": { - "encoding": "application/json", - "schema": { - "type": "ref", - "ref": "com.atproto.admin.defs#reportViewDetail" - } - } - } - } -} diff --git a/lexicons/com/atproto/admin/getModerationReports.json b/lexicons/com/atproto/admin/getModerationReports.json deleted file mode 100644 index 0caeac1a8d6..00000000000 --- a/lexicons/com/atproto/admin/getModerationReports.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "lexicon": 1, - "id": "com.atproto.admin.getModerationReports", - "defs": { - "main": { - "type": "query", - "description": "Get moderation reports related to a subject.", - "parameters": { - "type": "params", - "properties": { - "subject": { "type": "string" }, - "ignoreSubjects": { "type": "array", "items": { "type": "string" } }, - "actionedBy": { - "type": "string", - "format": "did", - "description": "Get all reports that were actioned by a specific moderator." - }, - "reporters": { - "type": "array", - "items": { "type": "string" }, - "description": "Filter reports made by one or more DIDs." - }, - "resolved": { "type": "boolean" }, - "actionType": { - "type": "string", - "knownValues": [ - "com.atproto.admin.defs#takedown", - "com.atproto.admin.defs#flag", - "com.atproto.admin.defs#acknowledge", - "com.atproto.admin.defs#escalate" - ] - }, - "limit": { - "type": "integer", - "minimum": 1, - "maximum": 100, - "default": 50 - }, - "cursor": { "type": "string" }, - "reverse": { - "type": "boolean", - "description": "Reverse the order of the returned records. When true, returns reports in chronological order." - } - } - }, - "output": { - "encoding": "application/json", - "schema": { - "type": "object", - "required": ["reports"], - "properties": { - "cursor": { "type": "string" }, - "reports": { - "type": "array", - "items": { - "type": "ref", - "ref": "com.atproto.admin.defs#reportView" - } - } - } - } - } - } - } -} diff --git a/lexicons/com/atproto/admin/queryModerationEvents.json b/lexicons/com/atproto/admin/queryModerationEvents.json new file mode 100644 index 00000000000..70af1bf8ae5 --- /dev/null +++ b/lexicons/com/atproto/admin/queryModerationEvents.json @@ -0,0 +1,60 @@ +{ + "lexicon": 1, + "id": "com.atproto.admin.queryModerationEvents", + "defs": { + "main": { + "type": "query", + "description": "List moderation events related to a subject.", + "parameters": { + "type": "params", + "properties": { + "types": { + "type": "array", + "items": { "type": "string" }, + "description": "The types of events (fully qualified string in the format of com.atproto.admin#modEvent) to filter by. If not specified, all events are returned." + }, + "createdBy": { + "type": "string", + "format": "did" + }, + "sortDirection": { + "type": "string", + "default": "desc", + "enum": ["asc", "desc"], + "description": "Sort direction for the events. Defaults to descending order of created at timestamp." + }, + "subject": { "type": "string", "format": "uri" }, + "includeAllUserRecords": { + "type": "boolean", + "default": false, + "description": "If true, events on all record types (posts, lists, profile etc.) owned by the did are returned" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "default": 50 + }, + "cursor": { "type": "string" } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["events"], + "properties": { + "cursor": { "type": "string" }, + "events": { + "type": "array", + "items": { + "type": "ref", + "ref": "com.atproto.admin.defs#modEventView" + } + } + } + } + } + } + } +} diff --git a/lexicons/com/atproto/admin/queryModerationStatuses.json b/lexicons/com/atproto/admin/queryModerationStatuses.json new file mode 100644 index 00000000000..98fec5bd642 --- /dev/null +++ b/lexicons/com/atproto/admin/queryModerationStatuses.json @@ -0,0 +1,95 @@ +{ + "lexicon": 1, + "id": "com.atproto.admin.queryModerationStatuses", + "defs": { + "main": { + "type": "query", + "description": "View moderation statuses of subjects (record or repo).", + "parameters": { + "type": "params", + "properties": { + "subject": { "type": "string", "format": "uri" }, + "comment": { + "type": "string", + "description": "Search subjects by keyword from comments" + }, + "reportedAfter": { + "type": "string", + "format": "datetime", + "description": "Search subjects reported after a given timestamp" + }, + "reportedBefore": { + "type": "string", + "format": "datetime", + "description": "Search subjects reported before a given timestamp" + }, + "reviewedAfter": { + "type": "string", + "format": "datetime", + "description": "Search subjects reviewed after a given timestamp" + }, + "reviewedBefore": { + "type": "string", + "format": "datetime", + "description": "Search subjects reviewed before a given timestamp" + }, + "includeMuted": { + "type": "boolean", + "description": "By default, we don't include muted subjects in the results. Set this to true to include them." + }, + "reviewState": { + "type": "string", + "description": "Specify when fetching subjects in a certain state" + }, + "ignoreSubjects": { + "type": "array", + "items": { "type": "string", "format": "uri" } + }, + "lastReviewedBy": { + "type": "string", + "format": "did", + "description": "Get all subject statuses that were reviewed by a specific moderator" + }, + "sortField": { + "type": "string", + "default": "lastReportedAt", + "enum": ["lastReviewedAt", "lastReportedAt"] + }, + "sortDirection": { + "type": "string", + "default": "desc", + "enum": ["asc", "desc"] + }, + "takendown": { + "type": "boolean", + "description": "Get subjects that were taken down" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "default": 50 + }, + "cursor": { "type": "string" } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["subjectStatuses"], + "properties": { + "cursor": { "type": "string" }, + "subjectStatuses": { + "type": "array", + "items": { + "type": "ref", + "ref": "com.atproto.admin.defs#subjectStatusView" + } + } + } + } + } + } + } +} diff --git a/lexicons/com/atproto/admin/resolveModerationReports.json b/lexicons/com/atproto/admin/resolveModerationReports.json deleted file mode 100644 index 0cc5c1df2a2..00000000000 --- a/lexicons/com/atproto/admin/resolveModerationReports.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "lexicon": 1, - "id": "com.atproto.admin.resolveModerationReports", - "defs": { - "main": { - "type": "procedure", - "description": "Resolve moderation reports by an action.", - "input": { - "encoding": "application/json", - "schema": { - "type": "object", - "required": ["actionId", "reportIds", "createdBy"], - "properties": { - "actionId": { "type": "integer" }, - "reportIds": { "type": "array", "items": { "type": "integer" } }, - "createdBy": { "type": "string", "format": "did" } - } - } - }, - "output": { - "encoding": "application/json", - "schema": { - "type": "ref", - "ref": "com.atproto.admin.defs#actionView" - } - } - } - } -} diff --git a/lexicons/com/atproto/admin/reverseModerationAction.json b/lexicons/com/atproto/admin/reverseModerationAction.json deleted file mode 100644 index 9b479dcc8e1..00000000000 --- a/lexicons/com/atproto/admin/reverseModerationAction.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "lexicon": 1, - "id": "com.atproto.admin.reverseModerationAction", - "defs": { - "main": { - "type": "procedure", - "description": "Reverse a moderation action.", - "input": { - "encoding": "application/json", - "schema": { - "type": "object", - "required": ["id", "reason", "createdBy"], - "properties": { - "id": { "type": "integer" }, - "reason": { "type": "string" }, - "createdBy": { "type": "string", "format": "did" } - } - } - }, - "output": { - "encoding": "application/json", - "schema": { - "type": "ref", - "ref": "com.atproto.admin.defs#actionView" - } - } - } - } -} diff --git a/lexicons/com/atproto/admin/sendEmail.json b/lexicons/com/atproto/admin/sendEmail.json index c6af697edd2..8234460d1ba 100644 --- a/lexicons/com/atproto/admin/sendEmail.json +++ b/lexicons/com/atproto/admin/sendEmail.json @@ -9,11 +9,12 @@ "encoding": "application/json", "schema": { "type": "object", - "required": ["recipientDid", "content"], + "required": ["recipientDid", "content", "senderDid"], "properties": { "recipientDid": { "type": "string", "format": "did" }, "content": { "type": "string" }, - "subject": { "type": "string" } + "subject": { "type": "string" }, + "senderDid": { "type": "string", "format": "did" } } } }, diff --git a/packages/api/package.json b/packages/api/package.json index 4267152e641..8b7be93f6dc 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/api", - "version": "0.6.23", + "version": "0.6.24-next.1", "license": "MIT", "description": "Client library for atproto and Bluesky", "keywords": [ diff --git a/packages/api/src/client/index.ts b/packages/api/src/client/index.ts index 519ec284dc4..b295fb88b71 100644 --- a/packages/api/src/client/index.ts +++ b/packages/api/src/client/index.ts @@ -10,21 +10,18 @@ import { CID } from 'multiformats/cid' import * as ComAtprotoAdminDefs from './types/com/atproto/admin/defs' import * as ComAtprotoAdminDisableAccountInvites from './types/com/atproto/admin/disableAccountInvites' import * as ComAtprotoAdminDisableInviteCodes from './types/com/atproto/admin/disableInviteCodes' +import * as ComAtprotoAdminEmitModerationEvent from './types/com/atproto/admin/emitModerationEvent' import * as ComAtprotoAdminEnableAccountInvites from './types/com/atproto/admin/enableAccountInvites' import * as ComAtprotoAdminGetAccountInfo from './types/com/atproto/admin/getAccountInfo' import * as ComAtprotoAdminGetInviteCodes from './types/com/atproto/admin/getInviteCodes' -import * as ComAtprotoAdminGetModerationAction from './types/com/atproto/admin/getModerationAction' -import * as ComAtprotoAdminGetModerationActions from './types/com/atproto/admin/getModerationActions' -import * as ComAtprotoAdminGetModerationReport from './types/com/atproto/admin/getModerationReport' -import * as ComAtprotoAdminGetModerationReports from './types/com/atproto/admin/getModerationReports' +import * as ComAtprotoAdminGetModerationEvent from './types/com/atproto/admin/getModerationEvent' import * as ComAtprotoAdminGetRecord from './types/com/atproto/admin/getRecord' import * as ComAtprotoAdminGetRepo from './types/com/atproto/admin/getRepo' import * as ComAtprotoAdminGetSubjectStatus from './types/com/atproto/admin/getSubjectStatus' -import * as ComAtprotoAdminResolveModerationReports from './types/com/atproto/admin/resolveModerationReports' -import * as ComAtprotoAdminReverseModerationAction from './types/com/atproto/admin/reverseModerationAction' +import * as ComAtprotoAdminQueryModerationEvents from './types/com/atproto/admin/queryModerationEvents' +import * as ComAtprotoAdminQueryModerationStatuses from './types/com/atproto/admin/queryModerationStatuses' import * as ComAtprotoAdminSearchRepos from './types/com/atproto/admin/searchRepos' import * as ComAtprotoAdminSendEmail from './types/com/atproto/admin/sendEmail' -import * as ComAtprotoAdminTakeModerationAction from './types/com/atproto/admin/takeModerationAction' import * as ComAtprotoAdminUpdateAccountEmail from './types/com/atproto/admin/updateAccountEmail' import * as ComAtprotoAdminUpdateAccountHandle from './types/com/atproto/admin/updateAccountHandle' import * as ComAtprotoAdminUpdateSubjectStatus from './types/com/atproto/admin/updateSubjectStatus' @@ -148,21 +145,18 @@ import * as AppBskyUnspeccedSearchPostsSkeleton from './types/app/bsky/unspecced export * as ComAtprotoAdminDefs from './types/com/atproto/admin/defs' export * as ComAtprotoAdminDisableAccountInvites from './types/com/atproto/admin/disableAccountInvites' export * as ComAtprotoAdminDisableInviteCodes from './types/com/atproto/admin/disableInviteCodes' +export * as ComAtprotoAdminEmitModerationEvent from './types/com/atproto/admin/emitModerationEvent' export * as ComAtprotoAdminEnableAccountInvites from './types/com/atproto/admin/enableAccountInvites' export * as ComAtprotoAdminGetAccountInfo from './types/com/atproto/admin/getAccountInfo' export * as ComAtprotoAdminGetInviteCodes from './types/com/atproto/admin/getInviteCodes' -export * as ComAtprotoAdminGetModerationAction from './types/com/atproto/admin/getModerationAction' -export * as ComAtprotoAdminGetModerationActions from './types/com/atproto/admin/getModerationActions' -export * as ComAtprotoAdminGetModerationReport from './types/com/atproto/admin/getModerationReport' -export * as ComAtprotoAdminGetModerationReports from './types/com/atproto/admin/getModerationReports' +export * as ComAtprotoAdminGetModerationEvent from './types/com/atproto/admin/getModerationEvent' export * as ComAtprotoAdminGetRecord from './types/com/atproto/admin/getRecord' export * as ComAtprotoAdminGetRepo from './types/com/atproto/admin/getRepo' export * as ComAtprotoAdminGetSubjectStatus from './types/com/atproto/admin/getSubjectStatus' -export * as ComAtprotoAdminResolveModerationReports from './types/com/atproto/admin/resolveModerationReports' -export * as ComAtprotoAdminReverseModerationAction from './types/com/atproto/admin/reverseModerationAction' +export * as ComAtprotoAdminQueryModerationEvents from './types/com/atproto/admin/queryModerationEvents' +export * as ComAtprotoAdminQueryModerationStatuses from './types/com/atproto/admin/queryModerationStatuses' export * as ComAtprotoAdminSearchRepos from './types/com/atproto/admin/searchRepos' export * as ComAtprotoAdminSendEmail from './types/com/atproto/admin/sendEmail' -export * as ComAtprotoAdminTakeModerationAction from './types/com/atproto/admin/takeModerationAction' export * as ComAtprotoAdminUpdateAccountEmail from './types/com/atproto/admin/updateAccountEmail' export * as ComAtprotoAdminUpdateAccountHandle from './types/com/atproto/admin/updateAccountHandle' export * as ComAtprotoAdminUpdateSubjectStatus from './types/com/atproto/admin/updateSubjectStatus' @@ -284,10 +278,9 @@ export * as AppBskyUnspeccedSearchActorsSkeleton from './types/app/bsky/unspecce export * as AppBskyUnspeccedSearchPostsSkeleton from './types/app/bsky/unspecced/searchPostsSkeleton' export const COM_ATPROTO_ADMIN = { - DefsTakedown: 'com.atproto.admin.defs#takedown', - DefsFlag: 'com.atproto.admin.defs#flag', - DefsAcknowledge: 'com.atproto.admin.defs#acknowledge', - DefsEscalate: 'com.atproto.admin.defs#escalate', + DefsReviewOpen: 'com.atproto.admin.defs#reviewOpen', + DefsReviewEscalated: 'com.atproto.admin.defs#reviewEscalated', + DefsReviewClosed: 'com.atproto.admin.defs#reviewClosed', } export const COM_ATPROTO_MODERATION = { DefsReasonSpam: 'com.atproto.moderation.defs#reasonSpam', @@ -395,6 +388,17 @@ export class AdminNS { }) } + emitModerationEvent( + data?: ComAtprotoAdminEmitModerationEvent.InputSchema, + opts?: ComAtprotoAdminEmitModerationEvent.CallOptions, + ): Promise { + return this._service.xrpc + .call('com.atproto.admin.emitModerationEvent', opts?.qp, data, opts) + .catch((e) => { + throw ComAtprotoAdminEmitModerationEvent.toKnownErr(e) + }) + } + enableAccountInvites( data?: ComAtprotoAdminEnableAccountInvites.InputSchema, opts?: ComAtprotoAdminEnableAccountInvites.CallOptions, @@ -428,47 +432,14 @@ export class AdminNS { }) } - getModerationAction( - params?: ComAtprotoAdminGetModerationAction.QueryParams, - opts?: ComAtprotoAdminGetModerationAction.CallOptions, - ): Promise { - return this._service.xrpc - .call('com.atproto.admin.getModerationAction', params, undefined, opts) - .catch((e) => { - throw ComAtprotoAdminGetModerationAction.toKnownErr(e) - }) - } - - getModerationActions( - params?: ComAtprotoAdminGetModerationActions.QueryParams, - opts?: ComAtprotoAdminGetModerationActions.CallOptions, - ): Promise { - return this._service.xrpc - .call('com.atproto.admin.getModerationActions', params, undefined, opts) - .catch((e) => { - throw ComAtprotoAdminGetModerationActions.toKnownErr(e) - }) - } - - getModerationReport( - params?: ComAtprotoAdminGetModerationReport.QueryParams, - opts?: ComAtprotoAdminGetModerationReport.CallOptions, - ): Promise { + getModerationEvent( + params?: ComAtprotoAdminGetModerationEvent.QueryParams, + opts?: ComAtprotoAdminGetModerationEvent.CallOptions, + ): Promise { return this._service.xrpc - .call('com.atproto.admin.getModerationReport', params, undefined, opts) + .call('com.atproto.admin.getModerationEvent', params, undefined, opts) .catch((e) => { - throw ComAtprotoAdminGetModerationReport.toKnownErr(e) - }) - } - - getModerationReports( - params?: ComAtprotoAdminGetModerationReports.QueryParams, - opts?: ComAtprotoAdminGetModerationReports.CallOptions, - ): Promise { - return this._service.xrpc - .call('com.atproto.admin.getModerationReports', params, undefined, opts) - .catch((e) => { - throw ComAtprotoAdminGetModerationReports.toKnownErr(e) + throw ComAtprotoAdminGetModerationEvent.toKnownErr(e) }) } @@ -505,25 +476,30 @@ export class AdminNS { }) } - resolveModerationReports( - data?: ComAtprotoAdminResolveModerationReports.InputSchema, - opts?: ComAtprotoAdminResolveModerationReports.CallOptions, - ): Promise { + queryModerationEvents( + params?: ComAtprotoAdminQueryModerationEvents.QueryParams, + opts?: ComAtprotoAdminQueryModerationEvents.CallOptions, + ): Promise { return this._service.xrpc - .call('com.atproto.admin.resolveModerationReports', opts?.qp, data, opts) + .call('com.atproto.admin.queryModerationEvents', params, undefined, opts) .catch((e) => { - throw ComAtprotoAdminResolveModerationReports.toKnownErr(e) + throw ComAtprotoAdminQueryModerationEvents.toKnownErr(e) }) } - reverseModerationAction( - data?: ComAtprotoAdminReverseModerationAction.InputSchema, - opts?: ComAtprotoAdminReverseModerationAction.CallOptions, - ): Promise { + queryModerationStatuses( + params?: ComAtprotoAdminQueryModerationStatuses.QueryParams, + opts?: ComAtprotoAdminQueryModerationStatuses.CallOptions, + ): Promise { return this._service.xrpc - .call('com.atproto.admin.reverseModerationAction', opts?.qp, data, opts) + .call( + 'com.atproto.admin.queryModerationStatuses', + params, + undefined, + opts, + ) .catch((e) => { - throw ComAtprotoAdminReverseModerationAction.toKnownErr(e) + throw ComAtprotoAdminQueryModerationStatuses.toKnownErr(e) }) } @@ -549,17 +525,6 @@ export class AdminNS { }) } - takeModerationAction( - data?: ComAtprotoAdminTakeModerationAction.InputSchema, - opts?: ComAtprotoAdminTakeModerationAction.CallOptions, - ): Promise { - return this._service.xrpc - .call('com.atproto.admin.takeModerationAction', opts?.qp, data, opts) - .catch((e) => { - throw ComAtprotoAdminTakeModerationAction.toKnownErr(e) - }) - } - updateAccountEmail( data?: ComAtprotoAdminUpdateAccountEmail.InputSchema, opts?: ComAtprotoAdminUpdateAccountEmail.CallOptions, diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index cc3f09f0c4e..25a60f90054 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -20,30 +20,33 @@ export const schemaDict = { }, }, }, - actionView: { + modEventView: { type: 'object', required: [ 'id', - 'action', + 'event', 'subject', 'subjectBlobCids', - 'reason', 'createdBy', 'createdAt', - 'resolvedReportIds', ], properties: { id: { type: 'integer', }, - action: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionType', - }, - durationInHours: { - type: 'integer', - description: - 'Indicates how long this action is meant to be in effect before automatically expiring.', + event: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#modEventTakedown', + 'lex:com.atproto.admin.defs#modEventReverseTakedown', + 'lex:com.atproto.admin.defs#modEventComment', + 'lex:com.atproto.admin.defs#modEventReport', + 'lex:com.atproto.admin.defs#modEventLabel', + 'lex:com.atproto.admin.defs#modEventAcknowledge', + 'lex:com.atproto.admin.defs#modEventEscalate', + 'lex:com.atproto.admin.defs#modEventMute', + 'lex:com.atproto.admin.defs#modEventEmail', + ], }, subject: { type: 'union', @@ -58,21 +61,6 @@ export const schemaDict = { type: 'string', }, }, - createLabelVals: { - type: 'array', - items: { - type: 'string', - }, - }, - negateLabelVals: { - type: 'array', - items: { - type: 'string', - }, - }, - reason: { - type: 'string', - }, createdBy: { type: 'string', format: 'did', @@ -81,42 +69,40 @@ export const schemaDict = { type: 'string', format: 'datetime', }, - reversal: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionReversal', + creatorHandle: { + type: 'string', }, - resolvedReportIds: { - type: 'array', - items: { - type: 'integer', - }, + subjectHandle: { + type: 'string', }, }, }, - actionViewDetail: { + modEventViewDetail: { type: 'object', required: [ 'id', - 'action', + 'event', 'subject', 'subjectBlobs', - 'reason', 'createdBy', 'createdAt', - 'resolvedReports', ], properties: { id: { type: 'integer', }, - action: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionType', - }, - durationInHours: { - type: 'integer', - description: - 'Indicates how long this action is meant to be in effect before automatically expiring.', + event: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#modEventTakedown', + 'lex:com.atproto.admin.defs#modEventReverseTakedown', + 'lex:com.atproto.admin.defs#modEventComment', + 'lex:com.atproto.admin.defs#modEventReport', + 'lex:com.atproto.admin.defs#modEventLabel', + 'lex:com.atproto.admin.defs#modEventAcknowledge', + 'lex:com.atproto.admin.defs#modEventEscalate', + 'lex:com.atproto.admin.defs#modEventMute', + ], }, subject: { type: 'union', @@ -134,67 +120,6 @@ export const schemaDict = { ref: 'lex:com.atproto.admin.defs#blobView', }, }, - createLabelVals: { - type: 'array', - items: { - type: 'string', - }, - }, - negateLabelVals: { - type: 'array', - items: { - type: 'string', - }, - }, - reason: { - type: 'string', - }, - createdBy: { - type: 'string', - format: 'did', - }, - createdAt: { - type: 'string', - format: 'datetime', - }, - reversal: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionReversal', - }, - resolvedReports: { - type: 'array', - items: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#reportView', - }, - }, - }, - }, - actionViewCurrent: { - type: 'object', - required: ['id', 'action'], - properties: { - id: { - type: 'integer', - }, - action: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionType', - }, - durationInHours: { - type: 'integer', - description: - 'Indicates how long this action is meant to be in effect before automatically expiring.', - }, - }, - }, - actionReversal: { - type: 'object', - required: ['reason', 'createdBy', 'createdAt'], - properties: { - reason: { - type: 'string', - }, createdBy: { type: 'string', format: 'did', @@ -205,35 +130,6 @@ export const schemaDict = { }, }, }, - actionType: { - type: 'string', - knownValues: [ - 'lex:com.atproto.admin.defs#takedown', - 'lex:com.atproto.admin.defs#flag', - 'lex:com.atproto.admin.defs#acknowledge', - 'lex:com.atproto.admin.defs#escalate', - ], - }, - takedown: { - type: 'token', - description: - 'Moderation action type: Takedown. Indicates that content should not be served by the PDS.', - }, - flag: { - type: 'token', - description: - 'Moderation action type: Flag. Indicates that the content was reviewed and considered to violate PDS rules, but may still be served.', - }, - acknowledge: { - type: 'token', - description: - 'Moderation action type: Acknowledge. Indicates that the content was reviewed and not considered to violate PDS rules.', - }, - escalate: { - type: 'token', - description: - 'Moderation action type: Escalate. Indicates that the content has been flagged for additional review.', - }, reportView: { type: 'object', required: [ @@ -252,7 +148,7 @@ export const schemaDict = { type: 'ref', ref: 'lex:com.atproto.moderation.defs#reasonType', }, - reason: { + comment: { type: 'string', }, subjectRepoHandle: { @@ -281,6 +177,75 @@ export const schemaDict = { }, }, }, + subjectStatusView: { + type: 'object', + required: ['id', 'subject', 'createdAt', 'updatedAt', 'reviewState'], + properties: { + id: { + type: 'integer', + }, + subject: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#repoRef', + 'lex:com.atproto.repo.strongRef', + ], + }, + subjectBlobCids: { + type: 'array', + items: { + type: 'string', + format: 'cid', + }, + }, + subjectRepoHandle: { + type: 'string', + }, + updatedAt: { + type: 'string', + format: 'datetime', + description: + 'Timestamp referencing when the last update was made to the moderation status of the subject', + }, + createdAt: { + type: 'string', + format: 'datetime', + description: + 'Timestamp referencing the first moderation status impacting event was emitted on the subject', + }, + reviewState: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#subjectReviewState', + }, + comment: { + type: 'string', + description: 'Sticky comment on the subject.', + }, + muteUntil: { + type: 'string', + format: 'datetime', + }, + lastReviewedBy: { + type: 'string', + format: 'did', + }, + lastReviewedAt: { + type: 'string', + format: 'datetime', + }, + lastReportedAt: { + type: 'string', + format: 'datetime', + }, + takendown: { + type: 'boolean', + }, + suspendUntil: { + type: 'string', + format: 'datetime', + }, + }, + }, reportViewDetail: { type: 'object', required: [ @@ -299,7 +264,7 @@ export const schemaDict = { type: 'ref', ref: 'lex:com.atproto.moderation.defs#reasonType', }, - reason: { + comment: { type: 'string', }, subject: { @@ -311,6 +276,10 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#recordViewNotFound', ], }, + subjectStatus: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#subjectStatusView', + }, reportedBy: { type: 'string', format: 'did', @@ -323,7 +292,7 @@ export const schemaDict = { type: 'array', items: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionView', + ref: 'lex:com.atproto.admin.defs#modEventView', }, }, }, @@ -628,33 +597,18 @@ export const schemaDict = { moderation: { type: 'object', properties: { - currentAction: { + subjectStatus: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionViewCurrent', + ref: 'lex:com.atproto.admin.defs#subjectStatusView', }, }, }, moderationDetail: { type: 'object', - required: ['actions', 'reports'], properties: { - currentAction: { + subjectStatus: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionViewCurrent', - }, - actions: { - type: 'array', - items: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionView', - }, - }, - reports: { - type: 'array', - items: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#reportView', - }, + ref: 'lex:com.atproto.admin.defs#subjectStatusView', }, }, }, @@ -716,74 +670,164 @@ export const schemaDict = { }, }, }, - }, - }, - ComAtprotoAdminDisableAccountInvites: { - lexicon: 1, - id: 'com.atproto.admin.disableAccountInvites', - defs: { - main: { - type: 'procedure', + subjectReviewState: { + type: 'string', + knownValues: [ + 'lex:com.atproto.admin.defs#reviewOpen', + 'lex:com.atproto.admin.defs#reviewEscalated', + 'lex:com.atproto.admin.defs#reviewClosed', + ], + }, + reviewOpen: { + type: 'token', description: - 'Disable an account from receiving new invite codes, but does not invalidate existing codes.', - input: { - encoding: 'application/json', - schema: { - type: 'object', - required: ['account'], - properties: { - account: { - type: 'string', - format: 'did', - }, - note: { - type: 'string', - description: 'Optional reason for disabled invites.', - }, - }, + 'Moderator review status of a subject: Open. Indicates that the subject needs to be reviewed by a moderator', + }, + reviewEscalated: { + type: 'token', + description: + 'Moderator review status of a subject: Escalated. Indicates that the subject was escalated for review by a moderator', + }, + reviewClosed: { + type: 'token', + description: + 'Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator', + }, + modEventTakedown: { + type: 'object', + description: 'Take down a subject permanently or temporarily', + properties: { + comment: { + type: 'string', + }, + durationInHours: { + type: 'integer', + description: + 'Indicates how long the takedown should be in effect before automatically expiring.', }, }, }, - }, - }, - ComAtprotoAdminDisableInviteCodes: { - lexicon: 1, - id: 'com.atproto.admin.disableInviteCodes', - defs: { - main: { - type: 'procedure', - description: - 'Disable some set of codes and/or all codes associated with a set of users.', - input: { - encoding: 'application/json', - schema: { - type: 'object', - properties: { - codes: { - type: 'array', - items: { - type: 'string', - }, - }, - accounts: { - type: 'array', - items: { - type: 'string', - }, - }, - }, + modEventReverseTakedown: { + type: 'object', + description: 'Revert take down action on a subject', + properties: { + comment: { + type: 'string', + description: 'Describe reasoning behind the reversal.', }, }, }, - }, - }, - ComAtprotoAdminEnableAccountInvites: { - lexicon: 1, - id: 'com.atproto.admin.enableAccountInvites', + modEventComment: { + type: 'object', + description: 'Add a comment to a subject', + required: ['comment'], + properties: { + comment: { + type: 'string', + }, + sticky: { + type: 'boolean', + description: 'Make the comment persistent on the subject', + }, + }, + }, + modEventReport: { + type: 'object', + description: 'Report a subject', + required: ['reportType'], + properties: { + comment: { + type: 'string', + }, + reportType: { + type: 'ref', + ref: 'lex:com.atproto.moderation.defs#reasonType', + }, + }, + }, + modEventLabel: { + type: 'object', + description: 'Apply/Negate labels on a subject', + required: ['createLabelVals', 'negateLabelVals'], + properties: { + comment: { + type: 'string', + }, + createLabelVals: { + type: 'array', + items: { + type: 'string', + }, + }, + negateLabelVals: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + modEventAcknowledge: { + type: 'object', + properties: { + comment: { + type: 'string', + }, + }, + }, + modEventEscalate: { + type: 'object', + properties: { + comment: { + type: 'string', + }, + }, + }, + modEventMute: { + type: 'object', + description: 'Mute incoming reports on a subject', + required: ['durationInHours'], + properties: { + comment: { + type: 'string', + }, + durationInHours: { + type: 'integer', + description: 'Indicates how long the subject should remain muted.', + }, + }, + }, + modEventUnmute: { + type: 'object', + description: 'Unmute action on a subject', + properties: { + comment: { + type: 'string', + description: 'Describe reasoning behind the reversal.', + }, + }, + }, + modEventEmail: { + type: 'object', + description: 'Keep a log of outgoing email to a user', + required: ['subjectLine'], + properties: { + subjectLine: { + type: 'string', + description: 'The subject line of the email sent to the user.', + }, + }, + }, + }, + }, + ComAtprotoAdminDisableAccountInvites: { + lexicon: 1, + id: 'com.atproto.admin.disableAccountInvites', defs: { main: { type: 'procedure', - description: "Re-enable an account's ability to receive invite codes.", + description: + 'Disable an account from receiving new invite codes, but does not invalidate existing codes.', input: { encoding: 'application/json', schema: { @@ -796,7 +840,7 @@ export const schemaDict = { }, note: { type: 'string', - description: 'Optional reason for enabled invites.', + description: 'Optional reason for disabled invites.', }, }, }, @@ -804,20 +848,83 @@ export const schemaDict = { }, }, }, - ComAtprotoAdminGetAccountInfo: { + ComAtprotoAdminDisableInviteCodes: { lexicon: 1, - id: 'com.atproto.admin.getAccountInfo', + id: 'com.atproto.admin.disableInviteCodes', defs: { main: { - type: 'query', - description: 'Get details about an account.', - parameters: { - type: 'params', - required: ['did'], - properties: { - did: { - type: 'string', - format: 'did', + type: 'procedure', + description: + 'Disable some set of codes and/or all codes associated with a set of users.', + input: { + encoding: 'application/json', + schema: { + type: 'object', + properties: { + codes: { + type: 'array', + items: { + type: 'string', + }, + }, + accounts: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + }, + }, + }, + }, + ComAtprotoAdminEmitModerationEvent: { + lexicon: 1, + id: 'com.atproto.admin.emitModerationEvent', + defs: { + main: { + type: 'procedure', + description: 'Take a moderation action on an actor.', + input: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['event', 'subject', 'createdBy'], + properties: { + event: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#modEventTakedown', + 'lex:com.atproto.admin.defs#modEventAcknowledge', + 'lex:com.atproto.admin.defs#modEventEscalate', + 'lex:com.atproto.admin.defs#modEventComment', + 'lex:com.atproto.admin.defs#modEventLabel', + 'lex:com.atproto.admin.defs#modEventReport', + 'lex:com.atproto.admin.defs#modEventMute', + 'lex:com.atproto.admin.defs#modEventReverseTakedown', + 'lex:com.atproto.admin.defs#modEventUnmute', + 'lex:com.atproto.admin.defs#modEventEmail', + ], + }, + subject: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#repoRef', + 'lex:com.atproto.repo.strongRef', + ], + }, + subjectBlobCids: { + type: 'array', + items: { + type: 'string', + format: 'cid', + }, + }, + createdBy: { + type: 'string', + format: 'did', + }, }, }, }, @@ -825,53 +932,37 @@ export const schemaDict = { encoding: 'application/json', schema: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#accountView', + ref: 'lex:com.atproto.admin.defs#modEventView', }, }, + errors: [ + { + name: 'SubjectHasAction', + }, + ], }, }, }, - ComAtprotoAdminGetInviteCodes: { + ComAtprotoAdminEnableAccountInvites: { lexicon: 1, - id: 'com.atproto.admin.getInviteCodes', + id: 'com.atproto.admin.enableAccountInvites', defs: { main: { - type: 'query', - description: 'Get an admin view of invite codes.', - parameters: { - type: 'params', - properties: { - sort: { - type: 'string', - knownValues: ['recent', 'usage'], - default: 'recent', - }, - limit: { - type: 'integer', - minimum: 1, - maximum: 500, - default: 100, - }, - cursor: { - type: 'string', - }, - }, - }, - output: { + type: 'procedure', + description: "Re-enable an account's ability to receive invite codes.", + input: { encoding: 'application/json', schema: { type: 'object', - required: ['codes'], + required: ['account'], properties: { - cursor: { + account: { type: 'string', + format: 'did', }, - codes: { - type: 'array', - items: { - type: 'ref', - ref: 'lex:com.atproto.server.defs#inviteCode', - }, + note: { + type: 'string', + description: 'Optional reason for enabled invites.', }, }, }, @@ -879,19 +970,20 @@ export const schemaDict = { }, }, }, - ComAtprotoAdminGetModerationAction: { + ComAtprotoAdminGetAccountInfo: { lexicon: 1, - id: 'com.atproto.admin.getModerationAction', + id: 'com.atproto.admin.getAccountInfo', defs: { main: { type: 'query', - description: 'Get details about a moderation action.', + description: 'Get details about an account.', parameters: { type: 'params', - required: ['id'], + required: ['did'], properties: { - id: { - type: 'integer', + did: { + type: 'string', + format: 'did', }, }, }, @@ -899,30 +991,32 @@ export const schemaDict = { encoding: 'application/json', schema: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionViewDetail', + ref: 'lex:com.atproto.admin.defs#accountView', }, }, }, }, }, - ComAtprotoAdminGetModerationActions: { + ComAtprotoAdminGetInviteCodes: { lexicon: 1, - id: 'com.atproto.admin.getModerationActions', + id: 'com.atproto.admin.getInviteCodes', defs: { main: { type: 'query', - description: 'Get a list of moderation actions related to a subject.', + description: 'Get an admin view of invite codes.', parameters: { type: 'params', properties: { - subject: { + sort: { type: 'string', + knownValues: ['recent', 'usage'], + default: 'recent', }, limit: { type: 'integer', minimum: 1, - maximum: 100, - default: 50, + maximum: 500, + default: 100, }, cursor: { type: 'string', @@ -933,16 +1027,16 @@ export const schemaDict = { encoding: 'application/json', schema: { type: 'object', - required: ['actions'], + required: ['codes'], properties: { cursor: { type: 'string', }, - actions: { + codes: { type: 'array', items: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionView', + ref: 'lex:com.atproto.server.defs#inviteCode', }, }, }, @@ -951,13 +1045,13 @@ export const schemaDict = { }, }, }, - ComAtprotoAdminGetModerationReport: { + ComAtprotoAdminGetModerationEvent: { lexicon: 1, - id: 'com.atproto.admin.getModerationReport', + id: 'com.atproto.admin.getModerationEvent', defs: { main: { type: 'query', - description: 'Get details about a moderation report.', + description: 'Get details about a moderation event.', parameters: { type: 'params', required: ['id'], @@ -971,89 +1065,7 @@ export const schemaDict = { encoding: 'application/json', schema: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#reportViewDetail', - }, - }, - }, - }, - }, - ComAtprotoAdminGetModerationReports: { - lexicon: 1, - id: 'com.atproto.admin.getModerationReports', - defs: { - main: { - type: 'query', - description: 'Get moderation reports related to a subject.', - parameters: { - type: 'params', - properties: { - subject: { - type: 'string', - }, - ignoreSubjects: { - type: 'array', - items: { - type: 'string', - }, - }, - actionedBy: { - type: 'string', - format: 'did', - description: - 'Get all reports that were actioned by a specific moderator.', - }, - reporters: { - type: 'array', - items: { - type: 'string', - }, - description: 'Filter reports made by one or more DIDs.', - }, - resolved: { - type: 'boolean', - }, - actionType: { - type: 'string', - knownValues: [ - 'com.atproto.admin.defs#takedown', - 'com.atproto.admin.defs#flag', - 'com.atproto.admin.defs#acknowledge', - 'com.atproto.admin.defs#escalate', - ], - }, - limit: { - type: 'integer', - minimum: 1, - maximum: 100, - default: 50, - }, - cursor: { - type: 'string', - }, - reverse: { - type: 'boolean', - description: - 'Reverse the order of the returned records. When true, returns reports in chronological order.', - }, - }, - }, - output: { - encoding: 'application/json', - schema: { - type: 'object', - required: ['reports'], - properties: { - cursor: { - type: 'string', - }, - reports: { - type: 'array', - items: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#reportView', - }, - }, - }, + ref: 'lex:com.atproto.admin.defs#modEventViewDetail', }, }, }, @@ -1176,76 +1188,180 @@ export const schemaDict = { }, }, }, - ComAtprotoAdminResolveModerationReports: { + ComAtprotoAdminQueryModerationEvents: { lexicon: 1, - id: 'com.atproto.admin.resolveModerationReports', + id: 'com.atproto.admin.queryModerationEvents', defs: { main: { - type: 'procedure', - description: 'Resolve moderation reports by an action.', - input: { + type: 'query', + description: 'List moderation events related to a subject.', + parameters: { + type: 'params', + properties: { + types: { + type: 'array', + items: { + type: 'string', + }, + description: + 'The types of events (fully qualified string in the format of com.atproto.admin#modEvent) to filter by. If not specified, all events are returned.', + }, + createdBy: { + type: 'string', + format: 'did', + }, + sortDirection: { + type: 'string', + default: 'desc', + enum: ['asc', 'desc'], + description: + 'Sort direction for the events. Defaults to descending order of created at timestamp.', + }, + subject: { + type: 'string', + format: 'uri', + }, + includeAllUserRecords: { + type: 'boolean', + default: false, + description: + 'If true, events on all record types (posts, lists, profile etc.) owned by the did are returned', + }, + limit: { + type: 'integer', + minimum: 1, + maximum: 100, + default: 50, + }, + cursor: { + type: 'string', + }, + }, + }, + output: { encoding: 'application/json', schema: { type: 'object', - required: ['actionId', 'reportIds', 'createdBy'], + required: ['events'], properties: { - actionId: { - type: 'integer', + cursor: { + type: 'string', }, - reportIds: { + events: { type: 'array', items: { - type: 'integer', + type: 'ref', + ref: 'lex:com.atproto.admin.defs#modEventView', }, }, - createdBy: { - type: 'string', - format: 'did', - }, }, }, }, - output: { - encoding: 'application/json', - schema: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionView', - }, - }, }, }, }, - ComAtprotoAdminReverseModerationAction: { + ComAtprotoAdminQueryModerationStatuses: { lexicon: 1, - id: 'com.atproto.admin.reverseModerationAction', + id: 'com.atproto.admin.queryModerationStatuses', defs: { main: { - type: 'procedure', - description: 'Reverse a moderation action.', - input: { - encoding: 'application/json', - schema: { - type: 'object', - required: ['id', 'reason', 'createdBy'], - properties: { - id: { - type: 'integer', - }, - reason: { - type: 'string', - }, - createdBy: { + type: 'query', + description: 'View moderation statuses of subjects (record or repo).', + parameters: { + type: 'params', + properties: { + subject: { + type: 'string', + format: 'uri', + }, + comment: { + type: 'string', + description: 'Search subjects by keyword from comments', + }, + reportedAfter: { + type: 'string', + format: 'datetime', + description: 'Search subjects reported after a given timestamp', + }, + reportedBefore: { + type: 'string', + format: 'datetime', + description: 'Search subjects reported before a given timestamp', + }, + reviewedAfter: { + type: 'string', + format: 'datetime', + description: 'Search subjects reviewed after a given timestamp', + }, + reviewedBefore: { + type: 'string', + format: 'datetime', + description: 'Search subjects reviewed before a given timestamp', + }, + includeMuted: { + type: 'boolean', + description: + "By default, we don't include muted subjects in the results. Set this to true to include them.", + }, + reviewState: { + type: 'string', + description: 'Specify when fetching subjects in a certain state', + }, + ignoreSubjects: { + type: 'array', + items: { type: 'string', - format: 'did', + format: 'uri', }, }, + lastReviewedBy: { + type: 'string', + format: 'did', + description: + 'Get all subject statuses that were reviewed by a specific moderator', + }, + sortField: { + type: 'string', + default: 'lastReportedAt', + enum: ['lastReviewedAt', 'lastReportedAt'], + }, + sortDirection: { + type: 'string', + default: 'desc', + enum: ['asc', 'desc'], + }, + takendown: { + type: 'boolean', + description: 'Get subjects that were taken down', + }, + limit: { + type: 'integer', + minimum: 1, + maximum: 100, + default: 50, + }, + cursor: { + type: 'string', + }, }, }, output: { encoding: 'application/json', schema: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionView', + type: 'object', + required: ['subjectStatuses'], + properties: { + cursor: { + type: 'string', + }, + subjectStatuses: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#subjectStatusView', + }, + }, + }, }, }, }, @@ -1312,7 +1428,7 @@ export const schemaDict = { encoding: 'application/json', schema: { type: 'object', - required: ['recipientDid', 'content'], + required: ['recipientDid', 'content', 'senderDid'], properties: { recipientDid: { type: 'string', @@ -1324,6 +1440,10 @@ export const schemaDict = { subject: { type: 'string', }, + senderDid: { + type: 'string', + format: 'did', + }, }, }, }, @@ -1342,83 +1462,6 @@ export const schemaDict = { }, }, }, - ComAtprotoAdminTakeModerationAction: { - lexicon: 1, - id: 'com.atproto.admin.takeModerationAction', - defs: { - main: { - type: 'procedure', - description: 'Take a moderation action on an actor.', - input: { - encoding: 'application/json', - schema: { - type: 'object', - required: ['action', 'subject', 'reason', 'createdBy'], - properties: { - action: { - type: 'string', - knownValues: [ - 'com.atproto.admin.defs#takedown', - 'com.atproto.admin.defs#flag', - 'com.atproto.admin.defs#acknowledge', - ], - }, - subject: { - type: 'union', - refs: [ - 'lex:com.atproto.admin.defs#repoRef', - 'lex:com.atproto.repo.strongRef', - ], - }, - subjectBlobCids: { - type: 'array', - items: { - type: 'string', - format: 'cid', - }, - }, - createLabelVals: { - type: 'array', - items: { - type: 'string', - }, - }, - negateLabelVals: { - type: 'array', - items: { - type: 'string', - }, - }, - reason: { - type: 'string', - }, - durationInHours: { - type: 'integer', - description: - 'Indicates how long this action is meant to be in effect before automatically expiring.', - }, - createdBy: { - type: 'string', - format: 'did', - }, - }, - }, - }, - output: { - encoding: 'application/json', - schema: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionView', - }, - }, - errors: [ - { - name: 'SubjectHasAction', - }, - ], - }, - }, - }, ComAtprotoAdminUpdateAccountEmail: { lexicon: 1, id: 'com.atproto.admin.updateAccountEmail', @@ -7627,23 +7670,20 @@ export const ids = { ComAtprotoAdminDisableAccountInvites: 'com.atproto.admin.disableAccountInvites', ComAtprotoAdminDisableInviteCodes: 'com.atproto.admin.disableInviteCodes', + ComAtprotoAdminEmitModerationEvent: 'com.atproto.admin.emitModerationEvent', ComAtprotoAdminEnableAccountInvites: 'com.atproto.admin.enableAccountInvites', ComAtprotoAdminGetAccountInfo: 'com.atproto.admin.getAccountInfo', ComAtprotoAdminGetInviteCodes: 'com.atproto.admin.getInviteCodes', - ComAtprotoAdminGetModerationAction: 'com.atproto.admin.getModerationAction', - ComAtprotoAdminGetModerationActions: 'com.atproto.admin.getModerationActions', - ComAtprotoAdminGetModerationReport: 'com.atproto.admin.getModerationReport', - ComAtprotoAdminGetModerationReports: 'com.atproto.admin.getModerationReports', + ComAtprotoAdminGetModerationEvent: 'com.atproto.admin.getModerationEvent', ComAtprotoAdminGetRecord: 'com.atproto.admin.getRecord', ComAtprotoAdminGetRepo: 'com.atproto.admin.getRepo', ComAtprotoAdminGetSubjectStatus: 'com.atproto.admin.getSubjectStatus', - ComAtprotoAdminResolveModerationReports: - 'com.atproto.admin.resolveModerationReports', - ComAtprotoAdminReverseModerationAction: - 'com.atproto.admin.reverseModerationAction', + ComAtprotoAdminQueryModerationEvents: + 'com.atproto.admin.queryModerationEvents', + ComAtprotoAdminQueryModerationStatuses: + 'com.atproto.admin.queryModerationStatuses', ComAtprotoAdminSearchRepos: 'com.atproto.admin.searchRepos', ComAtprotoAdminSendEmail: 'com.atproto.admin.sendEmail', - ComAtprotoAdminTakeModerationAction: 'com.atproto.admin.takeModerationAction', ComAtprotoAdminUpdateAccountEmail: 'com.atproto.admin.updateAccountEmail', ComAtprotoAdminUpdateAccountHandle: 'com.atproto.admin.updateAccountHandle', ComAtprotoAdminUpdateSubjectStatus: 'com.atproto.admin.updateSubjectStatus', diff --git a/packages/api/src/client/types/com/atproto/admin/defs.ts b/packages/api/src/client/types/com/atproto/admin/defs.ts index 8b8197a06d0..cd55a41b97c 100644 --- a/packages/api/src/client/types/com/atproto/admin/defs.ts +++ b/packages/api/src/client/types/com/atproto/admin/defs.ts @@ -28,43 +28,55 @@ export function validateStatusAttr(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#statusAttr', v) } -export interface ActionView { +export interface ModEventView { id: number - action: ActionType - /** Indicates how long this action is meant to be in effect before automatically expiring. */ - durationInHours?: number + event: + | ModEventTakedown + | ModEventReverseTakedown + | ModEventComment + | ModEventReport + | ModEventLabel + | ModEventAcknowledge + | ModEventEscalate + | ModEventMute + | ModEventEmail + | { $type: string; [k: string]: unknown } subject: | RepoRef | ComAtprotoRepoStrongRef.Main | { $type: string; [k: string]: unknown } subjectBlobCids: string[] - createLabelVals?: string[] - negateLabelVals?: string[] - reason: string createdBy: string createdAt: string - reversal?: ActionReversal - resolvedReportIds: number[] + creatorHandle?: string + subjectHandle?: string [k: string]: unknown } -export function isActionView(v: unknown): v is ActionView { +export function isModEventView(v: unknown): v is ModEventView { return ( isObj(v) && hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#actionView' + v.$type === 'com.atproto.admin.defs#modEventView' ) } -export function validateActionView(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#actionView', v) +export function validateModEventView(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventView', v) } -export interface ActionViewDetail { +export interface ModEventViewDetail { id: number - action: ActionType - /** Indicates how long this action is meant to be in effect before automatically expiring. */ - durationInHours?: number + event: + | ModEventTakedown + | ModEventReverseTakedown + | ModEventComment + | ModEventReport + | ModEventLabel + | ModEventAcknowledge + | ModEventEscalate + | ModEventMute + | { $type: string; [k: string]: unknown } subject: | RepoView | RepoViewNotFound @@ -72,123 +84,100 @@ export interface ActionViewDetail { | RecordViewNotFound | { $type: string; [k: string]: unknown } subjectBlobs: BlobView[] - createLabelVals?: string[] - negateLabelVals?: string[] - reason: string createdBy: string createdAt: string - reversal?: ActionReversal - resolvedReports: ReportView[] [k: string]: unknown } -export function isActionViewDetail(v: unknown): v is ActionViewDetail { +export function isModEventViewDetail(v: unknown): v is ModEventViewDetail { return ( isObj(v) && hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#actionViewDetail' + v.$type === 'com.atproto.admin.defs#modEventViewDetail' ) } -export function validateActionViewDetail(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#actionViewDetail', v) +export function validateModEventViewDetail(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventViewDetail', v) } -export interface ActionViewCurrent { +export interface ReportView { id: number - action: ActionType - /** Indicates how long this action is meant to be in effect before automatically expiring. */ - durationInHours?: number - [k: string]: unknown -} - -export function isActionViewCurrent(v: unknown): v is ActionViewCurrent { - return ( - isObj(v) && - hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#actionViewCurrent' - ) -} - -export function validateActionViewCurrent(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#actionViewCurrent', v) -} - -export interface ActionReversal { - reason: string - createdBy: string + reasonType: ComAtprotoModerationDefs.ReasonType + comment?: string + subjectRepoHandle?: string + subject: + | RepoRef + | ComAtprotoRepoStrongRef.Main + | { $type: string; [k: string]: unknown } + reportedBy: string createdAt: string + resolvedByActionIds: number[] [k: string]: unknown } -export function isActionReversal(v: unknown): v is ActionReversal { +export function isReportView(v: unknown): v is ReportView { return ( isObj(v) && hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#actionReversal' + v.$type === 'com.atproto.admin.defs#reportView' ) } -export function validateActionReversal(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#actionReversal', v) +export function validateReportView(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#reportView', v) } -export type ActionType = - | 'lex:com.atproto.admin.defs#takedown' - | 'lex:com.atproto.admin.defs#flag' - | 'lex:com.atproto.admin.defs#acknowledge' - | 'lex:com.atproto.admin.defs#escalate' - | (string & {}) - -/** Moderation action type: Takedown. Indicates that content should not be served by the PDS. */ -export const TAKEDOWN = 'com.atproto.admin.defs#takedown' -/** Moderation action type: Flag. Indicates that the content was reviewed and considered to violate PDS rules, but may still be served. */ -export const FLAG = 'com.atproto.admin.defs#flag' -/** Moderation action type: Acknowledge. Indicates that the content was reviewed and not considered to violate PDS rules. */ -export const ACKNOWLEDGE = 'com.atproto.admin.defs#acknowledge' -/** Moderation action type: Escalate. Indicates that the content has been flagged for additional review. */ -export const ESCALATE = 'com.atproto.admin.defs#escalate' - -export interface ReportView { +export interface SubjectStatusView { id: number - reasonType: ComAtprotoModerationDefs.ReasonType - reason?: string - subjectRepoHandle?: string subject: | RepoRef | ComAtprotoRepoStrongRef.Main | { $type: string; [k: string]: unknown } - reportedBy: string + subjectBlobCids?: string[] + subjectRepoHandle?: string + /** Timestamp referencing when the last update was made to the moderation status of the subject */ + updatedAt: string + /** Timestamp referencing the first moderation status impacting event was emitted on the subject */ createdAt: string - resolvedByActionIds: number[] + reviewState: SubjectReviewState + /** Sticky comment on the subject. */ + comment?: string + muteUntil?: string + lastReviewedBy?: string + lastReviewedAt?: string + lastReportedAt?: string + takendown?: boolean + suspendUntil?: string [k: string]: unknown } -export function isReportView(v: unknown): v is ReportView { +export function isSubjectStatusView(v: unknown): v is SubjectStatusView { return ( isObj(v) && hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#reportView' + v.$type === 'com.atproto.admin.defs#subjectStatusView' ) } -export function validateReportView(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#reportView', v) +export function validateSubjectStatusView(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#subjectStatusView', v) } export interface ReportViewDetail { id: number reasonType: ComAtprotoModerationDefs.ReasonType - reason?: string + comment?: string subject: | RepoView | RepoViewNotFound | RecordView | RecordViewNotFound | { $type: string; [k: string]: unknown } + subjectStatus?: SubjectStatusView reportedBy: string createdAt: string - resolvedByActions: ActionView[] + resolvedByActions: ModEventView[] [k: string]: unknown } @@ -400,7 +389,7 @@ export function validateRecordViewNotFound(v: unknown): ValidationResult { } export interface Moderation { - currentAction?: ActionViewCurrent + subjectStatus?: SubjectStatusView [k: string]: unknown } @@ -417,9 +406,7 @@ export function validateModeration(v: unknown): ValidationResult { } export interface ModerationDetail { - currentAction?: ActionViewCurrent - actions: ActionView[] - reports: ReportView[] + subjectStatus?: SubjectStatusView [k: string]: unknown } @@ -496,3 +483,208 @@ export function isVideoDetails(v: unknown): v is VideoDetails { export function validateVideoDetails(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#videoDetails', v) } + +export type SubjectReviewState = + | 'lex:com.atproto.admin.defs#reviewOpen' + | 'lex:com.atproto.admin.defs#reviewEscalated' + | 'lex:com.atproto.admin.defs#reviewClosed' + | (string & {}) + +/** Moderator review status of a subject: Open. Indicates that the subject needs to be reviewed by a moderator */ +export const REVIEWOPEN = 'com.atproto.admin.defs#reviewOpen' +/** Moderator review status of a subject: Escalated. Indicates that the subject was escalated for review by a moderator */ +export const REVIEWESCALATED = 'com.atproto.admin.defs#reviewEscalated' +/** Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator */ +export const REVIEWCLOSED = 'com.atproto.admin.defs#reviewClosed' + +/** Take down a subject permanently or temporarily */ +export interface ModEventTakedown { + comment?: string + /** Indicates how long the takedown should be in effect before automatically expiring. */ + durationInHours?: number + [k: string]: unknown +} + +export function isModEventTakedown(v: unknown): v is ModEventTakedown { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventTakedown' + ) +} + +export function validateModEventTakedown(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventTakedown', v) +} + +/** Revert take down action on a subject */ +export interface ModEventReverseTakedown { + /** Describe reasoning behind the reversal. */ + comment?: string + [k: string]: unknown +} + +export function isModEventReverseTakedown( + v: unknown, +): v is ModEventReverseTakedown { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventReverseTakedown' + ) +} + +export function validateModEventReverseTakedown(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventReverseTakedown', v) +} + +/** Add a comment to a subject */ +export interface ModEventComment { + comment: string + /** Make the comment persistent on the subject */ + sticky?: boolean + [k: string]: unknown +} + +export function isModEventComment(v: unknown): v is ModEventComment { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventComment' + ) +} + +export function validateModEventComment(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventComment', v) +} + +/** Report a subject */ +export interface ModEventReport { + comment?: string + reportType: ComAtprotoModerationDefs.ReasonType + [k: string]: unknown +} + +export function isModEventReport(v: unknown): v is ModEventReport { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventReport' + ) +} + +export function validateModEventReport(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventReport', v) +} + +/** Apply/Negate labels on a subject */ +export interface ModEventLabel { + comment?: string + createLabelVals: string[] + negateLabelVals: string[] + [k: string]: unknown +} + +export function isModEventLabel(v: unknown): v is ModEventLabel { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventLabel' + ) +} + +export function validateModEventLabel(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventLabel', v) +} + +export interface ModEventAcknowledge { + comment?: string + [k: string]: unknown +} + +export function isModEventAcknowledge(v: unknown): v is ModEventAcknowledge { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventAcknowledge' + ) +} + +export function validateModEventAcknowledge(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventAcknowledge', v) +} + +export interface ModEventEscalate { + comment?: string + [k: string]: unknown +} + +export function isModEventEscalate(v: unknown): v is ModEventEscalate { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventEscalate' + ) +} + +export function validateModEventEscalate(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventEscalate', v) +} + +/** Mute incoming reports on a subject */ +export interface ModEventMute { + comment?: string + /** Indicates how long the subject should remain muted. */ + durationInHours: number + [k: string]: unknown +} + +export function isModEventMute(v: unknown): v is ModEventMute { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventMute' + ) +} + +export function validateModEventMute(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventMute', v) +} + +/** Unmute action on a subject */ +export interface ModEventUnmute { + /** Describe reasoning behind the reversal. */ + comment?: string + [k: string]: unknown +} + +export function isModEventUnmute(v: unknown): v is ModEventUnmute { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventUnmute' + ) +} + +export function validateModEventUnmute(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventUnmute', v) +} + +/** Keep a log of outgoing email to a user */ +export interface ModEventEmail { + /** The subject line of the email sent to the user. */ + subjectLine: string + [k: string]: unknown +} + +export function isModEventEmail(v: unknown): v is ModEventEmail { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventEmail' + ) +} + +export function validateModEventEmail(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventEmail', v) +} diff --git a/packages/api/src/client/types/com/atproto/admin/takeModerationAction.ts b/packages/api/src/client/types/com/atproto/admin/emitModerationEvent.ts similarity index 68% rename from packages/api/src/client/types/com/atproto/admin/takeModerationAction.ts rename to packages/api/src/client/types/com/atproto/admin/emitModerationEvent.ts index 49fba249af7..77b460ed1ff 100644 --- a/packages/api/src/client/types/com/atproto/admin/takeModerationAction.ts +++ b/packages/api/src/client/types/com/atproto/admin/emitModerationEvent.ts @@ -12,26 +12,28 @@ import * as ComAtprotoRepoStrongRef from '../repo/strongRef' export interface QueryParams {} export interface InputSchema { - action: - | 'com.atproto.admin.defs#takedown' - | 'com.atproto.admin.defs#flag' - | 'com.atproto.admin.defs#acknowledge' - | (string & {}) + event: + | ComAtprotoAdminDefs.ModEventTakedown + | ComAtprotoAdminDefs.ModEventAcknowledge + | ComAtprotoAdminDefs.ModEventEscalate + | ComAtprotoAdminDefs.ModEventComment + | ComAtprotoAdminDefs.ModEventLabel + | ComAtprotoAdminDefs.ModEventReport + | ComAtprotoAdminDefs.ModEventMute + | ComAtprotoAdminDefs.ModEventReverseTakedown + | ComAtprotoAdminDefs.ModEventUnmute + | ComAtprotoAdminDefs.ModEventEmail + | { $type: string; [k: string]: unknown } subject: | ComAtprotoAdminDefs.RepoRef | ComAtprotoRepoStrongRef.Main | { $type: string; [k: string]: unknown } subjectBlobCids?: string[] - createLabelVals?: string[] - negateLabelVals?: string[] - reason: string - /** Indicates how long this action is meant to be in effect before automatically expiring. */ - durationInHours?: number createdBy: string [k: string]: unknown } -export type OutputSchema = ComAtprotoAdminDefs.ActionView +export type OutputSchema = ComAtprotoAdminDefs.ModEventView export interface CallOptions { headers?: Headers diff --git a/packages/api/src/client/types/com/atproto/admin/getModerationAction.ts b/packages/api/src/client/types/com/atproto/admin/getModerationEvent.ts similarity index 90% rename from packages/api/src/client/types/com/atproto/admin/getModerationAction.ts rename to packages/api/src/client/types/com/atproto/admin/getModerationEvent.ts index 29edaa65c25..8a107172929 100644 --- a/packages/api/src/client/types/com/atproto/admin/getModerationAction.ts +++ b/packages/api/src/client/types/com/atproto/admin/getModerationEvent.ts @@ -13,7 +13,7 @@ export interface QueryParams { } export type InputSchema = undefined -export type OutputSchema = ComAtprotoAdminDefs.ActionViewDetail +export type OutputSchema = ComAtprotoAdminDefs.ModEventViewDetail export interface CallOptions { headers?: Headers diff --git a/packages/api/src/client/types/com/atproto/admin/getModerationReport.ts b/packages/api/src/client/types/com/atproto/admin/getModerationReport.ts deleted file mode 100644 index 23b1a4e69bf..00000000000 --- a/packages/api/src/client/types/com/atproto/admin/getModerationReport.ts +++ /dev/null @@ -1,32 +0,0 @@ -/** - * GENERATED CODE - DO NOT MODIFY - */ -import { Headers, XRPCError } from '@atproto/xrpc' -import { ValidationResult, BlobRef } from '@atproto/lexicon' -import { isObj, hasProp } from '../../../../util' -import { lexicons } from '../../../../lexicons' -import { CID } from 'multiformats/cid' -import * as ComAtprotoAdminDefs from './defs' - -export interface QueryParams { - id: number -} - -export type InputSchema = undefined -export type OutputSchema = ComAtprotoAdminDefs.ReportViewDetail - -export interface CallOptions { - headers?: Headers -} - -export interface Response { - success: boolean - headers: Headers - data: OutputSchema -} - -export function toKnownErr(e: any) { - if (e instanceof XRPCError) { - } - return e -} diff --git a/packages/api/src/client/types/com/atproto/admin/getModerationReports.ts b/packages/api/src/client/types/com/atproto/admin/getModerationReports.ts deleted file mode 100644 index 3a3e52f59b3..00000000000 --- a/packages/api/src/client/types/com/atproto/admin/getModerationReports.ts +++ /dev/null @@ -1,53 +0,0 @@ -/** - * GENERATED CODE - DO NOT MODIFY - */ -import { Headers, XRPCError } from '@atproto/xrpc' -import { ValidationResult, BlobRef } from '@atproto/lexicon' -import { isObj, hasProp } from '../../../../util' -import { lexicons } from '../../../../lexicons' -import { CID } from 'multiformats/cid' -import * as ComAtprotoAdminDefs from './defs' - -export interface QueryParams { - subject?: string - ignoreSubjects?: string[] - /** Get all reports that were actioned by a specific moderator. */ - actionedBy?: string - /** Filter reports made by one or more DIDs. */ - reporters?: string[] - resolved?: boolean - actionType?: - | 'com.atproto.admin.defs#takedown' - | 'com.atproto.admin.defs#flag' - | 'com.atproto.admin.defs#acknowledge' - | 'com.atproto.admin.defs#escalate' - | (string & {}) - limit?: number - cursor?: string - /** Reverse the order of the returned records. When true, returns reports in chronological order. */ - reverse?: boolean -} - -export type InputSchema = undefined - -export interface OutputSchema { - cursor?: string - reports: ComAtprotoAdminDefs.ReportView[] - [k: string]: unknown -} - -export interface CallOptions { - headers?: Headers -} - -export interface Response { - success: boolean - headers: Headers - data: OutputSchema -} - -export function toKnownErr(e: any) { - if (e instanceof XRPCError) { - } - return e -} diff --git a/packages/api/src/client/types/com/atproto/admin/getModerationActions.ts b/packages/api/src/client/types/com/atproto/admin/queryModerationEvents.ts similarity index 60% rename from packages/api/src/client/types/com/atproto/admin/getModerationActions.ts rename to packages/api/src/client/types/com/atproto/admin/queryModerationEvents.ts index 69ed008a28b..ed21c739bcb 100644 --- a/packages/api/src/client/types/com/atproto/admin/getModerationActions.ts +++ b/packages/api/src/client/types/com/atproto/admin/queryModerationEvents.ts @@ -9,7 +9,14 @@ import { CID } from 'multiformats/cid' import * as ComAtprotoAdminDefs from './defs' export interface QueryParams { + /** The types of events (fully qualified string in the format of com.atproto.admin#modEvent) to filter by. If not specified, all events are returned. */ + types?: string[] + createdBy?: string + /** Sort direction for the events. Defaults to descending order of created at timestamp. */ + sortDirection?: 'asc' | 'desc' subject?: string + /** If true, events on all record types (posts, lists, profile etc.) owned by the did are returned */ + includeAllUserRecords?: boolean limit?: number cursor?: string } @@ -18,7 +25,7 @@ export type InputSchema = undefined export interface OutputSchema { cursor?: string - actions: ComAtprotoAdminDefs.ActionView[] + events: ComAtprotoAdminDefs.ModEventView[] [k: string]: unknown } diff --git a/packages/api/src/client/types/com/atproto/admin/queryModerationStatuses.ts b/packages/api/src/client/types/com/atproto/admin/queryModerationStatuses.ts new file mode 100644 index 00000000000..80eb17d8cb3 --- /dev/null +++ b/packages/api/src/client/types/com/atproto/admin/queryModerationStatuses.ts @@ -0,0 +1,60 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import { Headers, XRPCError } from '@atproto/xrpc' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { isObj, hasProp } from '../../../../util' +import { lexicons } from '../../../../lexicons' +import { CID } from 'multiformats/cid' +import * as ComAtprotoAdminDefs from './defs' + +export interface QueryParams { + subject?: string + /** Search subjects by keyword from comments */ + comment?: string + /** Search subjects reported after a given timestamp */ + reportedAfter?: string + /** Search subjects reported before a given timestamp */ + reportedBefore?: string + /** Search subjects reviewed after a given timestamp */ + reviewedAfter?: string + /** Search subjects reviewed before a given timestamp */ + reviewedBefore?: string + /** By default, we don't include muted subjects in the results. Set this to true to include them. */ + includeMuted?: boolean + /** Specify when fetching subjects in a certain state */ + reviewState?: string + ignoreSubjects?: string[] + /** Get all subject statuses that were reviewed by a specific moderator */ + lastReviewedBy?: string + sortField?: 'lastReviewedAt' | 'lastReportedAt' + sortDirection?: 'asc' | 'desc' + /** Get subjects that were taken down */ + takendown?: boolean + limit?: number + cursor?: string +} + +export type InputSchema = undefined + +export interface OutputSchema { + cursor?: string + subjectStatuses: ComAtprotoAdminDefs.SubjectStatusView[] + [k: string]: unknown +} + +export interface CallOptions { + headers?: Headers +} + +export interface Response { + success: boolean + headers: Headers + data: OutputSchema +} + +export function toKnownErr(e: any) { + if (e instanceof XRPCError) { + } + return e +} diff --git a/packages/api/src/client/types/com/atproto/admin/resolveModerationReports.ts b/packages/api/src/client/types/com/atproto/admin/resolveModerationReports.ts deleted file mode 100644 index 2330cc804e3..00000000000 --- a/packages/api/src/client/types/com/atproto/admin/resolveModerationReports.ts +++ /dev/null @@ -1,38 +0,0 @@ -/** - * GENERATED CODE - DO NOT MODIFY - */ -import { Headers, XRPCError } from '@atproto/xrpc' -import { ValidationResult, BlobRef } from '@atproto/lexicon' -import { isObj, hasProp } from '../../../../util' -import { lexicons } from '../../../../lexicons' -import { CID } from 'multiformats/cid' -import * as ComAtprotoAdminDefs from './defs' - -export interface QueryParams {} - -export interface InputSchema { - actionId: number - reportIds: number[] - createdBy: string - [k: string]: unknown -} - -export type OutputSchema = ComAtprotoAdminDefs.ActionView - -export interface CallOptions { - headers?: Headers - qp?: QueryParams - encoding: 'application/json' -} - -export interface Response { - success: boolean - headers: Headers - data: OutputSchema -} - -export function toKnownErr(e: any) { - if (e instanceof XRPCError) { - } - return e -} diff --git a/packages/api/src/client/types/com/atproto/admin/reverseModerationAction.ts b/packages/api/src/client/types/com/atproto/admin/reverseModerationAction.ts deleted file mode 100644 index d7e4ae159ff..00000000000 --- a/packages/api/src/client/types/com/atproto/admin/reverseModerationAction.ts +++ /dev/null @@ -1,38 +0,0 @@ -/** - * GENERATED CODE - DO NOT MODIFY - */ -import { Headers, XRPCError } from '@atproto/xrpc' -import { ValidationResult, BlobRef } from '@atproto/lexicon' -import { isObj, hasProp } from '../../../../util' -import { lexicons } from '../../../../lexicons' -import { CID } from 'multiformats/cid' -import * as ComAtprotoAdminDefs from './defs' - -export interface QueryParams {} - -export interface InputSchema { - id: number - reason: string - createdBy: string - [k: string]: unknown -} - -export type OutputSchema = ComAtprotoAdminDefs.ActionView - -export interface CallOptions { - headers?: Headers - qp?: QueryParams - encoding: 'application/json' -} - -export interface Response { - success: boolean - headers: Headers - data: OutputSchema -} - -export function toKnownErr(e: any) { - if (e instanceof XRPCError) { - } - return e -} diff --git a/packages/api/src/client/types/com/atproto/admin/sendEmail.ts b/packages/api/src/client/types/com/atproto/admin/sendEmail.ts index d2d8b0fecbf..3357ef3f762 100644 --- a/packages/api/src/client/types/com/atproto/admin/sendEmail.ts +++ b/packages/api/src/client/types/com/atproto/admin/sendEmail.ts @@ -13,6 +13,7 @@ export interface InputSchema { recipientDid: string content: string subject?: string + senderDid: string [k: string]: unknown } diff --git a/packages/bsky/src/api/blob-resolver.ts b/packages/bsky/src/api/blob-resolver.ts index c366583c246..7eb245eedd5 100644 --- a/packages/bsky/src/api/blob-resolver.ts +++ b/packages/bsky/src/api/blob-resolver.ts @@ -6,11 +6,11 @@ import { CID } from 'multiformats/cid' import { ensureValidDid } from '@atproto/syntax' import { forwardStreamErrors, VerifyCidTransform } from '@atproto/common' import { IdResolver, DidNotFoundError } from '@atproto/identity' -import { TAKEDOWN } from '../lexicon/types/com/atproto/admin/defs' import AppContext from '../context' import { httpLogger as log } from '../logger' import { retryHttp } from '../util/retry' import { Database } from '../db' +import { sql } from 'kysely' // Resolve and verify blob from its origin host @@ -84,19 +84,14 @@ export async function resolveBlob( idResolver: IdResolver, ) { const cidStr = cid.toString() + const [{ pds }, takedown] = await Promise.all([ idResolver.did.resolveAtprotoData(did), // @TODO cache did info db.db - .selectFrom('moderation_action_subject_blob') - .select('actionId') - .innerJoin( - 'moderation_action', - 'moderation_action.id', - 'moderation_action_subject_blob.actionId', - ) - .where('cid', '=', cidStr) - .where('action', '=', TAKEDOWN) - .where('reversedAt', 'is', null) + .selectFrom('moderation_subject_status') + .select('id') + .where('blobCids', '@>', sql`CAST(${JSON.stringify([cidStr])} AS JSONB)`) + .where('takendown', 'is', true) .executeTakeFirst(), ]) if (takedown) { diff --git a/packages/bsky/src/api/com/atproto/admin/emitModerationEvent.ts b/packages/bsky/src/api/com/atproto/admin/emitModerationEvent.ts new file mode 100644 index 00000000000..8b007f64ca1 --- /dev/null +++ b/packages/bsky/src/api/com/atproto/admin/emitModerationEvent.ts @@ -0,0 +1,220 @@ +import { CID } from 'multiformats/cid' +import { AtUri } from '@atproto/syntax' +import { + AuthRequiredError, + InvalidRequestError, + UpstreamFailureError, +} from '@atproto/xrpc-server' +import { Server } from '../../../../lexicon' +import AppContext from '../../../../context' +import { getSubject } from '../moderation/util' +import { + isModEventLabel, + isModEventReverseTakedown, + isModEventTakedown, +} from '../../../../lexicon/types/com/atproto/admin/defs' +import { TakedownSubjects } from '../../../../services/moderation' +import { retryHttp } from '../../../../util/retry' + +export default function (server: Server, ctx: AppContext) { + server.com.atproto.admin.emitModerationEvent({ + auth: ctx.roleVerifier, + handler: async ({ input, auth }) => { + const access = auth.credentials + const db = ctx.db.getPrimary() + const moderationService = ctx.services.moderation(db) + const { subject, createdBy, subjectBlobCids, event } = input.body + const isTakedownEvent = isModEventTakedown(event) + const isReverseTakedownEvent = isModEventReverseTakedown(event) + const isLabelEvent = isModEventLabel(event) + + // apply access rules + + // if less than moderator access then can not takedown an account + if (!access.moderator && isTakedownEvent && 'did' in subject) { + throw new AuthRequiredError( + 'Must be a full moderator to perform an account takedown', + ) + } + // if less than moderator access then can only take ack and escalation actions + if (!access.moderator && (isTakedownEvent || isReverseTakedownEvent)) { + throw new AuthRequiredError( + 'Must be a full moderator to take this type of action', + ) + } + // if less than moderator access then can not apply labels + if (!access.moderator && isLabelEvent) { + throw new AuthRequiredError('Must be a full moderator to label content') + } + + if (isLabelEvent) { + validateLabels([ + ...(event.createLabelVals ?? []), + ...(event.negateLabelVals ?? []), + ]) + } + + const subjectInfo = getSubject(subject) + + if (isTakedownEvent || isReverseTakedownEvent) { + const isSubjectTakendown = await moderationService.isSubjectTakendown( + subjectInfo, + ) + + if (isSubjectTakendown && isTakedownEvent) { + throw new InvalidRequestError(`Subject is already taken down`) + } + + if (!isSubjectTakendown && isReverseTakedownEvent) { + throw new InvalidRequestError(`Subject is not taken down`) + } + } + + const { result: moderationEvent, takenDown } = await db.transaction( + async (dbTxn) => { + const moderationTxn = ctx.services.moderation(dbTxn) + const labelTxn = ctx.services.label(dbTxn) + + const result = await moderationTxn.logEvent({ + event, + subject: subjectInfo, + subjectBlobCids: + subjectBlobCids?.map((cid) => CID.parse(cid)) ?? [], + createdBy, + }) + + let takenDown: TakedownSubjects | undefined + + if ( + result.subjectType === 'com.atproto.admin.defs#repoRef' && + result.subjectDid + ) { + // No credentials to revoke on appview + if (isTakedownEvent) { + takenDown = await moderationTxn.takedownRepo({ + takedownId: result.id, + did: result.subjectDid, + }) + } + + if (isReverseTakedownEvent) { + await moderationTxn.reverseTakedownRepo({ + did: result.subjectDid, + }) + takenDown = { + subjects: [ + { + $type: 'com.atproto.admin.defs#repoRef', + did: result.subjectDid, + }, + ], + did: result.subjectDid, + } + } + } + + if ( + result.subjectType === 'com.atproto.repo.strongRef' && + result.subjectUri + ) { + const blobCids = subjectBlobCids?.map((cid) => CID.parse(cid)) ?? [] + if (isTakedownEvent) { + takenDown = await moderationTxn.takedownRecord({ + takedownId: result.id, + uri: new AtUri(result.subjectUri), + // TODO: I think this will always be available for strongRefs? + cid: CID.parse(result.subjectCid as string), + blobCids, + }) + } + + if (isReverseTakedownEvent) { + await moderationTxn.reverseTakedownRecord({ + uri: new AtUri(result.subjectUri), + }) + takenDown = { + did: result.subjectDid, + subjects: [ + { + $type: 'com.atproto.repo.strongRef', + uri: result.subjectUri, + cid: result.subjectCid ?? '', + }, + ...blobCids.map((cid) => ({ + $type: 'com.atproto.admin.defs#repoBlobRef', + did: result.subjectDid, + cid: cid.toString(), + recordUri: result.subjectUri, + })), + ], + } + } + } + + if (isLabelEvent) { + await labelTxn.formatAndCreate( + ctx.cfg.labelerDid, + result.subjectUri ?? result.subjectDid, + result.subjectCid, + { + create: result.createLabelVals?.length + ? result.createLabelVals.split(' ') + : undefined, + negate: result.negateLabelVals?.length + ? result.negateLabelVals.split(' ') + : undefined, + }, + ) + } + + return { result, takenDown } + }, + ) + + if (takenDown && ctx.moderationPushAgent) { + const { did, subjects } = takenDown + if (did && subjects.length > 0) { + const agent = ctx.moderationPushAgent + const results = await Promise.allSettled( + subjects.map((subject) => + retryHttp(() => + agent.api.com.atproto.admin.updateSubjectStatus({ + subject, + takedown: isTakedownEvent + ? { + applied: true, + ref: moderationEvent.id.toString(), + } + : { + applied: false, + }, + }), + ), + ), + ) + const hadFailure = results.some((r) => r.status === 'rejected') + if (hadFailure) { + throw new UpstreamFailureError('failed to apply action on PDS') + } + } + } + + return { + encoding: 'application/json', + body: await moderationService.views.event(moderationEvent), + } + }, + }) +} + +const validateLabels = (labels: string[]) => { + for (const label of labels) { + for (const char of badChars) { + if (label.includes(char)) { + throw new InvalidRequestError(`Invalid label: ${label}`) + } + } + } +} + +const badChars = [' ', ',', ';', `'`, `"`] diff --git a/packages/bsky/src/api/com/atproto/admin/getModerationAction.ts b/packages/bsky/src/api/com/atproto/admin/getModerationAction.ts deleted file mode 100644 index 51218077bcf..00000000000 --- a/packages/bsky/src/api/com/atproto/admin/getModerationAction.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' -import { addAccountInfoToRepoView, getPdsAccountInfo } from './util' -import { - isRecordView, - isRepoView, -} from '../../../../lexicon/types/com/atproto/admin/defs' - -export default function (server: Server, ctx: AppContext) { - server.com.atproto.admin.getModerationAction({ - auth: ctx.roleVerifier, - handler: async ({ params, auth }) => { - const { id } = params - const db = ctx.db.getPrimary() - const moderationService = ctx.services.moderation(db) - const result = await moderationService.getActionOrThrow(id) - - const [action, accountInfo] = await Promise.all([ - moderationService.views.actionDetail(result), - getPdsAccountInfo(ctx, result.subjectDid), - ]) - - // add in pds account info if available - if (isRepoView(action.subject)) { - action.subject = addAccountInfoToRepoView( - action.subject, - accountInfo, - auth.credentials.moderator, - ) - } else if (isRecordView(action.subject)) { - action.subject.repo = addAccountInfoToRepoView( - action.subject.repo, - accountInfo, - auth.credentials.moderator, - ) - } - - return { - encoding: 'application/json', - body: action, - } - }, - }) -} diff --git a/packages/bsky/src/api/com/atproto/admin/getModerationActions.ts b/packages/bsky/src/api/com/atproto/admin/getModerationEvent.ts similarity index 50% rename from packages/bsky/src/api/com/atproto/admin/getModerationActions.ts rename to packages/bsky/src/api/com/atproto/admin/getModerationEvent.ts index ef28ef10b7a..347a450c727 100644 --- a/packages/bsky/src/api/com/atproto/admin/getModerationActions.ts +++ b/packages/bsky/src/api/com/atproto/admin/getModerationEvent.ts @@ -2,23 +2,17 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' export default function (server: Server, ctx: AppContext) { - server.com.atproto.admin.getModerationActions({ + server.com.atproto.admin.getModerationEvent({ auth: ctx.roleVerifier, handler: async ({ params }) => { - const { subject, limit = 50, cursor } = params + const { id } = params const db = ctx.db.getPrimary() const moderationService = ctx.services.moderation(db) - const results = await moderationService.getActions({ - subject, - limit, - cursor, - }) + const event = await moderationService.getEventOrThrow(id) + const eventDetail = await moderationService.views.eventDetail(event) return { encoding: 'application/json', - body: { - cursor: results.at(-1)?.id.toString() ?? undefined, - actions: await moderationService.views.action(results), - }, + body: eventDetail, } }, }) diff --git a/packages/bsky/src/api/com/atproto/admin/getModerationReport.ts b/packages/bsky/src/api/com/atproto/admin/getModerationReport.ts deleted file mode 100644 index 814d1069e3f..00000000000 --- a/packages/bsky/src/api/com/atproto/admin/getModerationReport.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' -import { - isRecordView, - isRepoView, -} from '../../../../lexicon/types/com/atproto/admin/defs' -import { addAccountInfoToRepoView, getPdsAccountInfo } from './util' - -export default function (server: Server, ctx: AppContext) { - server.com.atproto.admin.getModerationReport({ - auth: ctx.roleVerifier, - handler: async ({ params, auth }) => { - const { id } = params - const db = ctx.db.getPrimary() - const moderationService = ctx.services.moderation(db) - const result = await moderationService.getReportOrThrow(id) - const [report, accountInfo] = await Promise.all([ - moderationService.views.reportDetail(result), - getPdsAccountInfo(ctx, result.subjectDid), - ]) - - // add in pds account info if available - if (isRepoView(report.subject)) { - report.subject = addAccountInfoToRepoView( - report.subject, - accountInfo, - auth.credentials.moderator, - ) - } else if (isRecordView(report.subject)) { - report.subject.repo = addAccountInfoToRepoView( - report.subject.repo, - accountInfo, - auth.credentials.moderator, - ) - } - - return { - encoding: 'application/json', - body: report, - } - }, - }) -} diff --git a/packages/bsky/src/api/com/atproto/admin/getRecord.ts b/packages/bsky/src/api/com/atproto/admin/getRecord.ts index 245ce2b8f26..8e459910806 100644 --- a/packages/bsky/src/api/com/atproto/admin/getRecord.ts +++ b/packages/bsky/src/api/com/atproto/admin/getRecord.ts @@ -18,6 +18,7 @@ export default function (server: Server, ctx: AppContext) { if (!result) { throw new InvalidRequestError('Record not found', 'RecordNotFound') } + const [record, accountInfo] = await Promise.all([ ctx.services.moderation(db).views.recordDetail(result), getPdsAccountInfo(ctx, result.did), diff --git a/packages/bsky/src/api/com/atproto/admin/getModerationReports.ts b/packages/bsky/src/api/com/atproto/admin/queryModerationEvents.ts similarity index 52% rename from packages/bsky/src/api/com/atproto/admin/getModerationReports.ts rename to packages/bsky/src/api/com/atproto/admin/queryModerationEvents.ts index d3956973f37..1868533295c 100644 --- a/packages/bsky/src/api/com/atproto/admin/getModerationReports.ts +++ b/packages/bsky/src/api/com/atproto/admin/queryModerationEvents.ts @@ -1,39 +1,36 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' +import { getEventType } from '../moderation/util' export default function (server: Server, ctx: AppContext) { - server.com.atproto.admin.getModerationReports({ + server.com.atproto.admin.queryModerationEvents({ auth: ctx.roleVerifier, handler: async ({ params }) => { const { subject, - resolved, - actionType, limit = 50, cursor, - ignoreSubjects, - reverse = false, - reporters = [], - actionedBy, + sortDirection = 'desc', + types, + includeAllUserRecords = false, + createdBy, } = params const db = ctx.db.getPrimary() const moderationService = ctx.services.moderation(db) - const results = await moderationService.getReports({ + const results = await moderationService.getEvents({ + types: types?.length ? types.map(getEventType) : [], subject, - resolved, - actionType, + createdBy, limit, cursor, - ignoreSubjects, - reverse, - reporters, - actionedBy, + sortDirection, + includeAllUserRecords, }) return { encoding: 'application/json', body: { - cursor: results.at(-1)?.id.toString() ?? undefined, - reports: await moderationService.views.report(results), + cursor: results.cursor, + events: await moderationService.views.event(results.events), }, } }, diff --git a/packages/bsky/src/api/com/atproto/admin/queryModerationStatuses.ts b/packages/bsky/src/api/com/atproto/admin/queryModerationStatuses.ts new file mode 100644 index 00000000000..5a74bfca3ae --- /dev/null +++ b/packages/bsky/src/api/com/atproto/admin/queryModerationStatuses.ts @@ -0,0 +1,55 @@ +import { Server } from '../../../../lexicon' +import AppContext from '../../../../context' +import { getReviewState } from '../moderation/util' + +export default function (server: Server, ctx: AppContext) { + server.com.atproto.admin.queryModerationStatuses({ + auth: ctx.roleVerifier, + handler: async ({ params }) => { + const { + subject, + takendown, + reviewState, + reviewedAfter, + reviewedBefore, + reportedAfter, + reportedBefore, + ignoreSubjects, + lastReviewedBy, + sortDirection = 'desc', + sortField = 'lastReportedAt', + includeMuted = false, + limit = 50, + cursor, + } = params + const db = ctx.db.getPrimary() + const moderationService = ctx.services.moderation(db) + const results = await moderationService.getSubjectStatuses({ + reviewState: getReviewState(reviewState), + subject, + takendown, + reviewedAfter, + reviewedBefore, + reportedAfter, + reportedBefore, + includeMuted, + ignoreSubjects, + sortDirection, + lastReviewedBy, + sortField, + limit, + cursor, + }) + const subjectStatuses = moderationService.views.subjectStatus( + results.statuses, + ) + return { + encoding: 'application/json', + body: { + cursor: results.cursor, + subjectStatuses, + }, + } + }, + }) +} diff --git a/packages/bsky/src/api/com/atproto/admin/resolveModerationReports.ts b/packages/bsky/src/api/com/atproto/admin/resolveModerationReports.ts deleted file mode 100644 index ed420e7d820..00000000000 --- a/packages/bsky/src/api/com/atproto/admin/resolveModerationReports.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' - -export default function (server: Server, ctx: AppContext) { - server.com.atproto.admin.resolveModerationReports({ - auth: ctx.roleVerifier, - handler: async ({ input }) => { - const db = ctx.db.getPrimary() - const moderationService = ctx.services.moderation(db) - const { actionId, reportIds, createdBy } = input.body - - const moderationAction = await db.transaction(async (dbTxn) => { - const moderationTxn = ctx.services.moderation(dbTxn) - await moderationTxn.resolveReports({ reportIds, actionId, createdBy }) - return await moderationTxn.getActionOrThrow(actionId) - }) - - return { - encoding: 'application/json', - body: await moderationService.views.action(moderationAction), - } - }, - }) -} diff --git a/packages/bsky/src/api/com/atproto/admin/reverseModerationAction.ts b/packages/bsky/src/api/com/atproto/admin/reverseModerationAction.ts deleted file mode 100644 index a441d2b934c..00000000000 --- a/packages/bsky/src/api/com/atproto/admin/reverseModerationAction.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { - AuthRequiredError, - InvalidRequestError, - UpstreamFailureError, -} from '@atproto/xrpc-server' -import { - ACKNOWLEDGE, - ESCALATE, - TAKEDOWN, -} from '../../../../lexicon/types/com/atproto/admin/defs' -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' -import { retryHttp } from '../../../../util/retry' - -export default function (server: Server, ctx: AppContext) { - server.com.atproto.admin.reverseModerationAction({ - auth: ctx.roleVerifier, - handler: async ({ input, auth }) => { - const access = auth.credentials - const db = ctx.db.getPrimary() - const moderationService = ctx.services.moderation(db) - const { id, createdBy, reason } = input.body - - const { result, restored } = await db.transaction(async (dbTxn) => { - const moderationTxn = ctx.services.moderation(dbTxn) - const labelTxn = ctx.services.label(dbTxn) - const now = new Date() - - const existing = await moderationTxn.getAction(id) - if (!existing) { - throw new InvalidRequestError('Moderation action does not exist') - } - if (existing.reversedAt !== null) { - throw new InvalidRequestError( - 'Moderation action has already been reversed', - ) - } - - // apply access rules - - // if less than moderator access then can only reverse ack and escalation actions - if ( - !access.moderator && - ![ACKNOWLEDGE, ESCALATE].includes(existing.action) - ) { - throw new AuthRequiredError( - 'Must be a full moderator to reverse this type of action', - ) - } - // if less than moderator access then cannot reverse takedown on an account - if ( - !access.moderator && - existing.action === TAKEDOWN && - existing.subjectType === 'com.atproto.admin.defs#repoRef' - ) { - throw new AuthRequiredError( - 'Must be a full moderator to reverse an account takedown', - ) - } - - const { result, restored } = await moderationTxn.revertAction({ - id, - createdAt: now, - createdBy, - reason, - }) - - // invert creates & negates - const { createLabelVals, negateLabelVals } = result - const negate = - createLabelVals && createLabelVals.length > 0 - ? createLabelVals.split(' ') - : undefined - const create = - negateLabelVals && negateLabelVals.length > 0 - ? negateLabelVals.split(' ') - : undefined - await labelTxn.formatAndCreate( - ctx.cfg.labelerDid, - result.subjectUri ?? result.subjectDid, - result.subjectCid, - { create, negate }, - ) - - return { result, restored } - }) - - if (restored && ctx.moderationPushAgent) { - const agent = ctx.moderationPushAgent - const { subjects } = restored - const results = await Promise.allSettled( - subjects.map((subject) => - retryHttp(() => - agent.api.com.atproto.admin.updateSubjectStatus({ - subject, - takedown: { - applied: false, - }, - }), - ), - ), - ) - const hadFailure = results.some((r) => r.status === 'rejected') - if (hadFailure) { - throw new UpstreamFailureError('failed to revert action on PDS') - } - } - - return { - encoding: 'application/json', - body: await moderationService.views.action(result), - } - }, - }) -} diff --git a/packages/bsky/src/api/com/atproto/admin/takeModerationAction.ts b/packages/bsky/src/api/com/atproto/admin/takeModerationAction.ts deleted file mode 100644 index 5239ddec42d..00000000000 --- a/packages/bsky/src/api/com/atproto/admin/takeModerationAction.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { CID } from 'multiformats/cid' -import { AtUri } from '@atproto/syntax' -import { - AuthRequiredError, - InvalidRequestError, - UpstreamFailureError, -} from '@atproto/xrpc-server' -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' -import { - ACKNOWLEDGE, - ESCALATE, - TAKEDOWN, -} from '../../../../lexicon/types/com/atproto/admin/defs' -import { getSubject, getAction } from '../moderation/util' -import { TakedownSubjects } from '../../../../services/moderation' -import { retryHttp } from '../../../../util/retry' - -export default function (server: Server, ctx: AppContext) { - server.com.atproto.admin.takeModerationAction({ - auth: ctx.roleVerifier, - handler: async ({ input, auth }) => { - const access = auth.credentials - const db = ctx.db.getPrimary() - const moderationService = ctx.services.moderation(db) - const { - action, - subject, - reason, - createdBy, - createLabelVals, - negateLabelVals, - subjectBlobCids, - durationInHours, - } = input.body - - // apply access rules - - // if less than admin access then can not takedown an account - if (!access.moderator && action === TAKEDOWN && 'did' in subject) { - throw new AuthRequiredError( - 'Must be a full moderator to perform an account takedown', - ) - } - // if less than moderator access then can only take ack and escalation actions - if (!access.moderator && ![ACKNOWLEDGE, ESCALATE].includes(action)) { - throw new AuthRequiredError( - 'Must be a full moderator to take this type of action', - ) - } - // if less than moderator access then can not apply labels - if ( - !access.moderator && - (createLabelVals?.length || negateLabelVals?.length) - ) { - throw new AuthRequiredError('Must be a full moderator to label content') - } - - validateLabels([...(createLabelVals ?? []), ...(negateLabelVals ?? [])]) - - const { result, takenDown } = await db.transaction(async (dbTxn) => { - const moderationTxn = ctx.services.moderation(dbTxn) - const labelTxn = ctx.services.label(dbTxn) - - const result = await moderationTxn.logAction({ - action: getAction(action), - subject: getSubject(subject), - subjectBlobCids: subjectBlobCids?.map((cid) => CID.parse(cid)) ?? [], - createLabelVals, - negateLabelVals, - createdBy, - reason, - durationInHours, - }) - - let takenDown: TakedownSubjects | undefined - - if ( - result.action === TAKEDOWN && - result.subjectType === 'com.atproto.admin.defs#repoRef' && - result.subjectDid - ) { - // No credentials to revoke on appview - takenDown = await moderationTxn.takedownRepo({ - takedownId: result.id, - did: result.subjectDid, - }) - } - - if ( - result.action === TAKEDOWN && - result.subjectType === 'com.atproto.repo.strongRef' && - result.subjectUri && - result.subjectCid - ) { - takenDown = await moderationTxn.takedownRecord({ - takedownId: result.id, - uri: new AtUri(result.subjectUri), - cid: CID.parse(result.subjectCid), - blobCids: subjectBlobCids?.map((cid) => CID.parse(cid)) ?? [], - }) - } - - await labelTxn.formatAndCreate( - ctx.cfg.labelerDid, - result.subjectUri ?? result.subjectDid, - result.subjectCid, - { create: createLabelVals, negate: negateLabelVals }, - ) - - return { result, takenDown } - }) - - if (takenDown && ctx.moderationPushAgent) { - const agent = ctx.moderationPushAgent - const { did, subjects } = takenDown - if (did && subjects.length > 0) { - const results = await Promise.allSettled( - subjects.map((subject) => - retryHttp(() => - agent.api.com.atproto.admin.updateSubjectStatus({ - subject, - takedown: { - applied: true, - ref: result.id.toString(), - }, - }), - ), - ), - ) - const hadFailure = results.some((r) => r.status === 'rejected') - if (hadFailure) { - throw new UpstreamFailureError('failed to apply action on PDS') - } - } - } - - return { - encoding: 'application/json', - body: await moderationService.views.action(result), - } - }, - }) -} - -const validateLabels = (labels: string[]) => { - for (const label of labels) { - for (const char of badChars) { - if (label.includes(char)) { - throw new InvalidRequestError(`Invalid label: ${label}`) - } - } - } -} - -const badChars = [' ', ',', ';', `'`, `"`] diff --git a/packages/bsky/src/api/com/atproto/moderation/createReport.ts b/packages/bsky/src/api/com/atproto/moderation/createReport.ts index 4cef67f1c65..b247a319527 100644 --- a/packages/bsky/src/api/com/atproto/moderation/createReport.ts +++ b/packages/bsky/src/api/com/atproto/moderation/createReport.ts @@ -22,15 +22,17 @@ export default function (server: Server, ctx: AppContext) { } } - const moderationService = ctx.services.moderation(db) - - const report = await moderationService.report({ - reasonType: getReasonType(reasonType), - reason, - subject: getSubject(subject), - reportedBy: requester || ctx.cfg.serverDid, + const report = await db.transaction(async (dbTxn) => { + const moderationTxn = ctx.services.moderation(dbTxn) + return moderationTxn.report({ + reasonType: getReasonType(reasonType), + reason, + subject: getSubject(subject), + reportedBy: requester || ctx.cfg.serverDid, + }) }) + const moderationService = ctx.services.moderation(db) return { encoding: 'application/json', body: moderationService.views.reportPublic(report), diff --git a/packages/bsky/src/api/com/atproto/moderation/util.ts b/packages/bsky/src/api/com/atproto/moderation/util.ts index d856148ee08..bc0ece2ff9f 100644 --- a/packages/bsky/src/api/com/atproto/moderation/util.ts +++ b/packages/bsky/src/api/com/atproto/moderation/util.ts @@ -1,16 +1,8 @@ import { CID } from 'multiformats/cid' import { InvalidRequestError } from '@atproto/xrpc-server' import { AtUri } from '@atproto/syntax' -import { ModerationAction } from '../../../../db/tables/moderation' -import { ModerationReport } from '../../../../db/tables/moderation' import { InputSchema as ReportInput } from '../../../../lexicon/types/com/atproto/moderation/createReport' -import { InputSchema as ActionInput } from '../../../../lexicon/types/com/atproto/admin/takeModerationAction' -import { - ACKNOWLEDGE, - FLAG, - TAKEDOWN, - ESCALATE, -} from '../../../../lexicon/types/com/atproto/admin/defs' +import { InputSchema as ActionInput } from '../../../../lexicon/types/com/atproto/admin/emitModerationEvent' import { REASONOTHER, REASONSPAM, @@ -19,6 +11,13 @@ import { REASONSEXUAL, REASONVIOLATION, } from '../../../../lexicon/types/com/atproto/moderation/defs' +import { + REVIEWCLOSED, + REVIEWESCALATED, + REVIEWOPEN, +} from '../../../../lexicon/types/com/atproto/admin/defs' +import { ModerationEvent } from '../../../../db/tables/moderation' +import { ModerationSubjectStatusRow } from '../../../../services/moderation/types' type SubjectInput = ReportInput['subject'] | ActionInput['subject'] @@ -34,8 +33,9 @@ export const getSubject = (subject: SubjectInput) => { typeof subject.uri === 'string' && typeof subject.cid === 'string' ) { + const uri = new AtUri(subject.uri) return { - uri: new AtUri(subject.uri), + uri, cid: CID.parse(subject.cid), } } @@ -44,23 +44,28 @@ export const getSubject = (subject: SubjectInput) => { export const getReasonType = (reasonType: ReportInput['reasonType']) => { if (reasonTypes.has(reasonType)) { - return reasonType as ModerationReport['reasonType'] + return reasonType as NonNullable['reportType'] } throw new InvalidRequestError('Invalid reason type') } -export const getAction = (action: ActionInput['action']) => { - if ( - action === TAKEDOWN || - action === FLAG || - action === ACKNOWLEDGE || - action === ESCALATE - ) { - return action as ModerationAction['action'] +export const getEventType = (type: string) => { + if (eventTypes.has(type)) { + return type as ModerationEvent['action'] } - throw new InvalidRequestError('Invalid action') + throw new InvalidRequestError('Invalid event type') } +export const getReviewState = (reviewState?: string) => { + if (!reviewState) return undefined + if (reviewStates.has(reviewState)) { + return reviewState as ModerationSubjectStatusRow['reviewState'] + } + throw new InvalidRequestError('Invalid review state') +} + +const reviewStates = new Set([REVIEWCLOSED, REVIEWESCALATED, REVIEWOPEN]) + const reasonTypes = new Set([ REASONOTHER, REASONSPAM, @@ -69,3 +74,16 @@ const reasonTypes = new Set([ REASONSEXUAL, REASONVIOLATION, ]) + +const eventTypes = new Set([ + 'com.atproto.admin.defs#modEventTakedown', + 'com.atproto.admin.defs#modEventAcknowledge', + 'com.atproto.admin.defs#modEventEscalate', + 'com.atproto.admin.defs#modEventComment', + 'com.atproto.admin.defs#modEventLabel', + 'com.atproto.admin.defs#modEventReport', + 'com.atproto.admin.defs#modEventMute', + 'com.atproto.admin.defs#modEventUnmute', + 'com.atproto.admin.defs#modEventReverseTakedown', + 'com.atproto.admin.defs#modEventEmail', +]) diff --git a/packages/bsky/src/api/index.ts b/packages/bsky/src/api/index.ts index 786fdd00e5d..da21b582019 100644 --- a/packages/bsky/src/api/index.ts +++ b/packages/bsky/src/api/index.ts @@ -41,18 +41,15 @@ import registerPush from './app/bsky/notification/registerPush' import getPopularFeedGenerators from './app/bsky/unspecced/getPopularFeedGenerators' import getTimelineSkeleton from './app/bsky/unspecced/getTimelineSkeleton' import createReport from './com/atproto/moderation/createReport' -import resolveModerationReports from './com/atproto/admin/resolveModerationReports' -import reverseModerationAction from './com/atproto/admin/reverseModerationAction' -import takeModerationAction from './com/atproto/admin/takeModerationAction' +import emitModerationEvent from './com/atproto/admin/emitModerationEvent' import searchRepos from './com/atproto/admin/searchRepos' import adminGetRecord from './com/atproto/admin/getRecord' import getRepo from './com/atproto/admin/getRepo' -import getModerationAction from './com/atproto/admin/getModerationAction' -import getModerationActions from './com/atproto/admin/getModerationActions' -import getModerationReport from './com/atproto/admin/getModerationReport' -import getModerationReports from './com/atproto/admin/getModerationReports' +import queryModerationStatuses from './com/atproto/admin/queryModerationStatuses' import resolveHandle from './com/atproto/identity/resolveHandle' import getRecord from './com/atproto/repo/getRecord' +import queryModerationEvents from './com/atproto/admin/queryModerationEvents' +import getModerationEvent from './com/atproto/admin/getModerationEvent' import fetchLabels from './com/atproto/temp/fetchLabels' export * as health from './health' @@ -105,16 +102,13 @@ export default function (server: Server, ctx: AppContext) { getTimelineSkeleton(server, ctx) // com.atproto createReport(server, ctx) - resolveModerationReports(server, ctx) - reverseModerationAction(server, ctx) - takeModerationAction(server, ctx) + emitModerationEvent(server, ctx) searchRepos(server, ctx) adminGetRecord(server, ctx) getRepo(server, ctx) - getModerationAction(server, ctx) - getModerationActions(server, ctx) - getModerationReport(server, ctx) - getModerationReports(server, ctx) + getModerationEvent(server, ctx) + queryModerationEvents(server, ctx) + queryModerationStatuses(server, ctx) resolveHandle(server, ctx) getRecord(server, ctx) fetchLabels(server, ctx) diff --git a/packages/bsky/src/auto-moderator/index.ts b/packages/bsky/src/auto-moderator/index.ts index 7118b95ac62..8925314808c 100644 --- a/packages/bsky/src/auto-moderator/index.ts +++ b/packages/bsky/src/auto-moderator/index.ts @@ -61,7 +61,6 @@ export class AutoModerator { 'moderation service not properly configured', ) } - this.imgLabeler = hiveApiKey ? new HiveLabeler(hiveApiKey, ctx) : undefined this.textLabeler = new KeywordLabeler(ctx.cfg.labelerKeywords) if (abyssEndpoint && abyssPassword) { @@ -157,18 +156,22 @@ export class AutoModerator { if (!this.textFlagger) return const matches = this.textFlagger.getMatches(text) if (matches.length < 1) return - if (!this.services.moderation) { - log.error( - { subject, text, matches }, - 'no moderation service setup to flag record text', - ) - return - } - await this.services.moderation(this.ctx.db).report({ - reasonType: REASONOTHER, - reason: `Automatically flagged for possible slurs: ${matches.join(', ')}`, - subject, - reportedBy: this.ctx.cfg.labelerDid, + await this.ctx.db.transaction(async (dbTxn) => { + if (!this.services.moderation) { + log.error( + { subject, text, matches }, + 'no moderation service setup to flag record text', + ) + return + } + return this.services.moderation(dbTxn).report({ + reasonType: REASONOTHER, + reason: `Automatically flagged for possible slurs: ${matches.join( + ', ', + )}`, + subject, + reportedBy: this.ctx.cfg.labelerDid, + }) }) } @@ -244,15 +247,17 @@ export class AutoModerator { } if (this.pushAgent) { - await this.pushAgent.com.atproto.admin.takeModerationAction({ - action: 'com.atproto.admin.defs#takedown', + await this.pushAgent.com.atproto.admin.emitModerationEvent({ + event: { + $type: 'com.atproto.admin.defs#modEventTakedown', + comment: takedownReason, + }, subject: { $type: 'com.atproto.repo.strongRef', uri: uri.toString(), cid: recordCid.toString(), }, subjectBlobCids: takedownCids.map((c) => c.toString()), - reason: takedownReason, createdBy: this.ctx.cfg.labelerDid, }) } else { @@ -261,11 +266,13 @@ export class AutoModerator { throw new Error('no mod push agent or uri invalidator setup') } const modSrvc = this.services.moderation(dbTxn) - const action = await modSrvc.logAction({ - action: 'com.atproto.admin.defs#takedown', + const action = await modSrvc.logEvent({ + event: { + $type: 'com.atproto.admin.defs#modEventTakedown', + comment: takedownReason, + }, subject: { uri, cid: recordCid }, subjectBlobCids: takedownCids, - reason: takedownReason, createdBy: this.ctx.cfg.labelerDid, }) await modSrvc.takedownRecord({ diff --git a/packages/bsky/src/db/migrations/20231003T202833377Z-create-moderation-subject-status.ts b/packages/bsky/src/db/migrations/20231003T202833377Z-create-moderation-subject-status.ts new file mode 100644 index 00000000000..5419233804e --- /dev/null +++ b/packages/bsky/src/db/migrations/20231003T202833377Z-create-moderation-subject-status.ts @@ -0,0 +1,123 @@ +import { Kysely } from 'kysely' + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('moderation_event') + .addColumn('id', 'serial', (col) => col.primaryKey()) + .addColumn('action', 'varchar', (col) => col.notNull()) + .addColumn('subjectType', 'varchar', (col) => col.notNull()) + .addColumn('subjectDid', 'varchar', (col) => col.notNull()) + .addColumn('subjectUri', 'varchar') + .addColumn('subjectCid', 'varchar') + .addColumn('comment', 'text') + .addColumn('meta', 'jsonb') + .addColumn('createdAt', 'varchar', (col) => col.notNull()) + .addColumn('createdBy', 'varchar', (col) => col.notNull()) + .addColumn('reversedAt', 'varchar') + .addColumn('reversedBy', 'varchar') + .addColumn('durationInHours', 'integer') + .addColumn('expiresAt', 'varchar') + .addColumn('reversedReason', 'text') + .addColumn('createLabelVals', 'varchar') + .addColumn('negateLabelVals', 'varchar') + .addColumn('legacyRefId', 'integer') + .execute() + await db.schema + .createTable('moderation_subject_status') + .addColumn('id', 'serial', (col) => col.primaryKey()) + + // Identifiers + .addColumn('did', 'varchar', (col) => col.notNull()) + // Default to '' so that we can apply unique constraints on did and recordPath columns + .addColumn('recordPath', 'varchar', (col) => col.notNull().defaultTo('')) + .addColumn('blobCids', 'jsonb') + .addColumn('recordCid', 'varchar') + + // human review team state + .addColumn('reviewState', 'varchar', (col) => col.notNull()) + .addColumn('comment', 'varchar') + .addColumn('muteUntil', 'varchar') + .addColumn('lastReviewedAt', 'varchar') + .addColumn('lastReviewedBy', 'varchar') + + // report state + .addColumn('lastReportedAt', 'varchar') + + // visibility/intervention state + .addColumn('takendown', 'boolean', (col) => col.defaultTo(false).notNull()) + .addColumn('suspendUntil', 'varchar') + + // timestamps + .addColumn('createdAt', 'varchar', (col) => col.notNull()) + .addColumn('updatedAt', 'varchar', (col) => col.notNull()) + .addUniqueConstraint('moderation_status_unique_idx', ['did', 'recordPath']) + .execute() + + await db.schema + .createIndex('moderation_subject_status_blob_cids_idx') + .on('moderation_subject_status') + .using('gin') + .column('blobCids') + .execute() + + // Move foreign keys from moderation_action to moderation_event + await db.schema + .alterTable('record') + .dropConstraint('record_takedown_id_fkey') + .execute() + await db.schema + .alterTable('actor') + .dropConstraint('actor_takedown_id_fkey') + .execute() + await db.schema + .alterTable('actor') + .addForeignKeyConstraint( + 'actor_takedown_id_fkey', + ['takedownId'], + 'moderation_event', + ['id'], + ) + .execute() + await db.schema + .alterTable('record') + .addForeignKeyConstraint( + 'record_takedown_id_fkey', + ['takedownId'], + 'moderation_event', + ['id'], + ) + .execute() +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable('moderation_event').execute() + await db.schema.dropTable('moderation_subject_status').execute() + + // Revert foreign key constraints + await db.schema + .alterTable('record') + .dropConstraint('record_takedown_id_fkey') + .execute() + await db.schema + .alterTable('actor') + .dropConstraint('actor_takedown_id_fkey') + .execute() + await db.schema + .alterTable('actor') + .addForeignKeyConstraint( + 'actor_takedown_id_fkey', + ['takedownId'], + 'moderation_action', + ['id'], + ) + .execute() + await db.schema + .alterTable('record') + .addForeignKeyConstraint( + 'record_takedown_id_fkey', + ['takedownId'], + 'moderation_action', + ['id'], + ) + .execute() +} diff --git a/packages/bsky/src/db/migrations/index.ts b/packages/bsky/src/db/migrations/index.ts index 630d4385e1d..da86bfdc669 100644 --- a/packages/bsky/src/db/migrations/index.ts +++ b/packages/bsky/src/db/migrations/index.ts @@ -30,3 +30,4 @@ export * as _20230904T211011773Z from './20230904T211011773Z-block-lists' export * as _20230906T222220386Z from './20230906T222220386Z-thread-gating' export * as _20230920T213858047Z from './20230920T213858047Z-add-tags-to-post' export * as _20230929T192920807Z from './20230929T192920807Z-record-cursor-indexes' +export * as _20231003T202833377Z from './20231003T202833377Z-create-moderation-subject-status' diff --git a/packages/bsky/src/db/pagination.ts b/packages/bsky/src/db/pagination.ts index bd498360ca2..b38c69e5ada 100644 --- a/packages/bsky/src/db/pagination.ts +++ b/packages/bsky/src/db/pagination.ts @@ -117,13 +117,36 @@ export const paginate = < direction?: 'asc' | 'desc' keyset: K tryIndex?: boolean + // By default, pg does nullsFirst + nullsLast?: boolean }, ): QB => { - const { limit, cursor, keyset, direction = 'desc', tryIndex } = opts + const { + limit, + cursor, + keyset, + direction = 'desc', + tryIndex, + nullsLast, + } = opts const keysetSql = keyset.getSql(keyset.unpack(cursor), direction, tryIndex) return qb .if(!!limit, (q) => q.limit(limit as number)) - .orderBy(keyset.primary, direction) - .orderBy(keyset.secondary, direction) + .if(!nullsLast, (q) => + q.orderBy(keyset.primary, direction).orderBy(keyset.secondary, direction), + ) + .if(!!nullsLast, (q) => + q + .orderBy( + direction === 'asc' + ? sql`${keyset.primary} asc nulls last` + : sql`${keyset.primary} desc nulls last`, + ) + .orderBy( + direction === 'asc' + ? sql`${keyset.secondary} asc nulls last` + : sql`${keyset.secondary} desc nulls last`, + ), + ) .if(!!keysetSql, (qb) => (keysetSql ? qb.where(keysetSql) : qb)) as QB } diff --git a/packages/bsky/src/db/periodic-moderation-action-reversal.ts b/packages/bsky/src/db/periodic-moderation-event-reversal.ts similarity index 54% rename from packages/bsky/src/db/periodic-moderation-action-reversal.ts rename to packages/bsky/src/db/periodic-moderation-event-reversal.ts index c148c408efe..9937c113d59 100644 --- a/packages/bsky/src/db/periodic-moderation-action-reversal.ts +++ b/packages/bsky/src/db/periodic-moderation-event-reversal.ts @@ -2,13 +2,15 @@ import { wait } from '@atproto/common' import { Leader } from './leader' import { dbLogger } from '../logger' import AppContext from '../context' +import { AtUri } from '@atproto/api' +import { ModerationSubjectStatusRow } from '../services/moderation/types' +import { CID } from 'multiformats/cid' import AtpAgent from '@atproto/api' -import { LabelService } from '../services/label' -import { ModerationActionRow } from '../services/moderation' +import { retryHttp } from '../util/retry' export const MODERATION_ACTION_REVERSAL_ID = 1011 -export class PeriodicModerationActionReversal { +export class PeriodicModerationEventReversal { leader = new Leader( MODERATION_ACTION_REVERSAL_ID, this.appContext.db.getPrimary(), @@ -20,48 +22,50 @@ export class PeriodicModerationActionReversal { this.pushAgent = appContext.moderationPushAgent } - // invert label creation & negations - async reverseLabels(labelTxn: LabelService, actionRow: ModerationActionRow) { - let uri: string - let cid: string | null = null - - if (actionRow.subjectUri && actionRow.subjectCid) { - uri = actionRow.subjectUri - cid = actionRow.subjectCid - } else { - uri = actionRow.subjectDid - } - - await labelTxn.formatAndCreate(this.appContext.cfg.labelerDid, uri, cid, { - create: actionRow.negateLabelVals - ? actionRow.negateLabelVals.split(' ') - : undefined, - negate: actionRow.createLabelVals - ? actionRow.createLabelVals.split(' ') - : undefined, - }) - } - - async revertAction(actionRow: ModerationActionRow) { - const reverseAction = { - id: actionRow.id, - createdBy: actionRow.createdBy, - createdAt: new Date(), - reason: `[SCHEDULED_REVERSAL] Reverting action as originally scheduled`, - } - - if (this.pushAgent) { - await this.pushAgent.com.atproto.admin.reverseModerationAction( - reverseAction, - ) - return - } - + async revertState(eventRow: ModerationSubjectStatusRow) { await this.appContext.db.getPrimary().transaction(async (dbTxn) => { const moderationTxn = this.appContext.services.moderation(dbTxn) - await moderationTxn.revertAction(reverseAction) - const labelTxn = this.appContext.services.label(dbTxn) - await this.reverseLabels(labelTxn, actionRow) + const originalEvent = + await moderationTxn.getLastReversibleEventForSubject(eventRow) + if (originalEvent) { + const { restored } = await moderationTxn.revertState({ + action: originalEvent.action, + createdBy: originalEvent.createdBy, + comment: + '[SCHEDULED_REVERSAL] Reverting action as originally scheduled', + subject: + eventRow.recordPath && eventRow.recordCid + ? { + uri: AtUri.make( + eventRow.did, + ...eventRow.recordPath.split('/'), + ), + cid: CID.parse(eventRow.recordCid), + } + : { did: eventRow.did }, + createdAt: new Date(), + }) + + const { pushAgent } = this + if ( + originalEvent.action === 'com.atproto.admin.defs#modEventTakedown' && + restored?.subjects?.length && + pushAgent + ) { + await Promise.allSettled( + restored.subjects.map((subject) => + retryHttp(() => + pushAgent.api.com.atproto.admin.updateSubjectStatus({ + subject, + takedown: { + applied: false, + }, + }), + ), + ), + ) + } + } }) } @@ -69,12 +73,12 @@ export class PeriodicModerationActionReversal { const moderationService = this.appContext.services.moderation( this.appContext.db.getPrimary(), ) - const actionsDueForReversal = - await moderationService.getActionsDueForReversal() + const subjectsDueForReversal = + await moderationService.getSubjectsDueForReversal() // We shouldn't have too many actions due for reversal at any given time, so running in parallel is probably fine // Internally, each reversal runs within its own transaction - await Promise.all(actionsDueForReversal.map(this.revertAction.bind(this))) + await Promise.all(subjectsDueForReversal.map(this.revertState.bind(this))) } async run() { diff --git a/packages/bsky/src/db/tables/moderation.ts b/packages/bsky/src/db/tables/moderation.ts index ef2bd3b5f6c..f1ac3572785 100644 --- a/packages/bsky/src/db/tables/moderation.ts +++ b/packages/bsky/src/db/tables/moderation.ts @@ -1,76 +1,59 @@ import { Generated } from 'kysely' import { - ACKNOWLEDGE, - FLAG, - TAKEDOWN, - ESCALATE, + REVIEWCLOSED, + REVIEWOPEN, + REVIEWESCALATED, } from '../../lexicon/types/com/atproto/admin/defs' -import { - REASONOTHER, - REASONSPAM, - REASONMISLEADING, - REASONRUDE, - REASONSEXUAL, - REASONVIOLATION, -} from '../../lexicon/types/com/atproto/moderation/defs' -export const actionTableName = 'moderation_action' -export const actionSubjectBlobTableName = 'moderation_action_subject_blob' -export const reportTableName = 'moderation_report' -export const reportResolutionTableName = 'moderation_report_resolution' +export const eventTableName = 'moderation_event' +export const subjectStatusTableName = 'moderation_subject_status' -export interface ModerationAction { +export interface ModerationEvent { id: Generated - action: typeof TAKEDOWN | typeof FLAG | typeof ACKNOWLEDGE | typeof ESCALATE + action: + | 'com.atproto.admin.defs#modEventTakedown' + | 'com.atproto.admin.defs#modEventAcknowledge' + | 'com.atproto.admin.defs#modEventEscalate' + | 'com.atproto.admin.defs#modEventComment' + | 'com.atproto.admin.defs#modEventLabel' + | 'com.atproto.admin.defs#modEventReport' + | 'com.atproto.admin.defs#modEventMute' + | 'com.atproto.admin.defs#modEventReverseTakedown' + | 'com.atproto.admin.defs#modEventEmail' subjectType: 'com.atproto.admin.defs#repoRef' | 'com.atproto.repo.strongRef' subjectDid: string subjectUri: string | null subjectCid: string | null createLabelVals: string | null negateLabelVals: string | null - reason: string + comment: string | null createdAt: string createdBy: string - reversedAt: string | null - reversedBy: string | null - reversedReason: string | null durationInHours: number | null expiresAt: string | null + meta: Record | null + legacyRefId: number | null } -export interface ModerationActionSubjectBlob { - actionId: number - cid: string -} - -export interface ModerationReport { +export interface ModerationSubjectStatus { id: Generated - subjectType: 'com.atproto.admin.defs#repoRef' | 'com.atproto.repo.strongRef' - subjectDid: string - subjectUri: string | null - subjectCid: string | null - reasonType: - | typeof REASONSPAM - | typeof REASONOTHER - | typeof REASONMISLEADING - | typeof REASONRUDE - | typeof REASONSEXUAL - | typeof REASONVIOLATION - reason: string | null - reportedByDid: string + did: string + recordPath: string + recordCid: string | null + blobCids: string[] | null + reviewState: typeof REVIEWCLOSED | typeof REVIEWOPEN | typeof REVIEWESCALATED createdAt: string -} - -export interface ModerationReportResolution { - reportId: number - actionId: number - createdAt: string - createdBy: string + updatedAt: string + lastReviewedBy: string | null + lastReviewedAt: string | null + lastReportedAt: string | null + muteUntil: string | null + suspendUntil: string | null + takendown: boolean + comment: string | null } export type PartialDB = { - [actionTableName]: ModerationAction - [actionSubjectBlobTableName]: ModerationActionSubjectBlob - [reportTableName]: ModerationReport - [reportResolutionTableName]: ModerationReportResolution + [eventTableName]: ModerationEvent + [subjectStatusTableName]: ModerationSubjectStatus } diff --git a/packages/bsky/src/index.ts b/packages/bsky/src/index.ts index 938d634356c..9e0075dce37 100644 --- a/packages/bsky/src/index.ts +++ b/packages/bsky/src/index.ts @@ -32,13 +32,14 @@ export type { ServerConfigValues } from './config' export type { MountedAlgos } from './feed-gen/types' export { ServerConfig } from './config' export { Database, PrimaryDatabase, DatabaseCoordinator } from './db' -export { PeriodicModerationActionReversal } from './db/periodic-moderation-action-reversal' +export { PeriodicModerationEventReversal } from './db/periodic-moderation-event-reversal' export { Redis } from './redis' export { ViewMaintainer } from './db/views' export { AppContext } from './context' export { makeAlgos } from './feed-gen' export * from './indexer' export * from './ingester' +export { MigrateModerationData } from './migrate-moderation-data' export class BskyAppView { public ctx: AppContext diff --git a/packages/bsky/src/lexicon/index.ts b/packages/bsky/src/lexicon/index.ts index 6474220edad..e4a075bee9f 100644 --- a/packages/bsky/src/lexicon/index.ts +++ b/packages/bsky/src/lexicon/index.ts @@ -11,21 +11,18 @@ import { import { schemas } from './lexicons' import * as ComAtprotoAdminDisableAccountInvites from './types/com/atproto/admin/disableAccountInvites' import * as ComAtprotoAdminDisableInviteCodes from './types/com/atproto/admin/disableInviteCodes' +import * as ComAtprotoAdminEmitModerationEvent from './types/com/atproto/admin/emitModerationEvent' import * as ComAtprotoAdminEnableAccountInvites from './types/com/atproto/admin/enableAccountInvites' import * as ComAtprotoAdminGetAccountInfo from './types/com/atproto/admin/getAccountInfo' import * as ComAtprotoAdminGetInviteCodes from './types/com/atproto/admin/getInviteCodes' -import * as ComAtprotoAdminGetModerationAction from './types/com/atproto/admin/getModerationAction' -import * as ComAtprotoAdminGetModerationActions from './types/com/atproto/admin/getModerationActions' -import * as ComAtprotoAdminGetModerationReport from './types/com/atproto/admin/getModerationReport' -import * as ComAtprotoAdminGetModerationReports from './types/com/atproto/admin/getModerationReports' +import * as ComAtprotoAdminGetModerationEvent from './types/com/atproto/admin/getModerationEvent' import * as ComAtprotoAdminGetRecord from './types/com/atproto/admin/getRecord' import * as ComAtprotoAdminGetRepo from './types/com/atproto/admin/getRepo' import * as ComAtprotoAdminGetSubjectStatus from './types/com/atproto/admin/getSubjectStatus' -import * as ComAtprotoAdminResolveModerationReports from './types/com/atproto/admin/resolveModerationReports' -import * as ComAtprotoAdminReverseModerationAction from './types/com/atproto/admin/reverseModerationAction' +import * as ComAtprotoAdminQueryModerationEvents from './types/com/atproto/admin/queryModerationEvents' +import * as ComAtprotoAdminQueryModerationStatuses from './types/com/atproto/admin/queryModerationStatuses' import * as ComAtprotoAdminSearchRepos from './types/com/atproto/admin/searchRepos' import * as ComAtprotoAdminSendEmail from './types/com/atproto/admin/sendEmail' -import * as ComAtprotoAdminTakeModerationAction from './types/com/atproto/admin/takeModerationAction' import * as ComAtprotoAdminUpdateAccountEmail from './types/com/atproto/admin/updateAccountEmail' import * as ComAtprotoAdminUpdateAccountHandle from './types/com/atproto/admin/updateAccountHandle' import * as ComAtprotoAdminUpdateSubjectStatus from './types/com/atproto/admin/updateSubjectStatus' @@ -123,10 +120,9 @@ import * as AppBskyUnspeccedSearchActorsSkeleton from './types/app/bsky/unspecce import * as AppBskyUnspeccedSearchPostsSkeleton from './types/app/bsky/unspecced/searchPostsSkeleton' export const COM_ATPROTO_ADMIN = { - DefsTakedown: 'com.atproto.admin.defs#takedown', - DefsFlag: 'com.atproto.admin.defs#flag', - DefsAcknowledge: 'com.atproto.admin.defs#acknowledge', - DefsEscalate: 'com.atproto.admin.defs#escalate', + DefsReviewOpen: 'com.atproto.admin.defs#reviewOpen', + DefsReviewEscalated: 'com.atproto.admin.defs#reviewEscalated', + DefsReviewClosed: 'com.atproto.admin.defs#reviewClosed', } export const COM_ATPROTO_MODERATION = { DefsReasonSpam: 'com.atproto.moderation.defs#reasonSpam', @@ -220,6 +216,17 @@ export class AdminNS { return this._server.xrpc.method(nsid, cfg) } + emitModerationEvent( + cfg: ConfigOf< + AV, + ComAtprotoAdminEmitModerationEvent.Handler>, + ComAtprotoAdminEmitModerationEvent.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.admin.emitModerationEvent' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } + enableAccountInvites( cfg: ConfigOf< AV, @@ -253,47 +260,14 @@ export class AdminNS { return this._server.xrpc.method(nsid, cfg) } - getModerationAction( - cfg: ConfigOf< - AV, - ComAtprotoAdminGetModerationAction.Handler>, - ComAtprotoAdminGetModerationAction.HandlerReqCtx> - >, - ) { - const nsid = 'com.atproto.admin.getModerationAction' // @ts-ignore - return this._server.xrpc.method(nsid, cfg) - } - - getModerationActions( + getModerationEvent( cfg: ConfigOf< AV, - ComAtprotoAdminGetModerationActions.Handler>, - ComAtprotoAdminGetModerationActions.HandlerReqCtx> + ComAtprotoAdminGetModerationEvent.Handler>, + ComAtprotoAdminGetModerationEvent.HandlerReqCtx> >, ) { - const nsid = 'com.atproto.admin.getModerationActions' // @ts-ignore - return this._server.xrpc.method(nsid, cfg) - } - - getModerationReport( - cfg: ConfigOf< - AV, - ComAtprotoAdminGetModerationReport.Handler>, - ComAtprotoAdminGetModerationReport.HandlerReqCtx> - >, - ) { - const nsid = 'com.atproto.admin.getModerationReport' // @ts-ignore - return this._server.xrpc.method(nsid, cfg) - } - - getModerationReports( - cfg: ConfigOf< - AV, - ComAtprotoAdminGetModerationReports.Handler>, - ComAtprotoAdminGetModerationReports.HandlerReqCtx> - >, - ) { - const nsid = 'com.atproto.admin.getModerationReports' // @ts-ignore + const nsid = 'com.atproto.admin.getModerationEvent' // @ts-ignore return this._server.xrpc.method(nsid, cfg) } @@ -330,25 +304,25 @@ export class AdminNS { return this._server.xrpc.method(nsid, cfg) } - resolveModerationReports( + queryModerationEvents( cfg: ConfigOf< AV, - ComAtprotoAdminResolveModerationReports.Handler>, - ComAtprotoAdminResolveModerationReports.HandlerReqCtx> + ComAtprotoAdminQueryModerationEvents.Handler>, + ComAtprotoAdminQueryModerationEvents.HandlerReqCtx> >, ) { - const nsid = 'com.atproto.admin.resolveModerationReports' // @ts-ignore + const nsid = 'com.atproto.admin.queryModerationEvents' // @ts-ignore return this._server.xrpc.method(nsid, cfg) } - reverseModerationAction( + queryModerationStatuses( cfg: ConfigOf< AV, - ComAtprotoAdminReverseModerationAction.Handler>, - ComAtprotoAdminReverseModerationAction.HandlerReqCtx> + ComAtprotoAdminQueryModerationStatuses.Handler>, + ComAtprotoAdminQueryModerationStatuses.HandlerReqCtx> >, ) { - const nsid = 'com.atproto.admin.reverseModerationAction' // @ts-ignore + const nsid = 'com.atproto.admin.queryModerationStatuses' // @ts-ignore return this._server.xrpc.method(nsid, cfg) } @@ -374,17 +348,6 @@ export class AdminNS { return this._server.xrpc.method(nsid, cfg) } - takeModerationAction( - cfg: ConfigOf< - AV, - ComAtprotoAdminTakeModerationAction.Handler>, - ComAtprotoAdminTakeModerationAction.HandlerReqCtx> - >, - ) { - const nsid = 'com.atproto.admin.takeModerationAction' // @ts-ignore - return this._server.xrpc.method(nsid, cfg) - } - updateAccountEmail( cfg: ConfigOf< AV, diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index cc3f09f0c4e..25a60f90054 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -20,30 +20,33 @@ export const schemaDict = { }, }, }, - actionView: { + modEventView: { type: 'object', required: [ 'id', - 'action', + 'event', 'subject', 'subjectBlobCids', - 'reason', 'createdBy', 'createdAt', - 'resolvedReportIds', ], properties: { id: { type: 'integer', }, - action: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionType', - }, - durationInHours: { - type: 'integer', - description: - 'Indicates how long this action is meant to be in effect before automatically expiring.', + event: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#modEventTakedown', + 'lex:com.atproto.admin.defs#modEventReverseTakedown', + 'lex:com.atproto.admin.defs#modEventComment', + 'lex:com.atproto.admin.defs#modEventReport', + 'lex:com.atproto.admin.defs#modEventLabel', + 'lex:com.atproto.admin.defs#modEventAcknowledge', + 'lex:com.atproto.admin.defs#modEventEscalate', + 'lex:com.atproto.admin.defs#modEventMute', + 'lex:com.atproto.admin.defs#modEventEmail', + ], }, subject: { type: 'union', @@ -58,21 +61,6 @@ export const schemaDict = { type: 'string', }, }, - createLabelVals: { - type: 'array', - items: { - type: 'string', - }, - }, - negateLabelVals: { - type: 'array', - items: { - type: 'string', - }, - }, - reason: { - type: 'string', - }, createdBy: { type: 'string', format: 'did', @@ -81,42 +69,40 @@ export const schemaDict = { type: 'string', format: 'datetime', }, - reversal: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionReversal', + creatorHandle: { + type: 'string', }, - resolvedReportIds: { - type: 'array', - items: { - type: 'integer', - }, + subjectHandle: { + type: 'string', }, }, }, - actionViewDetail: { + modEventViewDetail: { type: 'object', required: [ 'id', - 'action', + 'event', 'subject', 'subjectBlobs', - 'reason', 'createdBy', 'createdAt', - 'resolvedReports', ], properties: { id: { type: 'integer', }, - action: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionType', - }, - durationInHours: { - type: 'integer', - description: - 'Indicates how long this action is meant to be in effect before automatically expiring.', + event: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#modEventTakedown', + 'lex:com.atproto.admin.defs#modEventReverseTakedown', + 'lex:com.atproto.admin.defs#modEventComment', + 'lex:com.atproto.admin.defs#modEventReport', + 'lex:com.atproto.admin.defs#modEventLabel', + 'lex:com.atproto.admin.defs#modEventAcknowledge', + 'lex:com.atproto.admin.defs#modEventEscalate', + 'lex:com.atproto.admin.defs#modEventMute', + ], }, subject: { type: 'union', @@ -134,67 +120,6 @@ export const schemaDict = { ref: 'lex:com.atproto.admin.defs#blobView', }, }, - createLabelVals: { - type: 'array', - items: { - type: 'string', - }, - }, - negateLabelVals: { - type: 'array', - items: { - type: 'string', - }, - }, - reason: { - type: 'string', - }, - createdBy: { - type: 'string', - format: 'did', - }, - createdAt: { - type: 'string', - format: 'datetime', - }, - reversal: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionReversal', - }, - resolvedReports: { - type: 'array', - items: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#reportView', - }, - }, - }, - }, - actionViewCurrent: { - type: 'object', - required: ['id', 'action'], - properties: { - id: { - type: 'integer', - }, - action: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionType', - }, - durationInHours: { - type: 'integer', - description: - 'Indicates how long this action is meant to be in effect before automatically expiring.', - }, - }, - }, - actionReversal: { - type: 'object', - required: ['reason', 'createdBy', 'createdAt'], - properties: { - reason: { - type: 'string', - }, createdBy: { type: 'string', format: 'did', @@ -205,35 +130,6 @@ export const schemaDict = { }, }, }, - actionType: { - type: 'string', - knownValues: [ - 'lex:com.atproto.admin.defs#takedown', - 'lex:com.atproto.admin.defs#flag', - 'lex:com.atproto.admin.defs#acknowledge', - 'lex:com.atproto.admin.defs#escalate', - ], - }, - takedown: { - type: 'token', - description: - 'Moderation action type: Takedown. Indicates that content should not be served by the PDS.', - }, - flag: { - type: 'token', - description: - 'Moderation action type: Flag. Indicates that the content was reviewed and considered to violate PDS rules, but may still be served.', - }, - acknowledge: { - type: 'token', - description: - 'Moderation action type: Acknowledge. Indicates that the content was reviewed and not considered to violate PDS rules.', - }, - escalate: { - type: 'token', - description: - 'Moderation action type: Escalate. Indicates that the content has been flagged for additional review.', - }, reportView: { type: 'object', required: [ @@ -252,7 +148,7 @@ export const schemaDict = { type: 'ref', ref: 'lex:com.atproto.moderation.defs#reasonType', }, - reason: { + comment: { type: 'string', }, subjectRepoHandle: { @@ -281,6 +177,75 @@ export const schemaDict = { }, }, }, + subjectStatusView: { + type: 'object', + required: ['id', 'subject', 'createdAt', 'updatedAt', 'reviewState'], + properties: { + id: { + type: 'integer', + }, + subject: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#repoRef', + 'lex:com.atproto.repo.strongRef', + ], + }, + subjectBlobCids: { + type: 'array', + items: { + type: 'string', + format: 'cid', + }, + }, + subjectRepoHandle: { + type: 'string', + }, + updatedAt: { + type: 'string', + format: 'datetime', + description: + 'Timestamp referencing when the last update was made to the moderation status of the subject', + }, + createdAt: { + type: 'string', + format: 'datetime', + description: + 'Timestamp referencing the first moderation status impacting event was emitted on the subject', + }, + reviewState: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#subjectReviewState', + }, + comment: { + type: 'string', + description: 'Sticky comment on the subject.', + }, + muteUntil: { + type: 'string', + format: 'datetime', + }, + lastReviewedBy: { + type: 'string', + format: 'did', + }, + lastReviewedAt: { + type: 'string', + format: 'datetime', + }, + lastReportedAt: { + type: 'string', + format: 'datetime', + }, + takendown: { + type: 'boolean', + }, + suspendUntil: { + type: 'string', + format: 'datetime', + }, + }, + }, reportViewDetail: { type: 'object', required: [ @@ -299,7 +264,7 @@ export const schemaDict = { type: 'ref', ref: 'lex:com.atproto.moderation.defs#reasonType', }, - reason: { + comment: { type: 'string', }, subject: { @@ -311,6 +276,10 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#recordViewNotFound', ], }, + subjectStatus: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#subjectStatusView', + }, reportedBy: { type: 'string', format: 'did', @@ -323,7 +292,7 @@ export const schemaDict = { type: 'array', items: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionView', + ref: 'lex:com.atproto.admin.defs#modEventView', }, }, }, @@ -628,33 +597,18 @@ export const schemaDict = { moderation: { type: 'object', properties: { - currentAction: { + subjectStatus: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionViewCurrent', + ref: 'lex:com.atproto.admin.defs#subjectStatusView', }, }, }, moderationDetail: { type: 'object', - required: ['actions', 'reports'], properties: { - currentAction: { + subjectStatus: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionViewCurrent', - }, - actions: { - type: 'array', - items: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionView', - }, - }, - reports: { - type: 'array', - items: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#reportView', - }, + ref: 'lex:com.atproto.admin.defs#subjectStatusView', }, }, }, @@ -716,74 +670,164 @@ export const schemaDict = { }, }, }, - }, - }, - ComAtprotoAdminDisableAccountInvites: { - lexicon: 1, - id: 'com.atproto.admin.disableAccountInvites', - defs: { - main: { - type: 'procedure', + subjectReviewState: { + type: 'string', + knownValues: [ + 'lex:com.atproto.admin.defs#reviewOpen', + 'lex:com.atproto.admin.defs#reviewEscalated', + 'lex:com.atproto.admin.defs#reviewClosed', + ], + }, + reviewOpen: { + type: 'token', description: - 'Disable an account from receiving new invite codes, but does not invalidate existing codes.', - input: { - encoding: 'application/json', - schema: { - type: 'object', - required: ['account'], - properties: { - account: { - type: 'string', - format: 'did', - }, - note: { - type: 'string', - description: 'Optional reason for disabled invites.', - }, - }, + 'Moderator review status of a subject: Open. Indicates that the subject needs to be reviewed by a moderator', + }, + reviewEscalated: { + type: 'token', + description: + 'Moderator review status of a subject: Escalated. Indicates that the subject was escalated for review by a moderator', + }, + reviewClosed: { + type: 'token', + description: + 'Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator', + }, + modEventTakedown: { + type: 'object', + description: 'Take down a subject permanently or temporarily', + properties: { + comment: { + type: 'string', + }, + durationInHours: { + type: 'integer', + description: + 'Indicates how long the takedown should be in effect before automatically expiring.', }, }, }, - }, - }, - ComAtprotoAdminDisableInviteCodes: { - lexicon: 1, - id: 'com.atproto.admin.disableInviteCodes', - defs: { - main: { - type: 'procedure', - description: - 'Disable some set of codes and/or all codes associated with a set of users.', - input: { - encoding: 'application/json', - schema: { - type: 'object', - properties: { - codes: { - type: 'array', - items: { - type: 'string', - }, - }, - accounts: { - type: 'array', - items: { - type: 'string', - }, - }, - }, + modEventReverseTakedown: { + type: 'object', + description: 'Revert take down action on a subject', + properties: { + comment: { + type: 'string', + description: 'Describe reasoning behind the reversal.', }, }, }, - }, - }, - ComAtprotoAdminEnableAccountInvites: { - lexicon: 1, - id: 'com.atproto.admin.enableAccountInvites', + modEventComment: { + type: 'object', + description: 'Add a comment to a subject', + required: ['comment'], + properties: { + comment: { + type: 'string', + }, + sticky: { + type: 'boolean', + description: 'Make the comment persistent on the subject', + }, + }, + }, + modEventReport: { + type: 'object', + description: 'Report a subject', + required: ['reportType'], + properties: { + comment: { + type: 'string', + }, + reportType: { + type: 'ref', + ref: 'lex:com.atproto.moderation.defs#reasonType', + }, + }, + }, + modEventLabel: { + type: 'object', + description: 'Apply/Negate labels on a subject', + required: ['createLabelVals', 'negateLabelVals'], + properties: { + comment: { + type: 'string', + }, + createLabelVals: { + type: 'array', + items: { + type: 'string', + }, + }, + negateLabelVals: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + modEventAcknowledge: { + type: 'object', + properties: { + comment: { + type: 'string', + }, + }, + }, + modEventEscalate: { + type: 'object', + properties: { + comment: { + type: 'string', + }, + }, + }, + modEventMute: { + type: 'object', + description: 'Mute incoming reports on a subject', + required: ['durationInHours'], + properties: { + comment: { + type: 'string', + }, + durationInHours: { + type: 'integer', + description: 'Indicates how long the subject should remain muted.', + }, + }, + }, + modEventUnmute: { + type: 'object', + description: 'Unmute action on a subject', + properties: { + comment: { + type: 'string', + description: 'Describe reasoning behind the reversal.', + }, + }, + }, + modEventEmail: { + type: 'object', + description: 'Keep a log of outgoing email to a user', + required: ['subjectLine'], + properties: { + subjectLine: { + type: 'string', + description: 'The subject line of the email sent to the user.', + }, + }, + }, + }, + }, + ComAtprotoAdminDisableAccountInvites: { + lexicon: 1, + id: 'com.atproto.admin.disableAccountInvites', defs: { main: { type: 'procedure', - description: "Re-enable an account's ability to receive invite codes.", + description: + 'Disable an account from receiving new invite codes, but does not invalidate existing codes.', input: { encoding: 'application/json', schema: { @@ -796,7 +840,7 @@ export const schemaDict = { }, note: { type: 'string', - description: 'Optional reason for enabled invites.', + description: 'Optional reason for disabled invites.', }, }, }, @@ -804,20 +848,83 @@ export const schemaDict = { }, }, }, - ComAtprotoAdminGetAccountInfo: { + ComAtprotoAdminDisableInviteCodes: { lexicon: 1, - id: 'com.atproto.admin.getAccountInfo', + id: 'com.atproto.admin.disableInviteCodes', defs: { main: { - type: 'query', - description: 'Get details about an account.', - parameters: { - type: 'params', - required: ['did'], - properties: { - did: { - type: 'string', - format: 'did', + type: 'procedure', + description: + 'Disable some set of codes and/or all codes associated with a set of users.', + input: { + encoding: 'application/json', + schema: { + type: 'object', + properties: { + codes: { + type: 'array', + items: { + type: 'string', + }, + }, + accounts: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + }, + }, + }, + }, + ComAtprotoAdminEmitModerationEvent: { + lexicon: 1, + id: 'com.atproto.admin.emitModerationEvent', + defs: { + main: { + type: 'procedure', + description: 'Take a moderation action on an actor.', + input: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['event', 'subject', 'createdBy'], + properties: { + event: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#modEventTakedown', + 'lex:com.atproto.admin.defs#modEventAcknowledge', + 'lex:com.atproto.admin.defs#modEventEscalate', + 'lex:com.atproto.admin.defs#modEventComment', + 'lex:com.atproto.admin.defs#modEventLabel', + 'lex:com.atproto.admin.defs#modEventReport', + 'lex:com.atproto.admin.defs#modEventMute', + 'lex:com.atproto.admin.defs#modEventReverseTakedown', + 'lex:com.atproto.admin.defs#modEventUnmute', + 'lex:com.atproto.admin.defs#modEventEmail', + ], + }, + subject: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#repoRef', + 'lex:com.atproto.repo.strongRef', + ], + }, + subjectBlobCids: { + type: 'array', + items: { + type: 'string', + format: 'cid', + }, + }, + createdBy: { + type: 'string', + format: 'did', + }, }, }, }, @@ -825,53 +932,37 @@ export const schemaDict = { encoding: 'application/json', schema: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#accountView', + ref: 'lex:com.atproto.admin.defs#modEventView', }, }, + errors: [ + { + name: 'SubjectHasAction', + }, + ], }, }, }, - ComAtprotoAdminGetInviteCodes: { + ComAtprotoAdminEnableAccountInvites: { lexicon: 1, - id: 'com.atproto.admin.getInviteCodes', + id: 'com.atproto.admin.enableAccountInvites', defs: { main: { - type: 'query', - description: 'Get an admin view of invite codes.', - parameters: { - type: 'params', - properties: { - sort: { - type: 'string', - knownValues: ['recent', 'usage'], - default: 'recent', - }, - limit: { - type: 'integer', - minimum: 1, - maximum: 500, - default: 100, - }, - cursor: { - type: 'string', - }, - }, - }, - output: { + type: 'procedure', + description: "Re-enable an account's ability to receive invite codes.", + input: { encoding: 'application/json', schema: { type: 'object', - required: ['codes'], + required: ['account'], properties: { - cursor: { + account: { type: 'string', + format: 'did', }, - codes: { - type: 'array', - items: { - type: 'ref', - ref: 'lex:com.atproto.server.defs#inviteCode', - }, + note: { + type: 'string', + description: 'Optional reason for enabled invites.', }, }, }, @@ -879,19 +970,20 @@ export const schemaDict = { }, }, }, - ComAtprotoAdminGetModerationAction: { + ComAtprotoAdminGetAccountInfo: { lexicon: 1, - id: 'com.atproto.admin.getModerationAction', + id: 'com.atproto.admin.getAccountInfo', defs: { main: { type: 'query', - description: 'Get details about a moderation action.', + description: 'Get details about an account.', parameters: { type: 'params', - required: ['id'], + required: ['did'], properties: { - id: { - type: 'integer', + did: { + type: 'string', + format: 'did', }, }, }, @@ -899,30 +991,32 @@ export const schemaDict = { encoding: 'application/json', schema: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionViewDetail', + ref: 'lex:com.atproto.admin.defs#accountView', }, }, }, }, }, - ComAtprotoAdminGetModerationActions: { + ComAtprotoAdminGetInviteCodes: { lexicon: 1, - id: 'com.atproto.admin.getModerationActions', + id: 'com.atproto.admin.getInviteCodes', defs: { main: { type: 'query', - description: 'Get a list of moderation actions related to a subject.', + description: 'Get an admin view of invite codes.', parameters: { type: 'params', properties: { - subject: { + sort: { type: 'string', + knownValues: ['recent', 'usage'], + default: 'recent', }, limit: { type: 'integer', minimum: 1, - maximum: 100, - default: 50, + maximum: 500, + default: 100, }, cursor: { type: 'string', @@ -933,16 +1027,16 @@ export const schemaDict = { encoding: 'application/json', schema: { type: 'object', - required: ['actions'], + required: ['codes'], properties: { cursor: { type: 'string', }, - actions: { + codes: { type: 'array', items: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionView', + ref: 'lex:com.atproto.server.defs#inviteCode', }, }, }, @@ -951,13 +1045,13 @@ export const schemaDict = { }, }, }, - ComAtprotoAdminGetModerationReport: { + ComAtprotoAdminGetModerationEvent: { lexicon: 1, - id: 'com.atproto.admin.getModerationReport', + id: 'com.atproto.admin.getModerationEvent', defs: { main: { type: 'query', - description: 'Get details about a moderation report.', + description: 'Get details about a moderation event.', parameters: { type: 'params', required: ['id'], @@ -971,89 +1065,7 @@ export const schemaDict = { encoding: 'application/json', schema: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#reportViewDetail', - }, - }, - }, - }, - }, - ComAtprotoAdminGetModerationReports: { - lexicon: 1, - id: 'com.atproto.admin.getModerationReports', - defs: { - main: { - type: 'query', - description: 'Get moderation reports related to a subject.', - parameters: { - type: 'params', - properties: { - subject: { - type: 'string', - }, - ignoreSubjects: { - type: 'array', - items: { - type: 'string', - }, - }, - actionedBy: { - type: 'string', - format: 'did', - description: - 'Get all reports that were actioned by a specific moderator.', - }, - reporters: { - type: 'array', - items: { - type: 'string', - }, - description: 'Filter reports made by one or more DIDs.', - }, - resolved: { - type: 'boolean', - }, - actionType: { - type: 'string', - knownValues: [ - 'com.atproto.admin.defs#takedown', - 'com.atproto.admin.defs#flag', - 'com.atproto.admin.defs#acknowledge', - 'com.atproto.admin.defs#escalate', - ], - }, - limit: { - type: 'integer', - minimum: 1, - maximum: 100, - default: 50, - }, - cursor: { - type: 'string', - }, - reverse: { - type: 'boolean', - description: - 'Reverse the order of the returned records. When true, returns reports in chronological order.', - }, - }, - }, - output: { - encoding: 'application/json', - schema: { - type: 'object', - required: ['reports'], - properties: { - cursor: { - type: 'string', - }, - reports: { - type: 'array', - items: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#reportView', - }, - }, - }, + ref: 'lex:com.atproto.admin.defs#modEventViewDetail', }, }, }, @@ -1176,76 +1188,180 @@ export const schemaDict = { }, }, }, - ComAtprotoAdminResolveModerationReports: { + ComAtprotoAdminQueryModerationEvents: { lexicon: 1, - id: 'com.atproto.admin.resolveModerationReports', + id: 'com.atproto.admin.queryModerationEvents', defs: { main: { - type: 'procedure', - description: 'Resolve moderation reports by an action.', - input: { + type: 'query', + description: 'List moderation events related to a subject.', + parameters: { + type: 'params', + properties: { + types: { + type: 'array', + items: { + type: 'string', + }, + description: + 'The types of events (fully qualified string in the format of com.atproto.admin#modEvent) to filter by. If not specified, all events are returned.', + }, + createdBy: { + type: 'string', + format: 'did', + }, + sortDirection: { + type: 'string', + default: 'desc', + enum: ['asc', 'desc'], + description: + 'Sort direction for the events. Defaults to descending order of created at timestamp.', + }, + subject: { + type: 'string', + format: 'uri', + }, + includeAllUserRecords: { + type: 'boolean', + default: false, + description: + 'If true, events on all record types (posts, lists, profile etc.) owned by the did are returned', + }, + limit: { + type: 'integer', + minimum: 1, + maximum: 100, + default: 50, + }, + cursor: { + type: 'string', + }, + }, + }, + output: { encoding: 'application/json', schema: { type: 'object', - required: ['actionId', 'reportIds', 'createdBy'], + required: ['events'], properties: { - actionId: { - type: 'integer', + cursor: { + type: 'string', }, - reportIds: { + events: { type: 'array', items: { - type: 'integer', + type: 'ref', + ref: 'lex:com.atproto.admin.defs#modEventView', }, }, - createdBy: { - type: 'string', - format: 'did', - }, }, }, }, - output: { - encoding: 'application/json', - schema: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionView', - }, - }, }, }, }, - ComAtprotoAdminReverseModerationAction: { + ComAtprotoAdminQueryModerationStatuses: { lexicon: 1, - id: 'com.atproto.admin.reverseModerationAction', + id: 'com.atproto.admin.queryModerationStatuses', defs: { main: { - type: 'procedure', - description: 'Reverse a moderation action.', - input: { - encoding: 'application/json', - schema: { - type: 'object', - required: ['id', 'reason', 'createdBy'], - properties: { - id: { - type: 'integer', - }, - reason: { - type: 'string', - }, - createdBy: { + type: 'query', + description: 'View moderation statuses of subjects (record or repo).', + parameters: { + type: 'params', + properties: { + subject: { + type: 'string', + format: 'uri', + }, + comment: { + type: 'string', + description: 'Search subjects by keyword from comments', + }, + reportedAfter: { + type: 'string', + format: 'datetime', + description: 'Search subjects reported after a given timestamp', + }, + reportedBefore: { + type: 'string', + format: 'datetime', + description: 'Search subjects reported before a given timestamp', + }, + reviewedAfter: { + type: 'string', + format: 'datetime', + description: 'Search subjects reviewed after a given timestamp', + }, + reviewedBefore: { + type: 'string', + format: 'datetime', + description: 'Search subjects reviewed before a given timestamp', + }, + includeMuted: { + type: 'boolean', + description: + "By default, we don't include muted subjects in the results. Set this to true to include them.", + }, + reviewState: { + type: 'string', + description: 'Specify when fetching subjects in a certain state', + }, + ignoreSubjects: { + type: 'array', + items: { type: 'string', - format: 'did', + format: 'uri', }, }, + lastReviewedBy: { + type: 'string', + format: 'did', + description: + 'Get all subject statuses that were reviewed by a specific moderator', + }, + sortField: { + type: 'string', + default: 'lastReportedAt', + enum: ['lastReviewedAt', 'lastReportedAt'], + }, + sortDirection: { + type: 'string', + default: 'desc', + enum: ['asc', 'desc'], + }, + takendown: { + type: 'boolean', + description: 'Get subjects that were taken down', + }, + limit: { + type: 'integer', + minimum: 1, + maximum: 100, + default: 50, + }, + cursor: { + type: 'string', + }, }, }, output: { encoding: 'application/json', schema: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionView', + type: 'object', + required: ['subjectStatuses'], + properties: { + cursor: { + type: 'string', + }, + subjectStatuses: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#subjectStatusView', + }, + }, + }, }, }, }, @@ -1312,7 +1428,7 @@ export const schemaDict = { encoding: 'application/json', schema: { type: 'object', - required: ['recipientDid', 'content'], + required: ['recipientDid', 'content', 'senderDid'], properties: { recipientDid: { type: 'string', @@ -1324,6 +1440,10 @@ export const schemaDict = { subject: { type: 'string', }, + senderDid: { + type: 'string', + format: 'did', + }, }, }, }, @@ -1342,83 +1462,6 @@ export const schemaDict = { }, }, }, - ComAtprotoAdminTakeModerationAction: { - lexicon: 1, - id: 'com.atproto.admin.takeModerationAction', - defs: { - main: { - type: 'procedure', - description: 'Take a moderation action on an actor.', - input: { - encoding: 'application/json', - schema: { - type: 'object', - required: ['action', 'subject', 'reason', 'createdBy'], - properties: { - action: { - type: 'string', - knownValues: [ - 'com.atproto.admin.defs#takedown', - 'com.atproto.admin.defs#flag', - 'com.atproto.admin.defs#acknowledge', - ], - }, - subject: { - type: 'union', - refs: [ - 'lex:com.atproto.admin.defs#repoRef', - 'lex:com.atproto.repo.strongRef', - ], - }, - subjectBlobCids: { - type: 'array', - items: { - type: 'string', - format: 'cid', - }, - }, - createLabelVals: { - type: 'array', - items: { - type: 'string', - }, - }, - negateLabelVals: { - type: 'array', - items: { - type: 'string', - }, - }, - reason: { - type: 'string', - }, - durationInHours: { - type: 'integer', - description: - 'Indicates how long this action is meant to be in effect before automatically expiring.', - }, - createdBy: { - type: 'string', - format: 'did', - }, - }, - }, - }, - output: { - encoding: 'application/json', - schema: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionView', - }, - }, - errors: [ - { - name: 'SubjectHasAction', - }, - ], - }, - }, - }, ComAtprotoAdminUpdateAccountEmail: { lexicon: 1, id: 'com.atproto.admin.updateAccountEmail', @@ -7627,23 +7670,20 @@ export const ids = { ComAtprotoAdminDisableAccountInvites: 'com.atproto.admin.disableAccountInvites', ComAtprotoAdminDisableInviteCodes: 'com.atproto.admin.disableInviteCodes', + ComAtprotoAdminEmitModerationEvent: 'com.atproto.admin.emitModerationEvent', ComAtprotoAdminEnableAccountInvites: 'com.atproto.admin.enableAccountInvites', ComAtprotoAdminGetAccountInfo: 'com.atproto.admin.getAccountInfo', ComAtprotoAdminGetInviteCodes: 'com.atproto.admin.getInviteCodes', - ComAtprotoAdminGetModerationAction: 'com.atproto.admin.getModerationAction', - ComAtprotoAdminGetModerationActions: 'com.atproto.admin.getModerationActions', - ComAtprotoAdminGetModerationReport: 'com.atproto.admin.getModerationReport', - ComAtprotoAdminGetModerationReports: 'com.atproto.admin.getModerationReports', + ComAtprotoAdminGetModerationEvent: 'com.atproto.admin.getModerationEvent', ComAtprotoAdminGetRecord: 'com.atproto.admin.getRecord', ComAtprotoAdminGetRepo: 'com.atproto.admin.getRepo', ComAtprotoAdminGetSubjectStatus: 'com.atproto.admin.getSubjectStatus', - ComAtprotoAdminResolveModerationReports: - 'com.atproto.admin.resolveModerationReports', - ComAtprotoAdminReverseModerationAction: - 'com.atproto.admin.reverseModerationAction', + ComAtprotoAdminQueryModerationEvents: + 'com.atproto.admin.queryModerationEvents', + ComAtprotoAdminQueryModerationStatuses: + 'com.atproto.admin.queryModerationStatuses', ComAtprotoAdminSearchRepos: 'com.atproto.admin.searchRepos', ComAtprotoAdminSendEmail: 'com.atproto.admin.sendEmail', - ComAtprotoAdminTakeModerationAction: 'com.atproto.admin.takeModerationAction', ComAtprotoAdminUpdateAccountEmail: 'com.atproto.admin.updateAccountEmail', ComAtprotoAdminUpdateAccountHandle: 'com.atproto.admin.updateAccountHandle', ComAtprotoAdminUpdateSubjectStatus: 'com.atproto.admin.updateSubjectStatus', diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts index 8a21c42119e..27f080cbe31 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts @@ -28,43 +28,55 @@ export function validateStatusAttr(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#statusAttr', v) } -export interface ActionView { +export interface ModEventView { id: number - action: ActionType - /** Indicates how long this action is meant to be in effect before automatically expiring. */ - durationInHours?: number + event: + | ModEventTakedown + | ModEventReverseTakedown + | ModEventComment + | ModEventReport + | ModEventLabel + | ModEventAcknowledge + | ModEventEscalate + | ModEventMute + | ModEventEmail + | { $type: string; [k: string]: unknown } subject: | RepoRef | ComAtprotoRepoStrongRef.Main | { $type: string; [k: string]: unknown } subjectBlobCids: string[] - createLabelVals?: string[] - negateLabelVals?: string[] - reason: string createdBy: string createdAt: string - reversal?: ActionReversal - resolvedReportIds: number[] + creatorHandle?: string + subjectHandle?: string [k: string]: unknown } -export function isActionView(v: unknown): v is ActionView { +export function isModEventView(v: unknown): v is ModEventView { return ( isObj(v) && hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#actionView' + v.$type === 'com.atproto.admin.defs#modEventView' ) } -export function validateActionView(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#actionView', v) +export function validateModEventView(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventView', v) } -export interface ActionViewDetail { +export interface ModEventViewDetail { id: number - action: ActionType - /** Indicates how long this action is meant to be in effect before automatically expiring. */ - durationInHours?: number + event: + | ModEventTakedown + | ModEventReverseTakedown + | ModEventComment + | ModEventReport + | ModEventLabel + | ModEventAcknowledge + | ModEventEscalate + | ModEventMute + | { $type: string; [k: string]: unknown } subject: | RepoView | RepoViewNotFound @@ -72,123 +84,100 @@ export interface ActionViewDetail { | RecordViewNotFound | { $type: string; [k: string]: unknown } subjectBlobs: BlobView[] - createLabelVals?: string[] - negateLabelVals?: string[] - reason: string createdBy: string createdAt: string - reversal?: ActionReversal - resolvedReports: ReportView[] [k: string]: unknown } -export function isActionViewDetail(v: unknown): v is ActionViewDetail { +export function isModEventViewDetail(v: unknown): v is ModEventViewDetail { return ( isObj(v) && hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#actionViewDetail' + v.$type === 'com.atproto.admin.defs#modEventViewDetail' ) } -export function validateActionViewDetail(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#actionViewDetail', v) +export function validateModEventViewDetail(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventViewDetail', v) } -export interface ActionViewCurrent { +export interface ReportView { id: number - action: ActionType - /** Indicates how long this action is meant to be in effect before automatically expiring. */ - durationInHours?: number - [k: string]: unknown -} - -export function isActionViewCurrent(v: unknown): v is ActionViewCurrent { - return ( - isObj(v) && - hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#actionViewCurrent' - ) -} - -export function validateActionViewCurrent(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#actionViewCurrent', v) -} - -export interface ActionReversal { - reason: string - createdBy: string + reasonType: ComAtprotoModerationDefs.ReasonType + comment?: string + subjectRepoHandle?: string + subject: + | RepoRef + | ComAtprotoRepoStrongRef.Main + | { $type: string; [k: string]: unknown } + reportedBy: string createdAt: string + resolvedByActionIds: number[] [k: string]: unknown } -export function isActionReversal(v: unknown): v is ActionReversal { +export function isReportView(v: unknown): v is ReportView { return ( isObj(v) && hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#actionReversal' + v.$type === 'com.atproto.admin.defs#reportView' ) } -export function validateActionReversal(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#actionReversal', v) +export function validateReportView(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#reportView', v) } -export type ActionType = - | 'lex:com.atproto.admin.defs#takedown' - | 'lex:com.atproto.admin.defs#flag' - | 'lex:com.atproto.admin.defs#acknowledge' - | 'lex:com.atproto.admin.defs#escalate' - | (string & {}) - -/** Moderation action type: Takedown. Indicates that content should not be served by the PDS. */ -export const TAKEDOWN = 'com.atproto.admin.defs#takedown' -/** Moderation action type: Flag. Indicates that the content was reviewed and considered to violate PDS rules, but may still be served. */ -export const FLAG = 'com.atproto.admin.defs#flag' -/** Moderation action type: Acknowledge. Indicates that the content was reviewed and not considered to violate PDS rules. */ -export const ACKNOWLEDGE = 'com.atproto.admin.defs#acknowledge' -/** Moderation action type: Escalate. Indicates that the content has been flagged for additional review. */ -export const ESCALATE = 'com.atproto.admin.defs#escalate' - -export interface ReportView { +export interface SubjectStatusView { id: number - reasonType: ComAtprotoModerationDefs.ReasonType - reason?: string - subjectRepoHandle?: string subject: | RepoRef | ComAtprotoRepoStrongRef.Main | { $type: string; [k: string]: unknown } - reportedBy: string + subjectBlobCids?: string[] + subjectRepoHandle?: string + /** Timestamp referencing when the last update was made to the moderation status of the subject */ + updatedAt: string + /** Timestamp referencing the first moderation status impacting event was emitted on the subject */ createdAt: string - resolvedByActionIds: number[] + reviewState: SubjectReviewState + /** Sticky comment on the subject. */ + comment?: string + muteUntil?: string + lastReviewedBy?: string + lastReviewedAt?: string + lastReportedAt?: string + takendown?: boolean + suspendUntil?: string [k: string]: unknown } -export function isReportView(v: unknown): v is ReportView { +export function isSubjectStatusView(v: unknown): v is SubjectStatusView { return ( isObj(v) && hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#reportView' + v.$type === 'com.atproto.admin.defs#subjectStatusView' ) } -export function validateReportView(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#reportView', v) +export function validateSubjectStatusView(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#subjectStatusView', v) } export interface ReportViewDetail { id: number reasonType: ComAtprotoModerationDefs.ReasonType - reason?: string + comment?: string subject: | RepoView | RepoViewNotFound | RecordView | RecordViewNotFound | { $type: string; [k: string]: unknown } + subjectStatus?: SubjectStatusView reportedBy: string createdAt: string - resolvedByActions: ActionView[] + resolvedByActions: ModEventView[] [k: string]: unknown } @@ -400,7 +389,7 @@ export function validateRecordViewNotFound(v: unknown): ValidationResult { } export interface Moderation { - currentAction?: ActionViewCurrent + subjectStatus?: SubjectStatusView [k: string]: unknown } @@ -417,9 +406,7 @@ export function validateModeration(v: unknown): ValidationResult { } export interface ModerationDetail { - currentAction?: ActionViewCurrent - actions: ActionView[] - reports: ReportView[] + subjectStatus?: SubjectStatusView [k: string]: unknown } @@ -496,3 +483,208 @@ export function isVideoDetails(v: unknown): v is VideoDetails { export function validateVideoDetails(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#videoDetails', v) } + +export type SubjectReviewState = + | 'lex:com.atproto.admin.defs#reviewOpen' + | 'lex:com.atproto.admin.defs#reviewEscalated' + | 'lex:com.atproto.admin.defs#reviewClosed' + | (string & {}) + +/** Moderator review status of a subject: Open. Indicates that the subject needs to be reviewed by a moderator */ +export const REVIEWOPEN = 'com.atproto.admin.defs#reviewOpen' +/** Moderator review status of a subject: Escalated. Indicates that the subject was escalated for review by a moderator */ +export const REVIEWESCALATED = 'com.atproto.admin.defs#reviewEscalated' +/** Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator */ +export const REVIEWCLOSED = 'com.atproto.admin.defs#reviewClosed' + +/** Take down a subject permanently or temporarily */ +export interface ModEventTakedown { + comment?: string + /** Indicates how long the takedown should be in effect before automatically expiring. */ + durationInHours?: number + [k: string]: unknown +} + +export function isModEventTakedown(v: unknown): v is ModEventTakedown { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventTakedown' + ) +} + +export function validateModEventTakedown(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventTakedown', v) +} + +/** Revert take down action on a subject */ +export interface ModEventReverseTakedown { + /** Describe reasoning behind the reversal. */ + comment?: string + [k: string]: unknown +} + +export function isModEventReverseTakedown( + v: unknown, +): v is ModEventReverseTakedown { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventReverseTakedown' + ) +} + +export function validateModEventReverseTakedown(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventReverseTakedown', v) +} + +/** Add a comment to a subject */ +export interface ModEventComment { + comment: string + /** Make the comment persistent on the subject */ + sticky?: boolean + [k: string]: unknown +} + +export function isModEventComment(v: unknown): v is ModEventComment { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventComment' + ) +} + +export function validateModEventComment(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventComment', v) +} + +/** Report a subject */ +export interface ModEventReport { + comment?: string + reportType: ComAtprotoModerationDefs.ReasonType + [k: string]: unknown +} + +export function isModEventReport(v: unknown): v is ModEventReport { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventReport' + ) +} + +export function validateModEventReport(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventReport', v) +} + +/** Apply/Negate labels on a subject */ +export interface ModEventLabel { + comment?: string + createLabelVals: string[] + negateLabelVals: string[] + [k: string]: unknown +} + +export function isModEventLabel(v: unknown): v is ModEventLabel { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventLabel' + ) +} + +export function validateModEventLabel(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventLabel', v) +} + +export interface ModEventAcknowledge { + comment?: string + [k: string]: unknown +} + +export function isModEventAcknowledge(v: unknown): v is ModEventAcknowledge { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventAcknowledge' + ) +} + +export function validateModEventAcknowledge(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventAcknowledge', v) +} + +export interface ModEventEscalate { + comment?: string + [k: string]: unknown +} + +export function isModEventEscalate(v: unknown): v is ModEventEscalate { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventEscalate' + ) +} + +export function validateModEventEscalate(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventEscalate', v) +} + +/** Mute incoming reports on a subject */ +export interface ModEventMute { + comment?: string + /** Indicates how long the subject should remain muted. */ + durationInHours: number + [k: string]: unknown +} + +export function isModEventMute(v: unknown): v is ModEventMute { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventMute' + ) +} + +export function validateModEventMute(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventMute', v) +} + +/** Unmute action on a subject */ +export interface ModEventUnmute { + /** Describe reasoning behind the reversal. */ + comment?: string + [k: string]: unknown +} + +export function isModEventUnmute(v: unknown): v is ModEventUnmute { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventUnmute' + ) +} + +export function validateModEventUnmute(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventUnmute', v) +} + +/** Keep a log of outgoing email to a user */ +export interface ModEventEmail { + /** The subject line of the email sent to the user. */ + subjectLine: string + [k: string]: unknown +} + +export function isModEventEmail(v: unknown): v is ModEventEmail { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventEmail' + ) +} + +export function validateModEventEmail(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventEmail', v) +} diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/takeModerationAction.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/emitModerationEvent.ts similarity index 71% rename from packages/pds/src/lexicon/types/com/atproto/admin/takeModerationAction.ts rename to packages/bsky/src/lexicon/types/com/atproto/admin/emitModerationEvent.ts index 33877d90d11..df44702b51c 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/takeModerationAction.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/emitModerationEvent.ts @@ -13,26 +13,28 @@ import * as ComAtprotoRepoStrongRef from '../repo/strongRef' export interface QueryParams {} export interface InputSchema { - action: - | 'com.atproto.admin.defs#takedown' - | 'com.atproto.admin.defs#flag' - | 'com.atproto.admin.defs#acknowledge' - | (string & {}) + event: + | ComAtprotoAdminDefs.ModEventTakedown + | ComAtprotoAdminDefs.ModEventAcknowledge + | ComAtprotoAdminDefs.ModEventEscalate + | ComAtprotoAdminDefs.ModEventComment + | ComAtprotoAdminDefs.ModEventLabel + | ComAtprotoAdminDefs.ModEventReport + | ComAtprotoAdminDefs.ModEventMute + | ComAtprotoAdminDefs.ModEventReverseTakedown + | ComAtprotoAdminDefs.ModEventUnmute + | ComAtprotoAdminDefs.ModEventEmail + | { $type: string; [k: string]: unknown } subject: | ComAtprotoAdminDefs.RepoRef | ComAtprotoRepoStrongRef.Main | { $type: string; [k: string]: unknown } subjectBlobCids?: string[] - createLabelVals?: string[] - negateLabelVals?: string[] - reason: string - /** Indicates how long this action is meant to be in effect before automatically expiring. */ - durationInHours?: number createdBy: string [k: string]: unknown } -export type OutputSchema = ComAtprotoAdminDefs.ActionView +export type OutputSchema = ComAtprotoAdminDefs.ModEventView export interface HandlerInput { encoding: 'application/json' diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/getModerationAction.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/getModerationEvent.ts similarity index 94% rename from packages/bsky/src/lexicon/types/com/atproto/admin/getModerationAction.ts rename to packages/bsky/src/lexicon/types/com/atproto/admin/getModerationEvent.ts index 2ab52f237cc..7de567a73db 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/getModerationAction.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/getModerationEvent.ts @@ -14,7 +14,7 @@ export interface QueryParams { } export type InputSchema = undefined -export type OutputSchema = ComAtprotoAdminDefs.ActionViewDetail +export type OutputSchema = ComAtprotoAdminDefs.ModEventViewDetail export type HandlerInput = undefined export interface HandlerSuccess { diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/getModerationReport.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/getModerationReport.ts deleted file mode 100644 index 28d714453f2..00000000000 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/getModerationReport.ts +++ /dev/null @@ -1,41 +0,0 @@ -/** - * GENERATED CODE - DO NOT MODIFY - */ -import express from 'express' -import { ValidationResult, BlobRef } from '@atproto/lexicon' -import { lexicons } from '../../../../lexicons' -import { isObj, hasProp } from '../../../../util' -import { CID } from 'multiformats/cid' -import { HandlerAuth } from '@atproto/xrpc-server' -import * as ComAtprotoAdminDefs from './defs' - -export interface QueryParams { - id: number -} - -export type InputSchema = undefined -export type OutputSchema = ComAtprotoAdminDefs.ReportViewDetail -export type HandlerInput = undefined - -export interface HandlerSuccess { - encoding: 'application/json' - body: OutputSchema - headers?: { [key: string]: string } -} - -export interface HandlerError { - status: number - message?: string -} - -export type HandlerOutput = HandlerError | HandlerSuccess -export type HandlerReqCtx = { - auth: HA - params: QueryParams - input: HandlerInput - req: express.Request - res: express.Response -} -export type Handler = ( - ctx: HandlerReqCtx, -) => Promise | HandlerOutput diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/getModerationActions.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/queryModerationEvents.ts similarity index 69% rename from packages/bsky/src/lexicon/types/com/atproto/admin/getModerationActions.ts rename to packages/bsky/src/lexicon/types/com/atproto/admin/queryModerationEvents.ts index 4c29f965df6..f3c4f1fbb95 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/getModerationActions.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/queryModerationEvents.ts @@ -10,7 +10,14 @@ import { HandlerAuth } from '@atproto/xrpc-server' import * as ComAtprotoAdminDefs from './defs' export interface QueryParams { + /** The types of events (fully qualified string in the format of com.atproto.admin#modEvent) to filter by. If not specified, all events are returned. */ + types?: string[] + createdBy?: string + /** Sort direction for the events. Defaults to descending order of created at timestamp. */ + sortDirection: 'asc' | 'desc' subject?: string + /** If true, events on all record types (posts, lists, profile etc.) owned by the did are returned */ + includeAllUserRecords: boolean limit: number cursor?: string } @@ -19,7 +26,7 @@ export type InputSchema = undefined export interface OutputSchema { cursor?: string - actions: ComAtprotoAdminDefs.ActionView[] + events: ComAtprotoAdminDefs.ModEventView[] [k: string]: unknown } diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/getModerationReports.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/queryModerationStatuses.ts similarity index 56% rename from packages/bsky/src/lexicon/types/com/atproto/admin/getModerationReports.ts rename to packages/bsky/src/lexicon/types/com/atproto/admin/queryModerationStatuses.ts index b80811cf213..d4e55aff386 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/getModerationReports.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/queryModerationStatuses.ts @@ -11,29 +11,36 @@ import * as ComAtprotoAdminDefs from './defs' export interface QueryParams { subject?: string + /** Search subjects by keyword from comments */ + comment?: string + /** Search subjects reported after a given timestamp */ + reportedAfter?: string + /** Search subjects reported before a given timestamp */ + reportedBefore?: string + /** Search subjects reviewed after a given timestamp */ + reviewedAfter?: string + /** Search subjects reviewed before a given timestamp */ + reviewedBefore?: string + /** By default, we don't include muted subjects in the results. Set this to true to include them. */ + includeMuted?: boolean + /** Specify when fetching subjects in a certain state */ + reviewState?: string ignoreSubjects?: string[] - /** Get all reports that were actioned by a specific moderator. */ - actionedBy?: string - /** Filter reports made by one or more DIDs. */ - reporters?: string[] - resolved?: boolean - actionType?: - | 'com.atproto.admin.defs#takedown' - | 'com.atproto.admin.defs#flag' - | 'com.atproto.admin.defs#acknowledge' - | 'com.atproto.admin.defs#escalate' - | (string & {}) + /** Get all subject statuses that were reviewed by a specific moderator */ + lastReviewedBy?: string + sortField: 'lastReviewedAt' | 'lastReportedAt' + sortDirection: 'asc' | 'desc' + /** Get subjects that were taken down */ + takendown?: boolean limit: number cursor?: string - /** Reverse the order of the returned records. When true, returns reports in chronological order. */ - reverse?: boolean } export type InputSchema = undefined export interface OutputSchema { cursor?: string - reports: ComAtprotoAdminDefs.ReportView[] + subjectStatuses: ComAtprotoAdminDefs.SubjectStatusView[] [k: string]: unknown } diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/resolveModerationReports.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/resolveModerationReports.ts deleted file mode 100644 index e3f4d028202..00000000000 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/resolveModerationReports.ts +++ /dev/null @@ -1,49 +0,0 @@ -/** - * GENERATED CODE - DO NOT MODIFY - */ -import express from 'express' -import { ValidationResult, BlobRef } from '@atproto/lexicon' -import { lexicons } from '../../../../lexicons' -import { isObj, hasProp } from '../../../../util' -import { CID } from 'multiformats/cid' -import { HandlerAuth } from '@atproto/xrpc-server' -import * as ComAtprotoAdminDefs from './defs' - -export interface QueryParams {} - -export interface InputSchema { - actionId: number - reportIds: number[] - createdBy: string - [k: string]: unknown -} - -export type OutputSchema = ComAtprotoAdminDefs.ActionView - -export interface HandlerInput { - encoding: 'application/json' - body: InputSchema -} - -export interface HandlerSuccess { - encoding: 'application/json' - body: OutputSchema - headers?: { [key: string]: string } -} - -export interface HandlerError { - status: number - message?: string -} - -export type HandlerOutput = HandlerError | HandlerSuccess -export type HandlerReqCtx = { - auth: HA - params: QueryParams - input: HandlerInput - req: express.Request - res: express.Response -} -export type Handler = ( - ctx: HandlerReqCtx, -) => Promise | HandlerOutput diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/reverseModerationAction.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/reverseModerationAction.ts deleted file mode 100644 index 17dcb5085de..00000000000 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/reverseModerationAction.ts +++ /dev/null @@ -1,49 +0,0 @@ -/** - * GENERATED CODE - DO NOT MODIFY - */ -import express from 'express' -import { ValidationResult, BlobRef } from '@atproto/lexicon' -import { lexicons } from '../../../../lexicons' -import { isObj, hasProp } from '../../../../util' -import { CID } from 'multiformats/cid' -import { HandlerAuth } from '@atproto/xrpc-server' -import * as ComAtprotoAdminDefs from './defs' - -export interface QueryParams {} - -export interface InputSchema { - id: number - reason: string - createdBy: string - [k: string]: unknown -} - -export type OutputSchema = ComAtprotoAdminDefs.ActionView - -export interface HandlerInput { - encoding: 'application/json' - body: InputSchema -} - -export interface HandlerSuccess { - encoding: 'application/json' - body: OutputSchema - headers?: { [key: string]: string } -} - -export interface HandlerError { - status: number - message?: string -} - -export type HandlerOutput = HandlerError | HandlerSuccess -export type HandlerReqCtx = { - auth: HA - params: QueryParams - input: HandlerInput - req: express.Request - res: express.Response -} -export type Handler = ( - ctx: HandlerReqCtx, -) => Promise | HandlerOutput diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/sendEmail.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/sendEmail.ts index 87e7ceec172..91b53d9be81 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/sendEmail.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/sendEmail.ts @@ -14,6 +14,7 @@ export interface InputSchema { recipientDid: string content: string subject?: string + senderDid: string [k: string]: unknown } diff --git a/packages/bsky/src/migrate-moderation-data.ts b/packages/bsky/src/migrate-moderation-data.ts new file mode 100644 index 00000000000..6919358170a --- /dev/null +++ b/packages/bsky/src/migrate-moderation-data.ts @@ -0,0 +1,414 @@ +import { sql } from 'kysely' +import { DatabaseCoordinator, PrimaryDatabase } from './index' +import { adjustModerationSubjectStatus } from './services/moderation/status' +import { ModerationEventRow } from './services/moderation/types' + +type ModerationActionRow = Omit & { + reason: string | null +} + +const getEnv = () => ({ + DB_URL: + process.env.MODERATION_MIGRATION_DB_URL || + 'postgresql://pg:password@127.0.0.1:5433/postgres', + DB_POOL_SIZE: Number(process.env.MODERATION_MIGRATION_DB_POOL_SIZE) || 10, + DB_SCHEMA: process.env.MODERATION_MIGRATION_DB_SCHEMA || 'bsky', +}) + +const countEntries = async (db: PrimaryDatabase) => { + const [allActions, allReports] = await Promise.all([ + db.db + // @ts-ignore + .selectFrom('moderation_action') + // @ts-ignore + .select((eb) => eb.fn.count('id').as('count')) + .executeTakeFirstOrThrow(), + db.db + // @ts-ignore + .selectFrom('moderation_report') + // @ts-ignore + .select((eb) => eb.fn.count('id').as('count')) + .executeTakeFirstOrThrow(), + ]) + + return { reportsCount: allReports.count, actionsCount: allActions.count } +} + +const countEvents = async (db: PrimaryDatabase) => { + const events = await db.db + .selectFrom('moderation_event') + .select((eb) => eb.fn.count('id').as('count')) + .executeTakeFirstOrThrow() + + return events.count +} + +const getLatestReportLegacyRefId = async (db: PrimaryDatabase) => { + const events = await db.db + .selectFrom('moderation_event') + .select((eb) => eb.fn.max('legacyRefId').as('latestLegacyRefId')) + .where('action', '=', 'com.atproto.admin.defs#modEventReport') + .executeTakeFirstOrThrow() + + return events.latestLegacyRefId +} + +const countStatuses = async (db: PrimaryDatabase) => { + const events = await db.db + .selectFrom('moderation_subject_status') + .select((eb) => eb.fn.count('id').as('count')) + .executeTakeFirstOrThrow() + + return events.count +} + +const processLegacyReports = async ( + db: PrimaryDatabase, + legacyIds: number[], +) => { + if (!legacyIds.length) { + console.log('No legacy reports to process') + return + } + const reports = await db.db + .selectFrom('moderation_event') + .where('action', '=', 'com.atproto.admin.defs#modEventReport') + .where('legacyRefId', 'in', legacyIds) + .orderBy('legacyRefId', 'asc') + .selectAll() + .execute() + + console.log(`Processing ${reports.length} reports from ${legacyIds.length}`) + await db.transaction(async (tx) => { + // This will be slow but we need to run this in sequence + for (const report of reports) { + await adjustModerationSubjectStatus(tx, report) + } + }) + console.log(`Completed processing ${reports.length} reports`) +} + +const getReportEventsAboveLegacyId = async ( + db: PrimaryDatabase, + aboveLegacyId: number, +) => { + return await db.db + .selectFrom('moderation_event') + .where('action', '=', 'com.atproto.admin.defs#modEventReport') + .where('legacyRefId', '>', aboveLegacyId) + .select(sql`"legacyRefId"`.as('legacyRefId')) + .execute() +} + +const createEvents = async ( + db: PrimaryDatabase, + opts?: { onlyReportsAboveId: number }, +) => { + const commonColumnsToSelect = [ + 'subjectDid', + 'subjectUri', + 'subjectType', + 'subjectCid', + sql`reason`.as('comment'), + 'createdAt', + ] + const commonColumnsToInsert = [ + 'subjectDid', + 'subjectUri', + 'subjectType', + 'subjectCid', + 'comment', + 'createdAt', + 'action', + 'createdBy', + ] as const + + let totalActions: number + if (!opts?.onlyReportsAboveId) { + await db.db + .insertInto('moderation_event') + .columns([ + 'id', + ...commonColumnsToInsert, + 'createLabelVals', + 'negateLabelVals', + 'durationInHours', + 'expiresAt', + ]) + .expression((eb) => + eb + // @ts-ignore + .selectFrom('moderation_action') + // @ts-ignore + .select([ + 'id', + ...commonColumnsToSelect, + sql`CONCAT('com.atproto.admin.defs#modEvent', UPPER(SUBSTRING(SPLIT_PART(action, '#', 2) FROM 1 FOR 1)), SUBSTRING(SPLIT_PART(action, '#', 2) FROM 2))`.as( + 'action', + ), + 'createdBy', + 'createLabelVals', + 'negateLabelVals', + 'durationInHours', + 'expiresAt', + ]) + .orderBy('id', 'asc'), + ) + .execute() + + totalActions = await countEvents(db) + console.log(`Created ${totalActions} events from actions`) + + await sql`SELECT setval(pg_get_serial_sequence('moderation_event', 'id'), (select max(id) from moderation_event))`.execute( + db.db, + ) + console.log('Reset the id sequence for moderation_event') + } else { + totalActions = await countEvents(db) + } + + await db.db + .insertInto('moderation_event') + .columns([...commonColumnsToInsert, 'meta', 'legacyRefId']) + .expression((eb) => { + const builder = eb + // @ts-ignore + .selectFrom('moderation_report') + // @ts-ignore + .select([ + ...commonColumnsToSelect, + sql`'com.atproto.admin.defs#modEventReport'`.as('action'), + sql`"reportedByDid"`.as('createdBy'), + sql`json_build_object('reportType', "reasonType")`.as('meta'), + sql`id`.as('legacyRefId'), + ]) + + if (opts?.onlyReportsAboveId) { + // @ts-ignore + return builder.where('id', '>', opts.onlyReportsAboveId) + } + + return builder + }) + .execute() + + const totalEvents = await countEvents(db) + console.log(`Created ${totalEvents - totalActions} events from reports`) + + return +} + +const setReportedAtTimestamp = async (db: PrimaryDatabase) => { + console.log('Initiating lastReportedAt timestamp sync') + const didUpdate = await sql` + UPDATE moderation_subject_status + SET "lastReportedAt" = reports."createdAt" + FROM ( + select "subjectDid", "subjectUri", MAX("createdAt") as "createdAt" + from moderation_report + where "subjectUri" is null + group by "subjectDid", "subjectUri" + ) as reports + WHERE reports."subjectDid" = moderation_subject_status."did" + AND "recordPath" = '' + AND ("lastReportedAt" is null OR "lastReportedAt" < reports."createdAt") + `.execute(db.db) + + console.log( + `Updated lastReportedAt for ${didUpdate.numUpdatedOrDeletedRows} did subject`, + ) + + const contentUpdate = await sql` + UPDATE moderation_subject_status + SET "lastReportedAt" = reports."createdAt" + FROM ( + select "subjectDid", "subjectUri", MAX("createdAt") as "createdAt" + from moderation_report + where "subjectUri" is not null + group by "subjectDid", "subjectUri" + ) as reports + WHERE reports."subjectDid" = moderation_subject_status."did" + AND "recordPath" is not null + AND POSITION(moderation_subject_status."recordPath" IN reports."subjectUri") > 0 + AND ("lastReportedAt" is null OR "lastReportedAt" < reports."createdAt") + `.execute(db.db) + + console.log( + `Updated lastReportedAt for ${contentUpdate.numUpdatedOrDeletedRows} subject with uri`, + ) +} + +const createStatusFromActions = async (db: PrimaryDatabase) => { + const allEvents = await db.db + // @ts-ignore + .selectFrom('moderation_action') + // @ts-ignore + .where('reversedAt', 'is', null) + // @ts-ignore + .select((eb) => eb.fn.count('id').as('count')) + .executeTakeFirstOrThrow() + + const chunkSize = 2500 + const totalChunks = Math.ceil(allEvents.count / chunkSize) + + console.log(`Processing ${allEvents.count} actions in ${totalChunks} chunks`) + + await db.transaction(async (tx) => { + // This is not used for pagination but only for logging purposes + let currentChunk = 1 + let lastProcessedId: undefined | number = 0 + do { + const eventsQuery = tx.db + // @ts-ignore + .selectFrom('moderation_action') + // @ts-ignore + .where('reversedAt', 'is', null) + // @ts-ignore + .where('id', '>', lastProcessedId) + .limit(chunkSize) + .selectAll() + const events = (await eventsQuery.execute()) as ModerationActionRow[] + + for (const event of events) { + // Remap action to event data type + const actionParts = event.action.split('#') + await adjustModerationSubjectStatus(tx, { + ...event, + action: `com.atproto.admin.defs#modEvent${actionParts[1] + .charAt(0) + .toUpperCase()}${actionParts[1].slice( + 1, + )}` as ModerationEventRow['action'], + comment: event.reason, + meta: null, + }) + } + + console.log(`Processed events chunk ${currentChunk} of ${totalChunks}`) + lastProcessedId = events.at(-1)?.id + currentChunk++ + } while (lastProcessedId !== undefined) + }) + + console.log(`Events migration complete!`) + + const totalStatuses = await countStatuses(db) + console.log(`Created ${totalStatuses} statuses`) +} + +const remapFlagToAcknlowedge = async (db: PrimaryDatabase) => { + console.log('Initiating flag to ack remap') + const results = await sql` + UPDATE moderation_event + SET "action" = 'com.atproto.admin.defs#modEventAcknowledge' + WHERE action = 'com.atproto.admin.defs#modEventFlag' + `.execute(db.db) + console.log(`Remapped ${results.numUpdatedOrDeletedRows} flag actions to ack`) +} + +const syncBlobCids = async (db: PrimaryDatabase) => { + console.log('Initiating blob cid sync') + const results = await sql` + UPDATE moderation_subject_status + SET "blobCids" = blob_action."cids" + FROM ( + SELECT moderation_action."subjectUri", moderation_action."subjectDid", jsonb_agg(moderation_action_subject_blob."cid") as cids + FROM moderation_action_subject_blob + JOIN moderation_action + ON moderation_action.id = moderation_action_subject_blob."actionId" + WHERE moderation_action."reversedAt" is NULL + GROUP by moderation_action."subjectUri", moderation_action."subjectDid" + ) as blob_action + WHERE did = "subjectDid" AND position("recordPath" IN "subjectUri") > 0 + `.execute(db.db) + console.log(`Updated blob cids on ${results.numUpdatedOrDeletedRows} rows`) +} + +async function updateStatusFromUnresolvedReports(db: PrimaryDatabase) { + const { ref } = db.db.dynamic + const reports = await db.db + // @ts-ignore + .selectFrom('moderation_report') + .whereNotExists((qb) => + qb + .selectFrom('moderation_report_resolution') + .selectAll() + // @ts-ignore + .whereRef('reportId', '=', ref('moderation_report.id')), + ) + .select(sql`moderation_report.id`.as('legacyId')) + .execute() + + console.log('Updating statuses based on unresolved reports') + await processLegacyReports( + db, + reports.map((report) => report.legacyId), + ) + console.log('Completed updating statuses based on unresolved reports') +} + +export async function MigrateModerationData() { + const env = getEnv() + const db = new DatabaseCoordinator({ + schema: env.DB_SCHEMA, + primary: { + url: env.DB_URL, + poolSize: env.DB_POOL_SIZE, + }, + replicas: [], + }) + + const primaryDb = db.getPrimary() + + const [counts, existingEventsCount] = await Promise.all([ + countEntries(primaryDb), + countEvents(primaryDb), + ]) + + // If there are existing events in the moderation_event table, we assume that the migration has already been run + // so we just bring over any new reports since last run + if (existingEventsCount) { + console.log( + `Found ${existingEventsCount} existing events. Migrating ${counts.reportsCount} reports only, ignoring actions`, + ) + const reportMigrationStartedAt = Date.now() + const latestReportLegacyRefId = await getLatestReportLegacyRefId(primaryDb) + + if (latestReportLegacyRefId) { + await createEvents(primaryDb, { + onlyReportsAboveId: latestReportLegacyRefId, + }) + const newReportEvents = await getReportEventsAboveLegacyId( + primaryDb, + latestReportLegacyRefId, + ) + await processLegacyReports( + primaryDb, + newReportEvents.map((evt) => evt.legacyRefId), + ) + await setReportedAtTimestamp(primaryDb) + } else { + console.log('No reports have been migrated into events yet, bailing.') + } + + console.log( + `Time spent: ${(Date.now() - reportMigrationStartedAt) / 1000} seconds`, + ) + console.log('Migration complete!') + return + } + + const totalEntries = counts.actionsCount + counts.reportsCount + console.log(`Migrating ${totalEntries} rows of actions and reports`) + const startedAt = Date.now() + await createEvents(primaryDb) + // Important to run this before creation statuses from actions to ensure that we are not attempting to map flag actions + await remapFlagToAcknlowedge(primaryDb) + await createStatusFromActions(primaryDb) + await updateStatusFromUnresolvedReports(primaryDb) + await setReportedAtTimestamp(primaryDb) + await syncBlobCids(primaryDb) + + console.log(`Time spent: ${(Date.now() - startedAt) / 1000 / 60} minutes`) + console.log('Migration complete!') +} diff --git a/packages/bsky/src/services/moderation/index.ts b/packages/bsky/src/services/moderation/index.ts index e85f1218470..3ba845333d5 100644 --- a/packages/bsky/src/services/moderation/index.ts +++ b/packages/bsky/src/services/moderation/index.ts @@ -1,19 +1,37 @@ -import { Selectable, sql } from 'kysely' import { CID } from 'multiformats/cid' import { AtUri } from '@atproto/syntax' import { InvalidRequestError } from '@atproto/xrpc-server' import { PrimaryDatabase } from '../../db' -import { ModerationAction, ModerationReport } from '../../db/tables/moderation' import { ModerationViews } from './views' import { ImageUriBuilder } from '../../image/uri' +import { Main as StrongRef } from '../../lexicon/types/com/atproto/repo/strongRef' import { ImageInvalidator } from '../../image/invalidator' import { + isModEventComment, + isModEventLabel, + isModEventMute, + isModEventReport, + isModEventTakedown, + isModEventEmail, RepoRef, RepoBlobRef, - TAKEDOWN, } from '../../lexicon/types/com/atproto/admin/defs' -import { Main as StrongRef } from '../../lexicon/types/com/atproto/repo/strongRef' import { addHoursToDate } from '../../util/date' +import { + adjustModerationSubjectStatus, + getStatusIdentifierFromSubject, +} from './status' +import { + ModEventType, + ModerationEventRow, + ModerationEventRowWithHandle, + ModerationSubjectStatusRow, + ReversibleModerationEvent, + SubjectInfo, +} from './types' +import { ModerationEvent } from '../../db/tables/moderation' +import { paginate } from '../../db/pagination' +import { StatusKeyset, TimeIdKeyset } from './pagination' export class ModerationService { constructor( @@ -32,350 +50,311 @@ export class ModerationService { views = new ModerationViews(this.db) - async getAction(id: number): Promise { + async getEvent(id: number): Promise { return await this.db.db - .selectFrom('moderation_action') + .selectFrom('moderation_event') .selectAll() .where('id', '=', id) .executeTakeFirst() } - async getActionOrThrow(id: number): Promise { - const action = await this.getAction(id) - if (!action) throw new InvalidRequestError('Action not found') - return action + async getEventOrThrow(id: number): Promise { + const event = await this.getEvent(id) + if (!event) throw new InvalidRequestError('Moderation event not found') + return event } - async getActions(opts: { + async getEvents(opts: { subject?: string + createdBy?: string limit: number cursor?: string - }): Promise { - const { subject, limit, cursor } = opts - let builder = this.db.db.selectFrom('moderation_action') - if (subject) { - builder = builder.where((qb) => { - return qb - .where('subjectDid', '=', subject) - .orWhere('subjectUri', '=', subject) - }) - } - if (cursor) { - const cursorNumeric = parseInt(cursor, 10) - if (isNaN(cursorNumeric)) { - throw new InvalidRequestError('Malformed cursor') - } - builder = builder.where('id', '<', cursorNumeric) - } - return await builder - .selectAll() - .orderBy('id', 'desc') - .limit(limit) - .execute() - } - - async getReport(id: number): Promise { - return await this.db.db - .selectFrom('moderation_report') - .selectAll() - .where('id', '=', id) - .executeTakeFirst() - } - - async getReports(opts: { - subject?: string - resolved?: boolean - actionType?: string - limit: number - cursor?: string - ignoreSubjects?: string[] - reverse?: boolean - reporters?: string[] - actionedBy?: string - }): Promise { + includeAllUserRecords: boolean + types: ModerationEvent['action'][] + sortDirection?: 'asc' | 'desc' + }): Promise<{ cursor?: string; events: ModerationEventRowWithHandle[] }> { const { subject, - resolved, - actionType, + createdBy, limit, cursor, - ignoreSubjects, - reverse = false, - reporters, - actionedBy, + includeAllUserRecords, + sortDirection = 'desc', + types, } = opts - const { ref } = this.db.db.dynamic - let builder = this.db.db.selectFrom('moderation_report') + let builder = this.db.db + .selectFrom('moderation_event') + .leftJoin( + 'actor as creatorActor', + 'creatorActor.did', + 'moderation_event.createdBy', + ) + .leftJoin( + 'actor as subjectActor', + 'subjectActor.did', + 'moderation_event.subjectDid', + ) if (subject) { builder = builder.where((qb) => { + if (includeAllUserRecords) { + // If subject is an at-uri, we need to extract the DID from the at-uri + // otherwise, subject is probably a DID already + if (subject.startsWith('at://')) { + const uri = new AtUri(subject) + return qb.where('subjectDid', '=', uri.hostname) + } + return qb.where('subjectDid', '=', subject) + } return qb - .where('subjectDid', '=', subject) + .where((subQb) => + subQb + .where('subjectDid', '=', subject) + .where('subjectUri', 'is', null), + ) .orWhere('subjectUri', '=', subject) }) } - - if (ignoreSubjects?.length) { - const ignoreUris: string[] = [] - const ignoreDids: string[] = [] - - ignoreSubjects.forEach((subject) => { - if (subject.startsWith('at://')) { - ignoreUris.push(subject) - } else if (subject.startsWith('did:')) { - ignoreDids.push(subject) + if (types.length) { + builder = builder.where((qb) => { + if (types.length === 1) { + return qb.where('action', '=', types[0]) } - }) - if (ignoreDids.length) { - builder = builder.where('subjectDid', 'not in', ignoreDids) - } - if (ignoreUris.length) { - builder = builder.where((qb) => { - // Without the null condition, postgres will ignore all reports where `subjectUri` is null - // which will make all the account reports be ignored as well - return qb - .where('subjectUri', 'not in', ignoreUris) - .orWhere('subjectUri', 'is', null) - }) - } - } - - if (reporters?.length) { - builder = builder.where('reportedByDid', 'in', reporters) + return qb.where('action', 'in', types) + }) } - - if (resolved !== undefined) { - const resolutionsQuery = this.db.db - .selectFrom('moderation_report_resolution') - .selectAll() - .whereRef( - 'moderation_report_resolution.reportId', - '=', - ref('moderation_report.id'), - ) - builder = resolved - ? builder.whereExists(resolutionsQuery) - : builder.whereNotExists(resolutionsQuery) + if (createdBy) { + builder = builder.where('createdBy', '=', createdBy) } - if (actionType !== undefined || actionedBy !== undefined) { - let resolutionActionsQuery = this.db.db - .selectFrom('moderation_report_resolution') - .innerJoin( - 'moderation_action', - 'moderation_action.id', - 'moderation_report_resolution.actionId', - ) - .whereRef( - 'moderation_report_resolution.reportId', - '=', - ref('moderation_report.id'), - ) - if (actionType) { - resolutionActionsQuery = resolutionActionsQuery - .where('moderation_action.action', '=', sql`${actionType}`) - .where('moderation_action.reversedAt', 'is', null) - } - - if (actionedBy) { - resolutionActionsQuery = resolutionActionsQuery.where( - 'moderation_action.createdBy', - '=', - actionedBy, - ) - } + const { ref } = this.db.db.dynamic + const keyset = new TimeIdKeyset( + ref(`moderation_event.createdAt`), + ref('moderation_event.id'), + ) + const paginatedBuilder = paginate(builder, { + limit, + cursor, + keyset, + direction: sortDirection, + tryIndex: true, + }) - builder = builder.whereExists(resolutionActionsQuery.selectAll()) - } - if (cursor) { - const cursorNumeric = parseInt(cursor, 10) - if (isNaN(cursorNumeric)) { - throw new InvalidRequestError('Malformed cursor') - } - builder = builder.where('id', reverse ? '>' : '<', cursorNumeric) - } - return await builder - .leftJoin('actor', 'actor.did', 'moderation_report.subjectDid') - .selectAll(['moderation_report', 'actor']) - .orderBy('id', reverse ? 'asc' : 'desc') - .limit(limit) + const result = await paginatedBuilder + .selectAll(['moderation_event']) + .select([ + 'subjectActor.handle as subjectHandle', + 'creatorActor.handle as creatorHandle', + ]) .execute() + + return { cursor: keyset.packFromResult(result), events: result } } - async getReportOrThrow(id: number): Promise { - const report = await this.getReport(id) - if (!report) throw new InvalidRequestError('Report not found') - return report + async getReport(id: number): Promise { + return await this.db.db + .selectFrom('moderation_event') + .where('action', '=', 'com.atproto.admin.defs#modEventReport') + .selectAll() + .where('id', '=', id) + .executeTakeFirst() } - async getCurrentActions( + async getCurrentStatus( subject: { did: string } | { uri: AtUri } | { cids: CID[] }, ) { - const { ref } = this.db.db.dynamic - let builder = this.db.db - .selectFrom('moderation_action') - .selectAll() - .where('reversedAt', 'is', null) + let builder = this.db.db.selectFrom('moderation_subject_status').selectAll() if ('did' in subject) { - builder = builder - .where('subjectType', '=', 'com.atproto.admin.defs#repoRef') - .where('subjectDid', '=', subject.did) + builder = builder.where('did', '=', subject.did) } else if ('uri' in subject) { - builder = builder - .where('subjectType', '=', 'com.atproto.repo.strongRef') - .where('subjectUri', '=', subject.uri.toString()) - } else { - const blobsForAction = this.db.db - .selectFrom('moderation_action_subject_blob') - .selectAll() - .whereRef('actionId', '=', ref('moderation_action.id')) - .where( - 'cid', - 'in', - subject.cids.map((cid) => cid.toString()), - ) - builder = builder.whereExists(blobsForAction) + builder = builder.where('recordPath', '=', subject.uri.toString()) } + // TODO: Handle the cid status return await builder.execute() } - async logAction(info: { - action: ModerationActionRow['action'] + buildSubjectInfo( + subject: { did: string } | { uri: AtUri; cid: CID }, + subjectBlobCids?: CID[], + ): SubjectInfo { + if ('did' in subject) { + if (subjectBlobCids?.length) { + throw new InvalidRequestError('Blobs do not apply to repo subjects') + } + // Allowing dids that may not exist: may have been deleted but needs to remain actionable. + return { + subjectType: 'com.atproto.admin.defs#repoRef', + subjectDid: subject.did, + subjectUri: null, + subjectCid: null, + } + } + + // Allowing records/blobs that may not exist: may have been deleted but needs to remain actionable. + return { + subjectType: 'com.atproto.repo.strongRef', + subjectDid: subject.uri.host, + subjectUri: subject.uri.toString(), + subjectCid: subject.cid.toString(), + } + } + + async logEvent(info: { + event: ModEventType subject: { did: string } | { uri: AtUri; cid: CID } subjectBlobCids?: CID[] - reason: string - createLabelVals?: string[] - negateLabelVals?: string[] createdBy: string createdAt?: Date - durationInHours?: number - }): Promise { + }): Promise { this.db.assertTransaction() const { - action, + event, createdBy, - reason, subject, subjectBlobCids, - durationInHours, createdAt = new Date(), } = info + + // Resolve subject info + const subjectInfo = this.buildSubjectInfo(subject, subjectBlobCids) + const createLabelVals = - info.createLabelVals && info.createLabelVals.length > 0 - ? info.createLabelVals.join(' ') + isModEventLabel(event) && event.createLabelVals.length > 0 + ? event.createLabelVals.join(' ') : undefined const negateLabelVals = - info.negateLabelVals && info.negateLabelVals.length > 0 - ? info.negateLabelVals.join(' ') + isModEventLabel(event) && event.negateLabelVals.length > 0 + ? event.negateLabelVals.join(' ') : undefined - // Resolve subject info - let subjectInfo: SubjectInfo - if ('did' in subject) { - // Allowing dids that may not exist: may have been deleted but needs to remain actionable. - subjectInfo = { - subjectType: 'com.atproto.admin.defs#repoRef', - subjectDid: subject.did, - subjectUri: null, - subjectCid: null, - } - if (subjectBlobCids?.length) { - throw new InvalidRequestError('Blobs do not apply to repo subjects') - } - } else { - // Allowing records/blobs that may not exist: may have been deleted but needs to remain actionable. - subjectInfo = { - subjectType: 'com.atproto.repo.strongRef', - subjectDid: subject.uri.host, - subjectUri: subject.uri.toString(), - subjectCid: subject.cid.toString(), - } + const meta: Record = {} + + if (isModEventReport(event)) { + meta.reportType = event.reportType } - const subjectActions = await this.getCurrentActions(subject) - if (subjectActions.length) { - throw new InvalidRequestError( - `Subject already has an active action: #${subjectActions[0].id}`, - 'SubjectHasAction', - ) + if (isModEventComment(event) && event.sticky) { + meta.sticky = event.sticky + } + + if (isModEventEmail(event)) { + meta.subjectLine = event.subjectLine } - const actionResult = await this.db.db - .insertInto('moderation_action') + + const modEvent = await this.db.db + .insertInto('moderation_event') .values({ - action, - reason, + comment: event.comment ? `${event.comment}` : null, + action: event.$type as ModerationEvent['action'], createdAt: createdAt.toISOString(), createdBy, createLabelVals, negateLabelVals, - durationInHours, + durationInHours: event.durationInHours + ? Number(event.durationInHours) + : null, + meta, expiresAt: - durationInHours !== undefined - ? addHoursToDate(durationInHours, createdAt).toISOString() + (isModEventTakedown(event) || isModEventMute(event)) && + event.durationInHours + ? addHoursToDate(event.durationInHours, createdAt).toISOString() : undefined, ...subjectInfo, }) .returningAll() .executeTakeFirstOrThrow() - if (subjectBlobCids?.length && !('did' in subject)) { - const blobActions = await this.getCurrentActions({ - cids: subjectBlobCids, - }) - if (blobActions.length) { - throw new InvalidRequestError( - `Blob already has an active action: #${blobActions[0].id}`, - 'SubjectHasAction', - ) - } + await adjustModerationSubjectStatus(this.db, modEvent, subjectBlobCids) - await this.db.db - .insertInto('moderation_action_subject_blob') - .values( - subjectBlobCids.map((cid) => ({ - actionId: actionResult.id, - cid: cid.toString(), - })), - ) - .execute() + return modEvent + } + + async getLastReversibleEventForSubject({ + did, + muteUntil, + recordPath, + suspendUntil, + }: ModerationSubjectStatusRow) { + const isSuspended = suspendUntil && new Date(suspendUntil) < new Date() + const isMuted = muteUntil && new Date(muteUntil) < new Date() + + // If the subject is neither suspended nor muted don't bother finding the last reversible event + // Ideally, this should never happen because the caller of this method should only call this + // after ensuring that the suspended or muted subjects are being reversed + if (!isSuspended && !isMuted) { + return null + } + + let builder = this.db.db + .selectFrom('moderation_event') + .where('subjectDid', '=', did) + + if (recordPath) { + builder = builder.where('subjectUri', 'like', `%${recordPath}%`) + } + + // Means the subject was suspended and needs to be unsuspended + if (isSuspended) { + builder = builder + .where('action', '=', 'com.atproto.admin.defs#modEventTakedown') + .where('durationInHours', 'is not', null) + } + if (isMuted) { + builder = builder + .where('action', '=', 'com.atproto.admin.defs#modEventMute') + .where('durationInHours', 'is not', null) } - return actionResult + return await builder + .orderBy('id', 'desc') + .selectAll() + .limit(1) + .executeTakeFirst() } - async getActionsDueForReversal(): Promise { - const actionsDueForReversal = await this.db.db - .selectFrom('moderation_action') - .where('durationInHours', 'is not', null) - .where('expiresAt', '<', new Date().toISOString()) - .where('reversedAt', 'is', null) + async getSubjectsDueForReversal(): Promise { + const subjectsDueForReversal = await this.db.db + .selectFrom('moderation_subject_status') + .where('suspendUntil', '<', new Date().toISOString()) + .orWhere('muteUntil', '<', new Date().toISOString()) .selectAll() .execute() - return actionsDueForReversal + return subjectsDueForReversal } - async revertAction({ - id, + async revertState({ createdBy, createdAt, - reason, - }: ReversibleModerationAction): Promise<{ - result: ModerationActionRow + comment, + action, + subject, + }: ReversibleModerationEvent): Promise<{ + result: ModerationEventRow restored?: TakedownSubjects }> { + const isRevertingTakedown = + action === 'com.atproto.admin.defs#modEventTakedown' this.db.assertTransaction() - const result = await this.logReverseAction({ - id, + const result = await this.logEvent({ + event: { + $type: isRevertingTakedown + ? 'com.atproto.admin.defs#modEventReverseTakedown' + : 'com.atproto.admin.defs#modEventUnmute', + comment: comment ?? undefined, + }, createdAt, createdBy, - reason, + subject, }) let restored: TakedownSubjects | undefined + if (!isRevertingTakedown) { + return { result, restored } + } + if ( - result.action === TAKEDOWN && result.subjectType === 'com.atproto.admin.defs#repoRef' && result.subjectDid ) { @@ -394,7 +373,6 @@ export class ModerationService { } if ( - result.action === TAKEDOWN && result.subjectType === 'com.atproto.repo.strongRef' && result.subjectUri ) { @@ -403,11 +381,14 @@ export class ModerationService { uri, }) const did = uri.hostname - const actionBlobs = await this.db.db - .selectFrom('moderation_action_subject_blob') - .where('actionId', '=', id) - .select('cid') - .execute() + // TODO: MOD_EVENT This bit needs testing + const subjectStatus = await this.db.db + .selectFrom('moderation_subject_status') + .where('did', '=', uri.host) + .where('recordPath', '=', `${uri.collection}/${uri.rkey}`) + .select('blobCids') + .executeTakeFirst() + const blobCids = subjectStatus?.blobCids || [] restored = { did, subjects: [ @@ -416,10 +397,10 @@ export class ModerationService { uri: result.subjectUri, cid: result.subjectCid ?? '', }, - ...actionBlobs.map((row) => ({ + ...blobCids.map((cid) => ({ $type: 'com.atproto.admin.defs#repoBlobRef', did, - cid: row.cid, + cid, recordUri: result.subjectUri, })), ], @@ -429,29 +410,6 @@ export class ModerationService { return { result, restored } } - async logReverseAction( - info: ReversibleModerationAction, - ): Promise { - const { id, createdBy, reason, createdAt = new Date() } = info - - const result = await this.db.db - .updateTable('moderation_action') - .where('id', '=', id) - .set({ - reversedAt: createdAt.toISOString(), - reversedBy: createdBy, - reversedReason: reason, - }) - .returningAll() - .executeTakeFirst() - - if (!result) { - throw new InvalidRequestError('Moderation action not found') - } - - return result - } - async takedownRepo(info: { takedownId: number did: string @@ -536,64 +494,13 @@ export class ModerationService { .execute() } - async resolveReports(info: { - reportIds: number[] - actionId: number - createdBy: string - createdAt?: Date - }): Promise { - const { reportIds, actionId, createdBy, createdAt = new Date() } = info - const action = await this.getActionOrThrow(actionId) - - if (!reportIds.length) return - const reports = await this.db.db - .selectFrom('moderation_report') - .where('id', 'in', reportIds) - .select(['id', 'subjectType', 'subjectDid', 'subjectUri']) - .execute() - - reportIds.forEach((reportId) => { - const report = reports.find((r) => r.id === reportId) - if (!report) throw new InvalidRequestError('Report not found') - if (action.subjectDid !== report.subjectDid) { - // Report and action always must target repo or record from the same did - throw new InvalidRequestError( - `Report ${report.id} cannot be resolved by action`, - ) - } - if ( - action.subjectType === 'com.atproto.repo.strongRef' && - report.subjectType === 'com.atproto.repo.strongRef' && - report.subjectUri !== action.subjectUri - ) { - // If report and action are both for a record, they must be for the same record - throw new InvalidRequestError( - `Report ${report.id} cannot be resolved by action`, - ) - } - }) - - await this.db.db - .insertInto('moderation_report_resolution') - .values( - reportIds.map((reportId) => ({ - reportId, - actionId, - createdAt: createdAt.toISOString(), - createdBy, - })), - ) - .onConflict((oc) => oc.doNothing()) - .execute() - } - async report(info: { - reasonType: ModerationReportRow['reasonType'] + reasonType: NonNullable['reportType'] reason?: string subject: { did: string } | { uri: AtUri; cid: CID } reportedBy: string createdAt?: Date - }): Promise { + }): Promise { const { reasonType, reason, @@ -602,39 +509,144 @@ export class ModerationService { subject, } = info - // Resolve subject info - let subjectInfo: SubjectInfo - if ('did' in subject) { - // Allowing dids that may not exist: may not be known yet to appview but needs to remain reportable. - subjectInfo = { - subjectType: 'com.atproto.admin.defs#repoRef', - subjectDid: subject.did, - subjectUri: null, - subjectCid: null, - } - } else { - // Allowing records/blobs that may not exist: may not be known yet to appview but needs to remain reportable. - subjectInfo = { - subjectType: 'com.atproto.repo.strongRef', - subjectDid: subject.uri.host, - subjectUri: subject.uri.toString(), - subjectCid: subject.cid.toString(), - } + const event = await this.logEvent({ + event: { + $type: 'com.atproto.admin.defs#modEventReport', + reportType: reasonType, + comment: reason, + }, + createdBy: reportedBy, + subject, + createdAt, + }) + + return event + } + + async getSubjectStatuses({ + cursor, + limit = 50, + takendown, + reviewState, + reviewedAfter, + reviewedBefore, + reportedAfter, + reportedBefore, + includeMuted, + ignoreSubjects, + sortDirection, + lastReviewedBy, + sortField, + subject, + }: { + cursor?: string + limit?: number + takendown?: boolean + reviewedBefore?: string + reviewState?: ModerationSubjectStatusRow['reviewState'] + reviewedAfter?: string + reportedAfter?: string + reportedBefore?: string + includeMuted?: boolean + subject?: string + ignoreSubjects?: string[] + sortDirection: 'asc' | 'desc' + lastReviewedBy?: string + sortField: 'lastReviewedAt' | 'lastReportedAt' + }) { + let builder = this.db.db + .selectFrom('moderation_subject_status') + .leftJoin('actor', 'actor.did', 'moderation_subject_status.did') + + if (subject) { + const subjectInfo = getStatusIdentifierFromSubject(subject) + builder = builder + .where('moderation_subject_status.did', '=', subjectInfo.did) + .where((qb) => + subjectInfo.recordPath + ? qb.where('recordPath', '=', subjectInfo.recordPath) + : qb.where('recordPath', '=', ''), + ) } - const report = await this.db.db - .insertInto('moderation_report') - .values({ - reasonType, - reason: reason || null, - createdAt: createdAt.toISOString(), - reportedByDid: reportedBy, - ...subjectInfo, - }) - .returningAll() - .executeTakeFirstOrThrow() + if (ignoreSubjects?.length) { + builder = builder + .where('moderation_subject_status.did', 'not in', ignoreSubjects) + .where('recordPath', 'not in', ignoreSubjects) + } + + if (reviewState) { + builder = builder.where('reviewState', '=', reviewState) + } + + if (lastReviewedBy) { + builder = builder.where('lastReviewedBy', '=', lastReviewedBy) + } + + if (reviewedAfter) { + builder = builder.where('lastReviewedAt', '>', reviewedAfter) + } + + if (reviewedBefore) { + builder = builder.where('lastReviewedAt', '<', reviewedBefore) + } + + if (reportedAfter) { + builder = builder.where('lastReviewedAt', '>', reportedAfter) + } + + if (reportedBefore) { + builder = builder.where('lastReportedAt', '<', reportedBefore) + } + + if (takendown) { + builder = builder.where('takendown', '=', true) + } + + if (!includeMuted) { + builder = builder.where((qb) => + qb + .where('muteUntil', '<', new Date().toISOString()) + .orWhere('muteUntil', 'is', null), + ) + } + + const { ref } = this.db.db.dynamic + const keyset = new StatusKeyset( + ref(`moderation_subject_status.${sortField}`), + ref('moderation_subject_status.id'), + ) + const paginatedBuilder = paginate(builder, { + limit, + cursor, + keyset, + direction: sortDirection, + tryIndex: true, + nullsLast: true, + }) + + const results = await paginatedBuilder + .select('actor.handle as handle') + .selectAll('moderation_subject_status') + .execute() + + return { statuses: results, cursor: keyset.packFromResult(results) } + } + + async isSubjectTakendown( + subject: { did: string } | { uri: AtUri }, + ): Promise { + const { did, recordPath } = getStatusIdentifierFromSubject( + 'did' in subject ? subject.did : subject.uri, + ) + let builder = this.db.db + .selectFrom('moderation_subject_status') + .where('did', '=', did) + .where('recordPath', '=', recordPath || '') + + const result = await builder.select('takendown').executeTakeFirst() - return report + return !!result?.takendown } } @@ -642,30 +654,3 @@ export type TakedownSubjects = { did: string subjects: (RepoRef | RepoBlobRef | StrongRef)[] } - -export type ModerationActionRow = Selectable -export type ReversibleModerationAction = Pick< - ModerationActionRow, - 'id' | 'createdBy' | 'reason' -> & { - createdAt?: Date -} - -export type ModerationReportRow = Selectable -export type ModerationReportRowWithHandle = ModerationReportRow & { - handle?: string | null -} - -export type SubjectInfo = - | { - subjectType: 'com.atproto.admin.defs#repoRef' - subjectDid: string - subjectUri: null - subjectCid: null - } - | { - subjectType: 'com.atproto.repo.strongRef' - subjectDid: string - subjectUri: string - subjectCid: string - } diff --git a/packages/bsky/src/services/moderation/pagination.ts b/packages/bsky/src/services/moderation/pagination.ts new file mode 100644 index 00000000000..c68de0822d4 --- /dev/null +++ b/packages/bsky/src/services/moderation/pagination.ts @@ -0,0 +1,96 @@ +import { InvalidRequestError } from '@atproto/xrpc-server' +import { DynamicModule, sql } from 'kysely' + +import { Cursor, GenericKeyset } from '../../db/pagination' + +type StatusKeysetParam = { + lastReviewedAt: string | null + lastReportedAt: string | null + id: number +} + +export class StatusKeyset extends GenericKeyset { + labelResult(result: StatusKeysetParam): Cursor + labelResult(result: StatusKeysetParam) { + const primaryField = ( + this.primary as ReturnType + ).dynamicReference.includes('lastReviewedAt') + ? 'lastReviewedAt' + : 'lastReportedAt' + + return { + primary: result[primaryField] + ? new Date(`${result[primaryField]}`).getTime().toString() + : '', + secondary: result.id.toString(), + } + } + labeledResultToCursor(labeled: Cursor) { + return { + primary: labeled.primary, + secondary: labeled.secondary, + } + } + cursorToLabeledResult(cursor: Cursor) { + return { + primary: cursor.primary + ? new Date(parseInt(cursor.primary, 10)).toISOString() + : '', + secondary: cursor.secondary, + } + } + unpackCursor(cursorStr?: string): Cursor | undefined { + if (!cursorStr) return + const result = cursorStr.split('::') + const [primary, secondary, ...others] = result + if (!secondary || others.length > 0) { + throw new InvalidRequestError('Malformed cursor') + } + return { + primary, + secondary, + } + } + // This is specifically built to handle nullable columns as primary sorting column + getSql(labeled?: Cursor, direction?: 'asc' | 'desc') { + if (labeled === undefined) return + if (direction === 'asc') { + return !labeled.primary + ? sql`(${this.primary} IS NULL AND ${this.secondary} > ${labeled.secondary})` + : sql`((${this.primary}, ${this.secondary}) > (${labeled.primary}, ${labeled.secondary}) OR (${this.primary} is null))` + } else { + return !labeled.primary + ? sql`(${this.primary} IS NULL AND ${this.secondary} < ${labeled.secondary})` + : sql`((${this.primary}, ${this.secondary}) < (${labeled.primary}, ${labeled.secondary}) OR (${this.primary} is null))` + } + } +} + +type TimeIdKeysetParam = { + id: number + createdAt: string +} +type TimeIdResult = TimeIdKeysetParam + +export class TimeIdKeyset extends GenericKeyset { + labelResult(result: TimeIdResult): Cursor + labelResult(result: TimeIdResult) { + return { primary: result.createdAt, secondary: result.id.toString() } + } + labeledResultToCursor(labeled: Cursor) { + return { + primary: new Date(labeled.primary).getTime().toString(), + secondary: labeled.secondary, + } + } + cursorToLabeledResult(cursor: Cursor) { + const primaryDate = new Date(parseInt(cursor.primary, 10)) + if (isNaN(primaryDate.getTime())) { + throw new InvalidRequestError('Malformed cursor') + } + return { + primary: primaryDate.toISOString(), + secondary: cursor.secondary, + } + } +} diff --git a/packages/bsky/src/services/moderation/status.ts b/packages/bsky/src/services/moderation/status.ts new file mode 100644 index 00000000000..41fb3873226 --- /dev/null +++ b/packages/bsky/src/services/moderation/status.ts @@ -0,0 +1,244 @@ +// This may require better organization but for now, just dumping functions here containing DB queries for moderation status + +import { AtUri } from '@atproto/syntax' +import { PrimaryDatabase } from '../../db' +import { + ModerationEvent, + ModerationSubjectStatus, +} from '../../db/tables/moderation' +import { + REVIEWOPEN, + REVIEWCLOSED, + REVIEWESCALATED, +} from '../../lexicon/types/com/atproto/admin/defs' +import { ModerationEventRow, ModerationSubjectStatusRow } from './types' +import { HOUR } from '@atproto/common' +import { CID } from 'multiformats/cid' +import { sql } from 'kysely' + +const getSubjectStatusForModerationEvent = ({ + action, + createdBy, + createdAt, + durationInHours, +}: { + action: string + createdBy: string + createdAt: string + durationInHours: number | null +}): Partial | null => { + switch (action) { + case 'com.atproto.admin.defs#modEventAcknowledge': + return { + lastReviewedBy: createdBy, + reviewState: REVIEWCLOSED, + lastReviewedAt: createdAt, + } + case 'com.atproto.admin.defs#modEventReport': + return { + reviewState: REVIEWOPEN, + lastReportedAt: createdAt, + } + case 'com.atproto.admin.defs#modEventEscalate': + return { + lastReviewedBy: createdBy, + reviewState: REVIEWESCALATED, + lastReviewedAt: createdAt, + } + case 'com.atproto.admin.defs#modEventReverseTakedown': + return { + lastReviewedBy: createdBy, + reviewState: REVIEWCLOSED, + takendown: false, + suspendUntil: null, + lastReviewedAt: createdAt, + } + case 'com.atproto.admin.defs#modEventUnmute': + return { + lastReviewedBy: createdBy, + muteUntil: null, + reviewState: REVIEWOPEN, + lastReviewedAt: createdAt, + } + case 'com.atproto.admin.defs#modEventTakedown': + return { + takendown: true, + lastReviewedBy: createdBy, + reviewState: REVIEWCLOSED, + lastReviewedAt: createdAt, + suspendUntil: durationInHours + ? new Date(Date.now() + durationInHours * HOUR).toISOString() + : null, + } + case 'com.atproto.admin.defs#modEventMute': + return { + lastReviewedBy: createdBy, + reviewState: REVIEWOPEN, + lastReviewedAt: createdAt, + // By default, mute for 24hrs + muteUntil: new Date( + Date.now() + (durationInHours || 24) * HOUR, + ).toISOString(), + } + case 'com.atproto.admin.defs#modEventComment': + return { + lastReviewedBy: createdBy, + lastReviewedAt: createdAt, + } + default: + return null + } +} + +// Based on a given moderation action event, this function will update the moderation status of the subject +// If there's no existing status, it will create one +// If the action event does not affect the status, it will do nothing +export const adjustModerationSubjectStatus = async ( + db: PrimaryDatabase, + moderationEvent: ModerationEventRow, + blobCids?: CID[], +) => { + const { + action, + subjectDid, + subjectUri, + subjectCid, + createdBy, + meta, + comment, + createdAt, + } = moderationEvent + + const subjectStatus = getSubjectStatusForModerationEvent({ + action, + createdBy, + createdAt, + durationInHours: moderationEvent.durationInHours, + }) + + // If there are no subjectStatus that means there are no side-effect of the incoming event + if (!subjectStatus) { + return null + } + + const now = new Date().toISOString() + // If subjectUri exists, it's not a repoRef so pass along the uri to get identifier back + const identifier = getStatusIdentifierFromSubject(subjectUri || subjectDid) + + db.assertTransaction() + + const currentStatus = await db.db + .selectFrom('moderation_subject_status') + .where('did', '=', identifier.did) + .where('recordPath', '=', identifier.recordPath) + .selectAll() + .executeTakeFirst() + + if ( + currentStatus?.reviewState === REVIEWESCALATED && + subjectStatus.reviewState === REVIEWOPEN + ) { + // If the current status is escalated and the incoming event is to open the review + // We want to keep the status as escalated + subjectStatus.reviewState = REVIEWESCALATED + } + + // Set these because we don't want to override them if they're already set + const defaultData = { + comment: null, + // Defaulting reviewState to open for any event may not be the desired behavior. + // For instance, if a subject never had any event and we just want to leave a comment to keep an eye on it + // that shouldn't mean we want to review the subject + reviewState: REVIEWOPEN, + recordCid: subjectCid || null, + } + const newStatus = { + ...defaultData, + ...subjectStatus, + } + + if ( + action === 'com.atproto.admin.defs#modEventReverseTakedown' && + !subjectStatus.takendown + ) { + newStatus.takendown = false + subjectStatus.takendown = false + } + + if (action === 'com.atproto.admin.defs#modEventComment' && meta?.sticky) { + newStatus.comment = comment + subjectStatus.comment = comment + } + + if (blobCids?.length) { + const newBlobCids = sql`${JSON.stringify( + blobCids.map((c) => c.toString()), + )}` as unknown as ModerationSubjectStatusRow['blobCids'] + newStatus.blobCids = newBlobCids + subjectStatus.blobCids = newBlobCids + } + + const insertQuery = db.db + .insertInto('moderation_subject_status') + .values({ + ...identifier, + ...newStatus, + createdAt: now, + updatedAt: now, + // TODO: Need to get the types right here. + } as ModerationSubjectStatusRow) + .onConflict((oc) => + oc.constraint('moderation_status_unique_idx').doUpdateSet({ + ...subjectStatus, + updatedAt: now, + }), + ) + + const status = await insertQuery.executeTakeFirst() + return status +} + +type ModerationSubjectStatusFilter = + | Pick + | Pick + | Pick +export const getModerationSubjectStatus = async ( + db: PrimaryDatabase, + filters: ModerationSubjectStatusFilter, +) => { + let builder = db.db + .selectFrom('moderation_subject_status') + // DID will always be passed at the very least + .where('did', '=', filters.did) + .where('recordPath', '=', 'recordPath' in filters ? filters.recordPath : '') + + if ('recordCid' in filters) { + builder = builder.where('recordCid', '=', filters.recordCid) + } else { + builder = builder.where('recordCid', 'is', null) + } + + return builder.executeTakeFirst() +} + +export const getStatusIdentifierFromSubject = ( + subject: string | AtUri, +): { did: string; recordPath: string } => { + const isSubjectString = typeof subject === 'string' + if (isSubjectString && subject.startsWith('did:')) { + return { + did: subject, + recordPath: '', + } + } + + if (isSubjectString && !subject.startsWith('at://')) { + throw new Error('Subject is neither a did nor an at-uri') + } + + const uri = isSubjectString ? new AtUri(subject) : subject + return { + did: uri.host, + recordPath: `${uri.collection}/${uri.rkey}`, + } +} diff --git a/packages/bsky/src/services/moderation/types.ts b/packages/bsky/src/services/moderation/types.ts new file mode 100644 index 00000000000..77a8baf71ff --- /dev/null +++ b/packages/bsky/src/services/moderation/types.ts @@ -0,0 +1,49 @@ +import { Selectable } from 'kysely' +import { + ModerationEvent, + ModerationSubjectStatus, +} from '../../db/tables/moderation' +import { AtUri } from '@atproto/syntax' +import { CID } from 'multiformats/cid' +import { ComAtprotoAdminDefs } from '@atproto/api' + +export type SubjectInfo = + | { + subjectType: 'com.atproto.admin.defs#repoRef' + subjectDid: string + subjectUri: null + subjectCid: null + } + | { + subjectType: 'com.atproto.repo.strongRef' + subjectDid: string + subjectUri: string + subjectCid: string + } + +export type ModerationEventRow = Selectable +export type ReversibleModerationEvent = Pick< + ModerationEventRow, + 'createdBy' | 'comment' | 'action' +> & { + createdAt?: Date + subject: { did: string } | { uri: AtUri; cid: CID } +} + +export type ModerationEventRowWithHandle = ModerationEventRow & { + subjectHandle?: string | null + creatorHandle?: string | null +} +export type ModerationSubjectStatusRow = Selectable +export type ModerationSubjectStatusRowWithHandle = + ModerationSubjectStatusRow & { handle: string | null } + +export type ModEventType = + | ComAtprotoAdminDefs.ModEventTakedown + | ComAtprotoAdminDefs.ModEventAcknowledge + | ComAtprotoAdminDefs.ModEventEscalate + | ComAtprotoAdminDefs.ModEventComment + | ComAtprotoAdminDefs.ModEventLabel + | ComAtprotoAdminDefs.ModEventReport + | ComAtprotoAdminDefs.ModEventMute + | ComAtprotoAdminDefs.ModEventReverseTakedown diff --git a/packages/bsky/src/services/moderation/views.ts b/packages/bsky/src/services/moderation/views.ts index 06398c3427e..418253ba649 100644 --- a/packages/bsky/src/services/moderation/views.ts +++ b/packages/bsky/src/services/moderation/views.ts @@ -1,4 +1,4 @@ -import { Selectable } from 'kysely' +import { sql } from 'kysely' import { ArrayEl } from '@atproto/common' import { AtUri } from '@atproto/syntax' import { INVALID_HANDLE } from '@atproto/syntax' @@ -6,22 +6,25 @@ import { BlobRef, jsonStringToLex } from '@atproto/lexicon' import { Database } from '../../db' import { Actor } from '../../db/tables/actor' import { Record as RecordRow } from '../../db/tables/record' -import { ModerationAction } from '../../db/tables/moderation' import { + ModEventView, RepoView, RepoViewDetail, RecordView, RecordViewDetail, - ActionView, - ActionViewDetail, - ReportView, ReportViewDetail, BlobView, + SubjectStatusView, + ModEventViewDetail, } from '../../lexicon/types/com/atproto/admin/defs' import { OutputSchema as ReportOutput } from '../../lexicon/types/com/atproto/moderation/createReport' import { Label } from '../../lexicon/types/com/atproto/label/defs' -import { ModerationReportRowWithHandle } from '.' +import { + ModerationEventRowWithHandle, + ModerationSubjectStatusRowWithHandle, +} from './types' import { getSelfLabels } from '../label' +import { REASONOTHER } from '../../lexicon/types/com/atproto/moderation/defs' export class ModerationViews { constructor(private db: Database) {} @@ -34,7 +37,7 @@ export class ModerationViews { const results = Array.isArray(result) ? result : [result] if (results.length === 0) return [] - const [info, actionResults] = await Promise.all([ + const [info, subjectStatuses] = await Promise.all([ await this.db.db .selectFrom('actor') .leftJoin('profile', 'profile.creator', 'actor.did') @@ -50,31 +53,21 @@ export class ModerationViews { ) .select(['actor.did as did', 'profile_record.json as profileJson']) .execute(), - this.db.db - .selectFrom('moderation_action') - .where('reversedAt', 'is', null) - .where('subjectType', '=', 'com.atproto.admin.defs#repoRef') - .where( - 'subjectDid', - 'in', - results.map((r) => r.did), - ) - .select(['id', 'action', 'durationInHours', 'subjectDid']) - .execute(), + this.getSubjectStatus(results.map((r) => ({ did: r.did }))), ]) const infoByDid = info.reduce( (acc, cur) => Object.assign(acc, { [cur.did]: cur }), {} as Record>, ) - const actionByDid = actionResults.reduce( - (acc, cur) => Object.assign(acc, { [cur.subjectDid ?? '']: cur }), - {} as Record>, + const subjectStatusByDid = subjectStatuses.reduce( + (acc, cur) => + Object.assign(acc, { [cur.did ?? '']: this.subjectStatus(cur) }), + {}, ) const views = results.map((r) => { const { profileJson } = infoByDid[r.did] ?? {} - const action = actionByDid[r.did] const relatedRecords: object[] = [] if (profileJson) { relatedRecords.push( @@ -88,49 +81,125 @@ export class ModerationViews { relatedRecords, indexedAt: r.indexedAt, moderation: { - currentAction: action + subjectStatus: subjectStatusByDid[r.did] ?? undefined, + }, + } + }) + + return Array.isArray(result) ? views : views[0] + } + event(result: EventResult): Promise + event(result: EventResult[]): Promise + async event( + result: EventResult | EventResult[], + ): Promise { + const results = Array.isArray(result) ? result : [result] + if (results.length === 0) return [] + + const views = results.map((res) => { + const eventView: ModEventView = { + id: res.id, + event: { + $type: res.action, + comment: res.comment ?? undefined, + }, + subject: + res.subjectType === 'com.atproto.admin.defs#repoRef' ? { - id: action.id, - action: action.action, - durationInHours: action.durationInHours ?? undefined, + $type: 'com.atproto.admin.defs#repoRef', + did: res.subjectDid, } - : undefined, - }, + : { + $type: 'com.atproto.repo.strongRef', + uri: res.subjectUri, + cid: res.subjectCid, + }, + subjectBlobCids: [], + createdBy: res.createdBy, + createdAt: res.createdAt, + subjectHandle: res.subjectHandle ?? undefined, + creatorHandle: res.creatorHandle ?? undefined, + } + + if ( + [ + 'com.atproto.admin.defs#modEventTakedown', + 'com.atproto.admin.defs#modEventMute', + ].includes(res.action) + ) { + eventView.event = { + ...eventView.event, + durationInHours: res.durationInHours ?? undefined, + } + } + + if (res.action === 'com.atproto.admin.defs#modEventLabel') { + eventView.event = { + ...eventView.event, + createLabelVals: res.createLabelVals?.length + ? res.createLabelVals.split(' ') + : [], + negateLabelVals: res.negateLabelVals?.length + ? res.negateLabelVals.split(' ') + : [], + } } + + if (res.action === 'com.atproto.admin.defs#modEventReport') { + eventView.event = { + ...eventView.event, + reportType: res.meta?.reportType ?? undefined, + } + } + + if (res.action === 'com.atproto.admin.defs#modEventEmail') { + eventView.event = { + ...eventView.event, + subject: res.meta?.subject ?? undefined, + } + } + + if ( + res.action === 'com.atproto.admin.defs#modEventComment' && + res.meta?.sticky + ) { + eventView.event.sticky = true + } + + return eventView }) return Array.isArray(result) ? views : views[0] } - async repoDetail(result: RepoResult): Promise { - const repo = await this.repo(result) - const [reportResults, actionResults] = await Promise.all([ - this.db.db - .selectFrom('moderation_report') - .where('subjectType', '=', 'com.atproto.admin.defs#repoRef') - .where('subjectDid', '=', repo.did) - .orderBy('id', 'desc') - .selectAll() - .execute(), - this.db.db - .selectFrom('moderation_action') - .where('subjectType', '=', 'com.atproto.admin.defs#repoRef') - .where('subjectDid', '=', repo.did) - .orderBy('id', 'desc') - .selectAll() - .execute(), + async eventDetail(result: EventResult): Promise { + const [event, subject] = await Promise.all([ + this.event(result), + this.subject(result), ]) - const [reports, actions, labels] = await Promise.all([ - this.report(reportResults), - this.action(actionResults), - this.labels(repo.did), + const allBlobs = findBlobRefs(subject.value) + const subjectBlobs = await this.blob( + allBlobs.filter((blob) => + event.subjectBlobCids.includes(blob.ref.toString()), + ), + ) + return { + ...event, + subject, + subjectBlobs, + } + } + + async repoDetail(result: RepoResult): Promise { + const [repo, labels] = await Promise.all([ + this.repo(result), + this.labels(result.did), ]) + return { ...repo, moderation: { ...repo.moderation, - reports, - actions, }, labels, } @@ -144,7 +213,7 @@ export class ModerationViews { const results = Array.isArray(result) ? result : [result] if (results.length === 0) return [] - const [repoResults, actionResults] = await Promise.all([ + const [repoResults, subjectStatuses] = await Promise.all([ this.db.db .selectFrom('actor') .where( @@ -154,17 +223,7 @@ export class ModerationViews { ) .selectAll() .execute(), - this.db.db - .selectFrom('moderation_action') - .where('reversedAt', 'is', null) - .where('subjectType', '=', 'com.atproto.repo.strongRef') - .where( - 'subjectUri', - 'in', - results.map((r) => r.uri), - ) - .select(['id', 'action', 'durationInHours', 'subjectUri']) - .execute(), + this.getSubjectStatus(results.map((r) => didAndRecordPathFromUri(r.uri))), ]) const repos = await this.repo(repoResults) @@ -172,14 +231,18 @@ export class ModerationViews { (acc, cur) => Object.assign(acc, { [cur.did]: cur }), {} as Record>, ) - const actionByUri = actionResults.reduce( - (acc, cur) => Object.assign(acc, { [cur.subjectUri ?? '']: cur }), - {} as Record>, + const subjectStatusByUri = subjectStatuses.reduce( + (acc, cur) => + Object.assign(acc, { + [`${cur.did}/${cur.recordPath}` ?? '']: this.subjectStatus(cur), + }), + {}, ) const views = results.map((res) => { const repo = reposByDid[didFromUri(res.uri)] - const action = actionByUri[res.uri] + const { did, recordPath } = didAndRecordPathFromUri(res.uri) + const subjectStatus = subjectStatusByUri[`${did}/${recordPath}`] if (!repo) throw new Error(`Record repo is missing: ${res.uri}`) const value = jsonStringToLex(res.json) as Record return { @@ -190,13 +253,7 @@ export class ModerationViews { indexedAt: res.indexedAt, repo, moderation: { - currentAction: action - ? { - id: action.id, - action: action.action, - durationInHours: action.durationInHours ?? undefined, - } - : undefined, + subjectStatus, }, } }) @@ -205,29 +262,17 @@ export class ModerationViews { } async recordDetail(result: RecordResult): Promise { - const [record, reportResults, actionResults] = await Promise.all([ + const [record, subjectStatusResult] = await Promise.all([ this.record(result), - this.db.db - .selectFrom('moderation_report') - .where('subjectType', '=', 'com.atproto.repo.strongRef') - .where('subjectUri', '=', result.uri) - .leftJoin('actor', 'actor.did', 'moderation_report.subjectDid') - .orderBy('id', 'desc') - .selectAll() - .execute(), - this.db.db - .selectFrom('moderation_action') - .where('subjectType', '=', 'com.atproto.repo.strongRef') - .where('subjectUri', '=', result.uri) - .orderBy('id', 'desc') - .selectAll() - .execute(), + this.getSubjectStatus(didAndRecordPathFromUri(result.uri)), ]) - const [reports, actions, blobs, labels] = await Promise.all([ - this.report(reportResults), - this.action(actionResults), + + const [blobs, labels, subjectStatus] = await Promise.all([ this.blob(findBlobRefs(record.value)), this.labels(record.uri), + subjectStatusResult?.length + ? this.subjectStatus(subjectStatusResult[0]) + : Promise.resolve(undefined), ]) const selfLabels = getSelfLabels({ uri: result.uri, @@ -239,196 +284,22 @@ export class ModerationViews { blobs, moderation: { ...record.moderation, - reports, - actions, + subjectStatus, }, labels: [...labels, ...selfLabels], } } - - action(result: ActionResult): Promise - action(result: ActionResult[]): Promise - async action( - result: ActionResult | ActionResult[], - ): Promise { - const results = Array.isArray(result) ? result : [result] - if (results.length === 0) return [] - - const [resolutions, subjectBlobResults] = await Promise.all([ - this.db.db - .selectFrom('moderation_report_resolution') - .select(['reportId as id', 'actionId']) - .where( - 'actionId', - 'in', - results.map((r) => r.id), - ) - .orderBy('id', 'desc') - .execute(), - await this.db.db - .selectFrom('moderation_action_subject_blob') - .selectAll() - .where( - 'actionId', - 'in', - results.map((r) => r.id), - ) - .execute(), - ]) - - const reportIdsByActionId = resolutions.reduce((acc, cur) => { - acc[cur.actionId] ??= [] - acc[cur.actionId].push(cur.id) - return acc - }, {} as Record) - const subjectBlobCidsByActionId = subjectBlobResults.reduce((acc, cur) => { - acc[cur.actionId] ??= [] - acc[cur.actionId].push(cur.cid) - return acc - }, {} as Record) - - const views = results.map((res) => ({ - id: res.id, - action: res.action, - durationInHours: res.durationInHours ?? undefined, - subject: - res.subjectType === 'com.atproto.admin.defs#repoRef' - ? { - $type: 'com.atproto.admin.defs#repoRef', - did: res.subjectDid, - } - : { - $type: 'com.atproto.repo.strongRef', - uri: res.subjectUri, - cid: res.subjectCid, - }, - subjectBlobCids: subjectBlobCidsByActionId[res.id] ?? [], - reason: res.reason, - createdAt: res.createdAt, - createdBy: res.createdBy, - createLabelVals: - res.createLabelVals && res.createLabelVals.length > 0 - ? res.createLabelVals.split(' ') - : undefined, - negateLabelVals: - res.negateLabelVals && res.negateLabelVals.length > 0 - ? res.negateLabelVals.split(' ') - : undefined, - reversal: - res.reversedAt !== null && - res.reversedBy !== null && - res.reversedReason !== null - ? { - createdAt: res.reversedAt, - createdBy: res.reversedBy, - reason: res.reversedReason, - } - : undefined, - resolvedReportIds: reportIdsByActionId[res.id] ?? [], - })) - - return Array.isArray(result) ? views : views[0] - } - - async actionDetail(result: ActionResult): Promise { - const action = await this.action(result) - const reportResults = action.resolvedReportIds.length - ? await this.db.db - .selectFrom('moderation_report') - .where('id', 'in', action.resolvedReportIds) - .orderBy('id', 'desc') - .selectAll() - .execute() - : [] - const [subject, resolvedReports] = await Promise.all([ - this.subject(result), - this.report(reportResults), - ]) - const allBlobs = findBlobRefs(subject.value) - const subjectBlobs = await this.blob( - allBlobs.filter((blob) => - action.subjectBlobCids.includes(blob.ref.toString()), - ), - ) - return { - id: action.id, - action: action.action, - durationInHours: action.durationInHours, - subject, - subjectBlobs, - createLabelVals: action.createLabelVals, - negateLabelVals: action.negateLabelVals, - reason: action.reason, - createdAt: action.createdAt, - createdBy: action.createdBy, - reversal: action.reversal, - resolvedReports, - } - } - - report(result: ReportResult): Promise - report(result: ReportResult[]): Promise - async report( - result: ReportResult | ReportResult[], - ): Promise { - const results = Array.isArray(result) ? result : [result] - if (results.length === 0) return [] - - const resolutions = await this.db.db - .selectFrom('moderation_report_resolution') - .select(['actionId as id', 'reportId']) - .where( - 'reportId', - 'in', - results.map((r) => r.id), - ) - .orderBy('id', 'desc') - .execute() - - const actionIdsByReportId = resolutions.reduce((acc, cur) => { - acc[cur.reportId] ??= [] - acc[cur.reportId].push(cur.id) - return acc - }, {} as Record) - - const views: ReportView[] = results.map((res) => { - const decoratedView: ReportView = { - id: res.id, - createdAt: res.createdAt, - reasonType: res.reasonType, - reason: res.reason ?? undefined, - reportedBy: res.reportedByDid, - subject: - res.subjectType === 'com.atproto.admin.defs#repoRef' - ? { - $type: 'com.atproto.admin.defs#repoRef', - did: res.subjectDid, - } - : { - $type: 'com.atproto.repo.strongRef', - uri: res.subjectUri, - cid: res.subjectCid, - }, - resolvedByActionIds: actionIdsByReportId[res.id] ?? [], - } - - if (res.handle) { - decoratedView.subjectRepoHandle = res.handle - } - - return decoratedView - }) - - return Array.isArray(result) ? views : views[0] - } - reportPublic(report: ReportResult): ReportOutput { return { id: report.id, createdAt: report.createdAt, - reasonType: report.reasonType, - reason: report.reason ?? undefined, - reportedBy: report.reportedByDid, + // Ideally, we would never have a report entry that does not have a reasonType but at the schema level + // we are not guarantying that so in whatever case, if we end up with such entries, default to 'other' + reasonType: report.meta?.reportType + ? (report.meta?.reportType as string) + : REASONOTHER, + reason: report.comment ?? undefined, + reportedBy: report.createdBy, subject: report.subjectType === 'com.atproto.admin.defs#repoRef' ? { @@ -442,32 +313,6 @@ export class ModerationViews { }, } } - - async reportDetail(result: ReportResult): Promise { - const report = await this.report(result) - const actionResults = report.resolvedByActionIds.length - ? await this.db.db - .selectFrom('moderation_action') - .where('id', 'in', report.resolvedByActionIds) - .orderBy('id', 'desc') - .selectAll() - .execute() - : [] - const [subject, resolvedByActions] = await Promise.all([ - this.subject(result), - this.action(actionResults), - ]) - return { - id: report.id, - createdAt: report.createdAt, - reasonType: report.reasonType, - reason: report.reason ?? undefined, - reportedBy: report.reportedBy, - subject, - resolvedByActions, - } - } - // Partial view for subjects async subject(result: SubjectResult): Promise { @@ -511,44 +356,35 @@ export class ModerationViews { async blob(blobs: BlobRef[]): Promise { if (!blobs.length) return [] - const actionResults = await this.db.db - .selectFrom('moderation_action') - .where('reversedAt', 'is', null) - .innerJoin( - 'moderation_action_subject_blob as subject_blob', - 'subject_blob.actionId', - 'moderation_action.id', - ) + const { ref } = this.db.db.dynamic + const modStatusResults = await this.db.db + .selectFrom('moderation_subject_status') .where( - 'subject_blob.cid', - 'in', - blobs.map((blob) => blob.ref.toString()), + sql`${ref( + 'moderation_subject_status.blobCids', + )} @> ${JSON.stringify(blobs.map((blob) => blob.ref.toString()))}`, ) - .select(['id', 'action', 'durationInHours', 'cid']) - .execute() - const actionByCid = actionResults.reduce( - (acc, cur) => Object.assign(acc, { [cur.cid]: cur }), - {} as Record>, + .selectAll() + .executeTakeFirst() + const statusByCid = (modStatusResults?.blobCids || [])?.reduce( + (acc, cur) => Object.assign(acc, { [cur]: modStatusResults }), + {}, ) // Intentionally missing details field, since we don't have any on appview. // We also don't know when the blob was created, so we use a canned creation time. const unknownTime = new Date(0).toISOString() return blobs.map((blob) => { const cid = blob.ref.toString() - const action = actionByCid[cid] + const subjectStatus = statusByCid[cid] + ? this.subjectStatus(statusByCid[cid]) + : undefined return { cid, mimeType: blob.mimeType, size: blob.size, createdAt: unknownTime, moderation: { - currentAction: action - ? { - id: action.id, - action: action.action, - durationInHours: action.durationInHours ?? undefined, - } - : undefined, + subjectStatus, }, } }) @@ -567,27 +403,117 @@ export class ModerationViews { neg: l.neg, })) } + + async getSubjectStatus( + subject: + | { did: string; recordPath?: string } + | { did: string; recordPath?: string }[], + ): Promise { + const subjectFilters = Array.isArray(subject) ? subject : [subject] + const filterForSubject = + ({ did, recordPath }: { did: string; recordPath?: string }) => + // TODO: Fix the typing here? + (clause: any) => { + clause = clause + .where('moderation_subject_status.did', '=', did) + .where('moderation_subject_status.recordPath', '=', recordPath || '') + return clause + } + + const builder = this.db.db + .selectFrom('moderation_subject_status') + .leftJoin('actor', 'actor.did', 'moderation_subject_status.did') + .where((clause) => { + subjectFilters.forEach(({ did, recordPath }, i) => { + const applySubjectFilter = filterForSubject({ did, recordPath }) + if (i === 0) { + clause = clause.where(applySubjectFilter) + } else { + clause = clause.orWhere(applySubjectFilter) + } + }) + + return clause + }) + .selectAll('moderation_subject_status') + .select('actor.handle as handle') + + return builder.execute() + } + + subjectStatus(result: ModerationSubjectStatusRowWithHandle): SubjectStatusView + subjectStatus( + result: ModerationSubjectStatusRowWithHandle[], + ): SubjectStatusView[] + subjectStatus( + result: + | ModerationSubjectStatusRowWithHandle + | ModerationSubjectStatusRowWithHandle[], + ): SubjectStatusView | SubjectStatusView[] { + const results = Array.isArray(result) ? result : [result] + if (results.length === 0) return [] + + const decoratedSubjectStatuses = results.map((subjectStatus) => ({ + id: subjectStatus.id, + reviewState: subjectStatus.reviewState, + createdAt: subjectStatus.createdAt, + updatedAt: subjectStatus.updatedAt, + comment: subjectStatus.comment ?? undefined, + lastReviewedBy: subjectStatus.lastReviewedBy ?? undefined, + lastReviewedAt: subjectStatus.lastReviewedAt ?? undefined, + lastReportedAt: subjectStatus.lastReportedAt ?? undefined, + muteUntil: subjectStatus.muteUntil ?? undefined, + suspendUntil: subjectStatus.suspendUntil ?? undefined, + takendown: subjectStatus.takendown ?? undefined, + subjectRepoHandle: subjectStatus.handle ?? undefined, + subjectBlobCids: subjectStatus.blobCids || [], + subject: !subjectStatus.recordPath + ? { + $type: 'com.atproto.admin.defs#repoRef', + did: subjectStatus.did, + } + : { + $type: 'com.atproto.repo.strongRef', + uri: AtUri.make( + subjectStatus.did, + // Not too intuitive but the recordpath is basically / + // which is what the last 2 params of .make() arguments are + ...subjectStatus.recordPath.split('/'), + ).toString(), + cid: subjectStatus.recordCid, + }, + })) + + return Array.isArray(result) + ? decoratedSubjectStatuses + : decoratedSubjectStatuses[0] + } } type RepoResult = Actor -type ActionResult = Selectable +type EventResult = ModerationEventRowWithHandle -type ReportResult = ModerationReportRowWithHandle +type ReportResult = ModerationEventRowWithHandle type RecordResult = RecordRow type SubjectResult = Pick< - ActionResult & ReportResult, + EventResult & ReportResult, 'id' | 'subjectType' | 'subjectDid' | 'subjectUri' | 'subjectCid' > -type SubjectView = ActionViewDetail['subject'] & ReportViewDetail['subject'] +type SubjectView = ModEventViewDetail['subject'] & ReportViewDetail['subject'] function didFromUri(uri: string) { return new AtUri(uri).host } +function didAndRecordPathFromUri(uri: string) { + const atUri = new AtUri(uri) + return { did: atUri.host, recordPath: `${atUri.collection}/${atUri.rkey}` } +} + function findBlobRefs(value: unknown, refs: BlobRef[] = []) { if (value instanceof BlobRef) { refs.push(value) diff --git a/packages/bsky/tests/admin/__snapshots__/get-moderation-action.test.ts.snap b/packages/bsky/tests/admin/__snapshots__/get-moderation-action.test.ts.snap deleted file mode 100644 index fffc5678d9b..00000000000 --- a/packages/bsky/tests/admin/__snapshots__/get-moderation-action.test.ts.snap +++ /dev/null @@ -1,172 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`admin get moderation action view gets moderation action for a record. 1`] = ` -Object { - "action": "com.atproto.admin.defs#takedown", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 2, - "reason": "X", - "resolvedReports": Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 2, - "reason": "defamation", - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(1)", - "resolvedByActionIds": Array [ - 2, - 1, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - }, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#recordView", - "blobCids": Array [], - "cid": "cids(0)", - "indexedAt": "1970-01-01T00:00:00.000Z", - "moderation": Object { - "currentAction": Object { - "action": "com.atproto.admin.defs#takedown", - "id": 2, - }, - }, - "repo": Object { - "did": "user(0)", - "email": "alice@test.com", - "handle": "alice.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitesDisabled": false, - "moderation": Object {}, - "relatedRecords": Array [ - Object { - "$type": "app.bsky.actor.profile", - "avatar": Object { - "$type": "blob", - "mimeType": "image/jpeg", - "ref": Object { - "$link": "cids(1)", - }, - "size": 3976, - }, - "description": "its me!", - "displayName": "ali", - "labels": Object { - "$type": "com.atproto.label.defs#selfLabels", - "values": Array [ - Object { - "val": "self-label-a", - }, - Object { - "val": "self-label-b", - }, - ], - }, - }, - ], - }, - "uri": "record(0)", - "value": Object { - "$type": "app.bsky.feed.post", - "createdAt": "1970-01-01T00:00:00.000Z", - "labels": Object { - "$type": "com.atproto.label.defs#selfLabels", - "values": Array [ - Object { - "val": "self-label", - }, - ], - }, - "text": "hey there", - }, - }, - "subjectBlobs": Array [], -} -`; - -exports[`admin get moderation action view gets moderation action for a repo. 1`] = ` -Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 1, - "reason": "X", - "resolvedReports": Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 2, - "reason": "defamation", - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(1)", - "resolvedByActionIds": Array [ - 2, - 1, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(1)", - "uri": "record(0)", - }, - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(2)", - "resolvedByActionIds": Array [ - 1, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - }, - ], - "reversal": Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "reason": "X", - }, - "subject": Object { - "$type": "com.atproto.admin.defs#repoView", - "did": "user(0)", - "email": "alice@test.com", - "handle": "alice.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitesDisabled": false, - "moderation": Object {}, - "relatedRecords": Array [ - Object { - "$type": "app.bsky.actor.profile", - "avatar": Object { - "$type": "blob", - "mimeType": "image/jpeg", - "ref": Object { - "$link": "cids(0)", - }, - "size": 3976, - }, - "description": "its me!", - "displayName": "ali", - "labels": Object { - "$type": "com.atproto.label.defs#selfLabels", - "values": Array [ - Object { - "val": "self-label-a", - }, - Object { - "val": "self-label-b", - }, - ], - }, - }, - ], - }, - "subjectBlobs": Array [], -} -`; diff --git a/packages/bsky/tests/admin/__snapshots__/get-moderation-actions.test.ts.snap b/packages/bsky/tests/admin/__snapshots__/get-moderation-actions.test.ts.snap deleted file mode 100644 index 625df2076d8..00000000000 --- a/packages/bsky/tests/admin/__snapshots__/get-moderation-actions.test.ts.snap +++ /dev/null @@ -1,178 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`admin get moderation actions view gets all moderation actions for a record. 1`] = ` -Array [ - Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 1, - "reason": "X", - "resolvedReportIds": Array [ - 1, - ], - "reversal": Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "reason": "X", - }, - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectBlobCids": Array [], - }, -] -`; - -exports[`admin get moderation actions view gets all moderation actions for a repo. 1`] = ` -Array [ - Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 5, - "reason": "X", - "resolvedReportIds": Array [ - 3, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - "subjectBlobCids": Array [], - }, - Object { - "action": "com.atproto.admin.defs#acknowledge", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 2, - "reason": "X", - "resolvedReportIds": Array [], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectBlobCids": Array [], - }, - Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 1, - "reason": "X", - "resolvedReportIds": Array [ - 1, - ], - "reversal": Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "reason": "X", - }, - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(1)", - "uri": "record(1)", - }, - "subjectBlobCids": Array [], - }, -] -`; - -exports[`admin get moderation actions view gets all moderation actions. 1`] = ` -Array [ - Object { - "action": "com.atproto.admin.defs#acknowledge", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 6, - "reason": "X", - "resolvedReportIds": Array [], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - "subjectBlobCids": Array [], - }, - Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 5, - "reason": "X", - "resolvedReportIds": Array [ - 3, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(1)", - }, - "subjectBlobCids": Array [], - }, - Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 4, - "reason": "X", - "resolvedReportIds": Array [], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectBlobCids": Array [], - }, - Object { - "action": "com.atproto.admin.defs#takedown", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 3, - "reason": "X", - "resolvedReportIds": Array [], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(1)", - "uri": "record(1)", - }, - "subjectBlobCids": Array [], - }, - Object { - "action": "com.atproto.admin.defs#acknowledge", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 2, - "reason": "X", - "resolvedReportIds": Array [], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(2)", - "uri": "record(2)", - }, - "subjectBlobCids": Array [], - }, - Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 1, - "reason": "X", - "resolvedReportIds": Array [ - 1, - ], - "reversal": Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "reason": "X", - }, - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(3)", - "uri": "record(3)", - }, - "subjectBlobCids": Array [], - }, -] -`; diff --git a/packages/bsky/tests/admin/__snapshots__/get-moderation-report.test.ts.snap b/packages/bsky/tests/admin/__snapshots__/get-moderation-report.test.ts.snap deleted file mode 100644 index 44a42b129e7..00000000000 --- a/packages/bsky/tests/admin/__snapshots__/get-moderation-report.test.ts.snap +++ /dev/null @@ -1,177 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`admin get moderation action view gets moderation report for a record. 1`] = ` -Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 2, - "reason": "defamation", - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(0)", - "resolvedByActions": Array [ - Object { - "action": "com.atproto.admin.defs#takedown", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 2, - "reason": "X", - "resolvedReportIds": Array [ - 2, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectBlobCids": Array [], - }, - Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 1, - "reason": "X", - "resolvedReportIds": Array [ - 2, - 1, - ], - "reversal": Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "reason": "X", - }, - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(1)", - }, - "subjectBlobCids": Array [], - }, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#recordView", - "blobCids": Array [], - "cid": "cids(0)", - "indexedAt": "1970-01-01T00:00:00.000Z", - "moderation": Object { - "currentAction": Object { - "action": "com.atproto.admin.defs#takedown", - "id": 2, - }, - }, - "repo": Object { - "did": "user(1)", - "email": "alice@test.com", - "handle": "alice.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitesDisabled": false, - "moderation": Object {}, - "relatedRecords": Array [ - Object { - "$type": "app.bsky.actor.profile", - "avatar": Object { - "$type": "blob", - "mimeType": "image/jpeg", - "ref": Object { - "$link": "cids(1)", - }, - "size": 3976, - }, - "description": "its me!", - "displayName": "ali", - "labels": Object { - "$type": "com.atproto.label.defs#selfLabels", - "values": Array [ - Object { - "val": "self-label-a", - }, - Object { - "val": "self-label-b", - }, - ], - }, - }, - ], - }, - "uri": "record(0)", - "value": Object { - "$type": "app.bsky.feed.post", - "createdAt": "1970-01-01T00:00:00.000Z", - "labels": Object { - "$type": "com.atproto.label.defs#selfLabels", - "values": Array [ - Object { - "val": "self-label", - }, - ], - }, - "text": "hey there", - }, - }, -} -`; - -exports[`admin get moderation action view gets moderation report for a repo. 1`] = ` -Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(0)", - "resolvedByActions": Array [ - Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 1, - "reason": "X", - "resolvedReportIds": Array [ - 2, - 1, - ], - "reversal": Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "reason": "X", - }, - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(1)", - }, - "subjectBlobCids": Array [], - }, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#repoView", - "did": "user(1)", - "email": "alice@test.com", - "handle": "alice.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitesDisabled": false, - "moderation": Object {}, - "relatedRecords": Array [ - Object { - "$type": "app.bsky.actor.profile", - "avatar": Object { - "$type": "blob", - "mimeType": "image/jpeg", - "ref": Object { - "$link": "cids(0)", - }, - "size": 3976, - }, - "description": "its me!", - "displayName": "ali", - "labels": Object { - "$type": "com.atproto.label.defs#selfLabels", - "values": Array [ - Object { - "val": "self-label-a", - }, - Object { - "val": "self-label-b", - }, - ], - }, - }, - ], - }, -} -`; diff --git a/packages/bsky/tests/admin/__snapshots__/get-moderation-reports.test.ts.snap b/packages/bsky/tests/admin/__snapshots__/get-moderation-reports.test.ts.snap deleted file mode 100644 index 9708df52cc6..00000000000 --- a/packages/bsky/tests/admin/__snapshots__/get-moderation-reports.test.ts.snap +++ /dev/null @@ -1,307 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`admin get moderation reports view gets all moderation reports actioned by a certain moderator. 1`] = ` -Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(0)", - "resolvedByActionIds": Array [ - 1, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectRepoHandle": "alice.test", - }, -] -`; - -exports[`admin get moderation reports view gets all moderation reports actioned by a certain moderator. 2`] = ` -Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 3, - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(0)", - "resolvedByActionIds": Array [ - 3, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectRepoHandle": "bob.test", - }, -] -`; - -exports[`admin get moderation reports view gets all moderation reports by active resolution action type. 1`] = ` -Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 3, - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(0)", - "resolvedByActionIds": Array [ - 3, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectRepoHandle": "bob.test", - }, -] -`; - -exports[`admin get moderation reports view gets all moderation reports for a record. 1`] = ` -Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(0)", - "resolvedByActionIds": Array [ - 1, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectRepoHandle": "alice.test", - }, -] -`; - -exports[`admin get moderation reports view gets all moderation reports for a repo. 1`] = ` -Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 5, - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(0)", - "resolvedByActionIds": Array [ - 5, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - "subjectRepoHandle": "alice.test", - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 2, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(1)", - "resolvedByActionIds": Array [], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectRepoHandle": "alice.test", - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(0)", - "resolvedByActionIds": Array [ - 1, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(1)", - "uri": "record(1)", - }, - "subjectRepoHandle": "alice.test", - }, -] -`; - -exports[`admin get moderation reports view gets all moderation reports. 1`] = ` -Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 6, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(0)", - "resolvedByActionIds": Array [], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - "subjectRepoHandle": "carol.test", - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 5, - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(1)", - "resolvedByActionIds": Array [ - 5, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(1)", - }, - "subjectRepoHandle": "alice.test", - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 4, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(0)", - "resolvedByActionIds": Array [], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectRepoHandle": "dan.test", - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 3, - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(1)", - "resolvedByActionIds": Array [ - 3, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(1)", - "uri": "record(1)", - }, - "subjectRepoHandle": "bob.test", - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 2, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(0)", - "resolvedByActionIds": Array [], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(2)", - "uri": "record(2)", - }, - "subjectRepoHandle": "alice.test", - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(1)", - "resolvedByActionIds": Array [ - 1, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(3)", - "uri": "record(3)", - }, - "subjectRepoHandle": "alice.test", - }, -] -`; - -exports[`admin get moderation reports view gets all resolved/unresolved moderation reports. 1`] = ` -Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 5, - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(0)", - "resolvedByActionIds": Array [ - 5, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - "subjectRepoHandle": "alice.test", - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 3, - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(0)", - "resolvedByActionIds": Array [ - 3, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectRepoHandle": "bob.test", - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(0)", - "resolvedByActionIds": Array [ - 1, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(1)", - "uri": "record(1)", - }, - "subjectRepoHandle": "alice.test", - }, -] -`; - -exports[`admin get moderation reports view gets all resolved/unresolved moderation reports. 2`] = ` -Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 6, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(0)", - "resolvedByActionIds": Array [], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - "subjectRepoHandle": "carol.test", - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 4, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(0)", - "resolvedByActionIds": Array [], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectRepoHandle": "dan.test", - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 2, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(0)", - "resolvedByActionIds": Array [], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(1)", - "uri": "record(1)", - }, - "subjectRepoHandle": "alice.test", - }, -] -`; diff --git a/packages/bsky/tests/admin/__snapshots__/get-record.test.ts.snap b/packages/bsky/tests/admin/__snapshots__/get-record.test.ts.snap index cbb922003cb..14a83f9dfda 100644 --- a/packages/bsky/tests/admin/__snapshots__/get-record.test.ts.snap +++ b/packages/bsky/tests/admin/__snapshots__/get-record.test.ts.snap @@ -17,74 +17,23 @@ Object { }, ], "moderation": Object { - "actions": Array [ - Object { - "action": "com.atproto.admin.defs#takedown", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 2, - "reason": "X", - "resolvedReportIds": Array [], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectBlobCids": Array [], + "subjectStatus": Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "id": 1, + "lastReportedAt": "1970-01-01T00:00:00.000Z", + "lastReviewedAt": "1970-01-01T00:00:00.000Z", + "lastReviewedBy": "did:example:admin", + "reviewState": "com.atproto.admin.defs#reviewClosed", + "subject": Object { + "$type": "com.atproto.repo.strongRef", + "cid": "cids(0)", + "uri": "record(0)", }, - Object { - "action": "com.atproto.admin.defs#acknowledge", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 1, - "reason": "X", - "resolvedReportIds": Array [], - "reversal": Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "reason": "X", - }, - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectBlobCids": Array [], - }, - ], - "currentAction": Object { - "action": "com.atproto.admin.defs#takedown", - "id": 2, + "subjectBlobCids": Array [], + "subjectRepoHandle": "alice.test", + "takendown": true, + "updatedAt": "1970-01-01T00:00:00.000Z", }, - "reports": Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 2, - "reason": "defamation", - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(1)", - "resolvedByActionIds": Array [], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectRepoHandle": "alice.test", - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(2)", - "resolvedByActionIds": Array [], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectRepoHandle": "alice.test", - }, - ], }, "repo": Object { "did": "user(0)", @@ -154,74 +103,23 @@ Object { }, ], "moderation": Object { - "actions": Array [ - Object { - "action": "com.atproto.admin.defs#takedown", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 2, - "reason": "X", - "resolvedReportIds": Array [], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectBlobCids": Array [], + "subjectStatus": Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "id": 1, + "lastReportedAt": "1970-01-01T00:00:00.000Z", + "lastReviewedAt": "1970-01-01T00:00:00.000Z", + "lastReviewedBy": "did:example:admin", + "reviewState": "com.atproto.admin.defs#reviewClosed", + "subject": Object { + "$type": "com.atproto.repo.strongRef", + "cid": "cids(0)", + "uri": "record(0)", }, - Object { - "action": "com.atproto.admin.defs#acknowledge", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 1, - "reason": "X", - "resolvedReportIds": Array [], - "reversal": Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "reason": "X", - }, - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectBlobCids": Array [], - }, - ], - "currentAction": Object { - "action": "com.atproto.admin.defs#takedown", - "id": 2, + "subjectBlobCids": Array [], + "subjectRepoHandle": "alice.test", + "takendown": true, + "updatedAt": "1970-01-01T00:00:00.000Z", }, - "reports": Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 2, - "reason": "defamation", - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(1)", - "resolvedByActionIds": Array [], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectRepoHandle": "alice.test", - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(2)", - "resolvedByActionIds": Array [], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectRepoHandle": "alice.test", - }, - ], }, "repo": Object { "did": "user(0)", diff --git a/packages/bsky/tests/admin/__snapshots__/get-repo.test.ts.snap b/packages/bsky/tests/admin/__snapshots__/get-repo.test.ts.snap index 1a60b27b069..4ffd7e3564a 100644 --- a/packages/bsky/tests/admin/__snapshots__/get-repo.test.ts.snap +++ b/packages/bsky/tests/admin/__snapshots__/get-repo.test.ts.snap @@ -10,68 +10,22 @@ Object { "invitesDisabled": false, "labels": Array [], "moderation": Object { - "actions": Array [ - Object { - "action": "com.atproto.admin.defs#takedown", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 2, - "reason": "X", - "resolvedReportIds": Array [], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - "subjectBlobCids": Array [], - }, - Object { - "action": "com.atproto.admin.defs#acknowledge", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 1, - "reason": "X", - "resolvedReportIds": Array [], - "reversal": Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "reason": "X", - }, - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - "subjectBlobCids": Array [], + "subjectStatus": Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "id": 1, + "lastReportedAt": "1970-01-01T00:00:00.000Z", + "lastReviewedAt": "1970-01-01T00:00:00.000Z", + "lastReviewedBy": "did:example:admin", + "reviewState": "com.atproto.admin.defs#reviewClosed", + "subject": Object { + "$type": "com.atproto.admin.defs#repoRef", + "did": "user(0)", }, - ], - "currentAction": Object { - "action": "com.atproto.admin.defs#takedown", - "id": 2, + "subjectBlobCids": Array [], + "subjectRepoHandle": "alice.test", + "takendown": true, + "updatedAt": "1970-01-01T00:00:00.000Z", }, - "reports": Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 2, - "reason": "defamation", - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(1)", - "resolvedByActionIds": Array [], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(2)", - "resolvedByActionIds": Array [], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - }, - ], }, "relatedRecords": Array [ Object { diff --git a/packages/bsky/tests/admin/__snapshots__/moderation-events.test.ts.snap b/packages/bsky/tests/admin/__snapshots__/moderation-events.test.ts.snap new file mode 100644 index 00000000000..8fa16b311f2 --- /dev/null +++ b/packages/bsky/tests/admin/__snapshots__/moderation-events.test.ts.snap @@ -0,0 +1,146 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`moderation-events get event gets an event by specific id 1`] = ` +Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "createdBy": "user(2)", + "event": Object { + "$type": "com.atproto.admin.defs#modEventReport", + "comment": "X", + "reportType": "com.atproto.moderation.defs#reasonMisleading", + }, + "id": 1, + "subject": Object { + "$type": "com.atproto.admin.defs#repoView", + "did": "user(0)", + "handle": "alice.test", + "indexedAt": "1970-01-01T00:00:00.000Z", + "moderation": Object { + "subjectStatus": Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "id": 1, + "lastReportedAt": "1970-01-01T00:00:00.000Z", + "lastReviewedAt": "1970-01-01T00:00:00.000Z", + "lastReviewedBy": "user(1)", + "reviewState": "com.atproto.admin.defs#reviewEscalated", + "subject": Object { + "$type": "com.atproto.admin.defs#repoRef", + "did": "user(0)", + }, + "subjectBlobCids": Array [], + "subjectRepoHandle": "alice.test", + "takendown": false, + "updatedAt": "1970-01-01T00:00:00.000Z", + }, + }, + "relatedRecords": Array [ + Object { + "$type": "app.bsky.actor.profile", + "avatar": Object { + "$type": "blob", + "mimeType": "image/jpeg", + "ref": Object { + "$link": "cids(0)", + }, + "size": 3976, + }, + "description": "its me!", + "displayName": "ali", + "labels": Object { + "$type": "com.atproto.label.defs#selfLabels", + "values": Array [ + Object { + "val": "self-label-a", + }, + Object { + "val": "self-label-b", + }, + ], + }, + }, + ], + }, + "subjectBlobCids": Array [], + "subjectBlobs": Array [], +} +`; + +exports[`moderation-events query events returns all events for record or repo 1`] = ` +Array [ + Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "createdBy": "user(1)", + "creatorHandle": "alice.test", + "event": Object { + "$type": "com.atproto.admin.defs#modEventReport", + "comment": "X", + "reportType": "com.atproto.moderation.defs#reasonSpam", + }, + "id": 7, + "subject": Object { + "$type": "com.atproto.admin.defs#repoRef", + "did": "user(0)", + }, + "subjectBlobCids": Array [], + "subjectHandle": "bob.test", + }, + Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "createdBy": "user(1)", + "creatorHandle": "alice.test", + "event": Object { + "$type": "com.atproto.admin.defs#modEventReport", + "comment": "X", + "reportType": "com.atproto.moderation.defs#reasonSpam", + }, + "id": 3, + "subject": Object { + "$type": "com.atproto.admin.defs#repoRef", + "did": "user(0)", + }, + "subjectBlobCids": Array [], + "subjectHandle": "bob.test", + }, +] +`; + +exports[`moderation-events query events returns all events for record or repo 2`] = ` +Array [ + Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "createdBy": "user(0)", + "creatorHandle": "bob.test", + "event": Object { + "$type": "com.atproto.admin.defs#modEventReport", + "comment": "X", + "reportType": "com.atproto.moderation.defs#reasonSpam", + }, + "id": 6, + "subject": Object { + "$type": "com.atproto.repo.strongRef", + "cid": "cids(0)", + "uri": "record(0)", + }, + "subjectBlobCids": Array [], + "subjectHandle": "alice.test", + }, + Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "createdBy": "user(0)", + "creatorHandle": "bob.test", + "event": Object { + "$type": "com.atproto.admin.defs#modEventReport", + "comment": "X", + "reportType": "com.atproto.moderation.defs#reasonSpam", + }, + "id": 2, + "subject": Object { + "$type": "com.atproto.repo.strongRef", + "cid": "cids(0)", + "uri": "record(0)", + }, + "subjectBlobCids": Array [], + "subjectHandle": "alice.test", + }, +] +`; diff --git a/packages/bsky/tests/admin/__snapshots__/moderation-statuses.test.ts.snap b/packages/bsky/tests/admin/__snapshots__/moderation-statuses.test.ts.snap new file mode 100644 index 00000000000..a4939733d1a --- /dev/null +++ b/packages/bsky/tests/admin/__snapshots__/moderation-statuses.test.ts.snap @@ -0,0 +1,64 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`moderation-statuses query statuses returns statuses for subjects that received moderation events 1`] = ` +Array [ + Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "id": 4, + "lastReportedAt": "1970-01-01T00:00:00.000Z", + "reviewState": "com.atproto.admin.defs#reviewOpen", + "subject": Object { + "$type": "com.atproto.repo.strongRef", + "cid": "cids(0)", + "uri": "record(0)", + }, + "subjectBlobCids": Array [], + "subjectRepoHandle": "bob.test", + "takendown": false, + "updatedAt": "1970-01-01T00:00:00.000Z", + }, + Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "id": 3, + "lastReportedAt": "1970-01-01T00:00:00.000Z", + "reviewState": "com.atproto.admin.defs#reviewOpen", + "subject": Object { + "$type": "com.atproto.admin.defs#repoRef", + "did": "user(0)", + }, + "subjectBlobCids": Array [], + "subjectRepoHandle": "bob.test", + "takendown": false, + "updatedAt": "1970-01-01T00:00:00.000Z", + }, + Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "id": 2, + "lastReportedAt": "1970-01-01T00:00:00.000Z", + "reviewState": "com.atproto.admin.defs#reviewOpen", + "subject": Object { + "$type": "com.atproto.repo.strongRef", + "cid": "cids(1)", + "uri": "record(1)", + }, + "subjectBlobCids": Array [], + "subjectRepoHandle": "alice.test", + "takendown": false, + "updatedAt": "1970-01-01T00:00:00.000Z", + }, + Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "id": 1, + "lastReportedAt": "1970-01-01T00:00:00.000Z", + "reviewState": "com.atproto.admin.defs#reviewOpen", + "subject": Object { + "$type": "com.atproto.admin.defs#repoRef", + "did": "user(1)", + }, + "subjectBlobCids": Array [], + "subjectRepoHandle": "alice.test", + "takendown": false, + "updatedAt": "1970-01-01T00:00:00.000Z", + }, +] +`; diff --git a/packages/bsky/tests/admin/__snapshots__/moderation.test.ts.snap b/packages/bsky/tests/admin/__snapshots__/moderation.test.ts.snap index 55f863f6c14..33a973e714f 100644 --- a/packages/bsky/tests/admin/__snapshots__/moderation.test.ts.snap +++ b/packages/bsky/tests/admin/__snapshots__/moderation.test.ts.snap @@ -1,130 +1,5 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`moderation actioning resolves reports on missing repos and records. 1`] = ` -Object { - "recordActionDetail": Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 2, - "reason": "Y", - "resolvedReports": Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 11, - "reason": "defamation", - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(0)", - "resolvedByActionIds": Array [ - 2, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 10, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(1)", - "resolvedByActionIds": Array [ - 2, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(2)", - }, - }, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#recordViewNotFound", - "uri": "record(0)", - }, - "subjectBlobs": Array [], - }, - "reportADetail": Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 10, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(1)", - "resolvedByActions": Array [ - Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 2, - "reason": "Y", - "resolvedReportIds": Array [ - 11, - 10, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectBlobCids": Array [], - }, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#repoViewNotFound", - "did": "user(2)", - }, - }, - "reportBDetail": Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 11, - "reason": "defamation", - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(0)", - "resolvedByActions": Array [ - Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 2, - "reason": "Y", - "resolvedReportIds": Array [ - 11, - 10, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectBlobCids": Array [], - }, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#recordViewNotFound", - "uri": "record(0)", - }, - }, -} -`; - -exports[`moderation actioning resolves reports on repos and records. 1`] = ` -Object { - "action": "com.atproto.admin.defs#takedown", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 1, - "reason": "Y", - "resolvedReportIds": Array [ - 9, - 8, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - "subjectBlobCids": Array [], -} -`; - exports[`moderation reporting creates reports of a record. 1`] = ` Array [ Object { diff --git a/packages/bsky/tests/admin/get-moderation-action.test.ts b/packages/bsky/tests/admin/get-moderation-action.test.ts deleted file mode 100644 index 5c7fe3401db..00000000000 --- a/packages/bsky/tests/admin/get-moderation-action.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { TestNetwork, SeedClient } from '@atproto/dev-env' -import AtpAgent from '@atproto/api' -import { - FLAG, - TAKEDOWN, -} from '@atproto/api/src/client/types/com/atproto/admin/defs' -import { - REASONOTHER, - REASONSPAM, -} from '../../src/lexicon/types/com/atproto/moderation/defs' -import { forSnapshot } from '../_util' -import basicSeed from '../seeds/basic' - -describe('admin get moderation action view', () => { - let network: TestNetwork - let agent: AtpAgent - let sc: SeedClient - - beforeAll(async () => { - network = await TestNetwork.create({ - dbPostgresSchema: 'views_admin_get_moderation_action', - }) - agent = network.pds.getClient() - sc = network.getSeedClient() - await basicSeed(sc) - }) - - afterAll(async () => { - await network.close() - }) - - beforeAll(async () => { - const reportRepo = await sc.createReport({ - reportedBy: sc.dids.bob, - reasonType: REASONSPAM, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.alice, - }, - }) - const reportRecord = await sc.createReport({ - reportedBy: sc.dids.carol, - reasonType: REASONOTHER, - reason: 'defamation', - subject: { - $type: 'com.atproto.repo.strongRef', - uri: sc.posts[sc.dids.alice][0].ref.uriStr, - cid: sc.posts[sc.dids.alice][0].ref.cidStr, - }, - }) - const flagRepo = await sc.takeModerationAction({ - action: FLAG, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.alice, - }, - }) - const takedownRecord = await sc.takeModerationAction({ - action: TAKEDOWN, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: sc.posts[sc.dids.alice][0].ref.uriStr, - cid: sc.posts[sc.dids.alice][0].ref.cidStr, - }, - }) - await sc.resolveReports({ - actionId: flagRepo.id, - reportIds: [reportRepo.id, reportRecord.id], - }) - await sc.resolveReports({ - actionId: takedownRecord.id, - reportIds: [reportRecord.id], - }) - await sc.reverseModerationAction({ id: flagRepo.id }) - }) - - it('gets moderation action for a repo.', async () => { - const result = await agent.api.com.atproto.admin.getModerationAction( - { id: 1 }, - { headers: { authorization: network.pds.adminAuth() } }, - ) - expect(forSnapshot(result.data)).toMatchSnapshot() - }) - - it('gets moderation action for a record.', async () => { - const result = await agent.api.com.atproto.admin.getModerationAction( - { id: 2 }, - { headers: { authorization: network.pds.adminAuth() } }, - ) - expect(forSnapshot(result.data)).toMatchSnapshot() - }) - - it('fails when moderation action does not exist.', async () => { - const promise = agent.api.com.atproto.admin.getModerationAction( - { id: 100 }, - { headers: { authorization: network.pds.adminAuth() } }, - ) - await expect(promise).rejects.toThrow('Action not found') - }) -}) diff --git a/packages/bsky/tests/admin/get-moderation-actions.test.ts b/packages/bsky/tests/admin/get-moderation-actions.test.ts deleted file mode 100644 index dfc08aa82b5..00000000000 --- a/packages/bsky/tests/admin/get-moderation-actions.test.ts +++ /dev/null @@ -1,164 +0,0 @@ -import { SeedClient, TestNetwork } from '@atproto/dev-env' -import AtpAgent from '@atproto/api' -import { - ACKNOWLEDGE, - FLAG, - TAKEDOWN, -} from '@atproto/api/src/client/types/com/atproto/admin/defs' -import { - REASONOTHER, - REASONSPAM, -} from '../../src/lexicon/types/com/atproto/moderation/defs' -import { forSnapshot, paginateAll } from '../_util' -import basicSeed from '../seeds/basic' - -describe('admin get moderation actions view', () => { - let network: TestNetwork - let agent: AtpAgent - let sc: SeedClient - - beforeAll(async () => { - network = await TestNetwork.create({ - dbPostgresSchema: 'views_admin_get_moderation_actions', - }) - agent = network.pds.getClient() - sc = network.getSeedClient() - await basicSeed(sc) - }) - - afterAll(async () => { - await network.close() - }) - - beforeAll(async () => { - const oneIn = (n) => (_, i) => i % n === 0 - const getAction = (i) => [FLAG, ACKNOWLEDGE, TAKEDOWN][i % 3] - const posts = Object.values(sc.posts) - .flatMap((x) => x) - .filter(oneIn(2)) - const dids = Object.values(sc.dids).filter(oneIn(2)) - // Take actions on records - const recordActions: Awaited>[] = - [] - for (let i = 0; i < posts.length; ++i) { - const post = posts[i] - recordActions.push( - await sc.takeModerationAction({ - action: getAction(i), - subject: { - $type: 'com.atproto.repo.strongRef', - uri: post.ref.uriStr, - cid: post.ref.cidStr, - }, - }), - ) - } - // Reverse an action - await sc.reverseModerationAction({ - id: recordActions[0].id, - }) - // Take actions on repos - const repoActions: Awaited>[] = - [] - for (let i = 0; i < dids.length; ++i) { - const did = dids[i] - repoActions.push( - await sc.takeModerationAction({ - action: getAction(i), - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did, - }, - }), - ) - } - // Back some of the actions with a report, possibly resolved - const someRecordActions = recordActions.filter(oneIn(2)) - for (let i = 0; i < someRecordActions.length; ++i) { - const action = someRecordActions[i] - const ab = oneIn(2)(action, i) - const report = await sc.createReport({ - reportedBy: ab ? sc.dids.carol : sc.dids.alice, - reasonType: ab ? REASONSPAM : REASONOTHER, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: action.subject.uri, - cid: action.subject.cid, - }, - }) - if (ab) { - await sc.resolveReports({ - actionId: action.id, - reportIds: [report.id], - }) - } - } - const someRepoActions = repoActions.filter(oneIn(2)) - for (let i = 0; i < someRepoActions.length; ++i) { - const action = someRepoActions[i] - const ab = oneIn(2)(action, i) - const report = await sc.createReport({ - reportedBy: ab ? sc.dids.carol : sc.dids.alice, - reasonType: ab ? REASONSPAM : REASONOTHER, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: action.subject.did, - }, - }) - if (ab) { - await sc.resolveReports({ - actionId: action.id, - reportIds: [report.id], - }) - } - } - }) - - it('gets all moderation actions.', async () => { - const result = await agent.api.com.atproto.admin.getModerationActions( - {}, - { headers: network.pds.adminAuthHeaders() }, - ) - expect(forSnapshot(result.data.actions)).toMatchSnapshot() - }) - - it('gets all moderation actions for a repo.', async () => { - const result = await agent.api.com.atproto.admin.getModerationActions( - { subject: Object.values(sc.dids)[0] }, - { headers: network.pds.adminAuthHeaders() }, - ) - expect(forSnapshot(result.data.actions)).toMatchSnapshot() - }) - - it('gets all moderation actions for a record.', async () => { - const result = await agent.api.com.atproto.admin.getModerationActions( - { subject: Object.values(sc.posts)[0][0].ref.uriStr }, - { headers: network.pds.adminAuthHeaders() }, - ) - expect(forSnapshot(result.data.actions)).toMatchSnapshot() - }) - - it('paginates.', async () => { - const results = (results) => results.flatMap((res) => res.actions) - const paginator = async (cursor?: string) => { - const res = await agent.api.com.atproto.admin.getModerationActions( - { cursor, limit: 3 }, - { headers: network.pds.adminAuthHeaders() }, - ) - return res.data - } - - const paginatedAll = await paginateAll(paginator) - paginatedAll.forEach((res) => - expect(res.actions.length).toBeLessThanOrEqual(3), - ) - - const full = await agent.api.com.atproto.admin.getModerationActions( - {}, - { headers: network.pds.adminAuthHeaders() }, - ) - - expect(full.data.actions.length).toEqual(6) - expect(results(paginatedAll)).toEqual(results([full.data])) - }) -}) diff --git a/packages/bsky/tests/admin/get-moderation-report.test.ts b/packages/bsky/tests/admin/get-moderation-report.test.ts deleted file mode 100644 index 4a77750aa0a..00000000000 --- a/packages/bsky/tests/admin/get-moderation-report.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { SeedClient, TestNetwork } from '@atproto/dev-env' -import AtpAgent from '@atproto/api' -import { - FLAG, - TAKEDOWN, -} from '@atproto/api/src/client/types/com/atproto/admin/defs' -import { - REASONOTHER, - REASONSPAM, -} from '../../src/lexicon/types/com/atproto/moderation/defs' -import { forSnapshot } from '../_util' -import basicSeed from '../seeds/basic' - -describe('admin get moderation action view', () => { - let network: TestNetwork - let agent: AtpAgent - let sc: SeedClient - - beforeAll(async () => { - network = await TestNetwork.create({ - dbPostgresSchema: 'views_admin_get_moderation_report', - }) - agent = network.pds.getClient() - sc = network.getSeedClient() - await basicSeed(sc) - }) - - afterAll(async () => { - await network.close() - }) - - beforeAll(async () => { - const reportRepo = await sc.createReport({ - reportedBy: sc.dids.bob, - reasonType: REASONSPAM, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.alice, - }, - }) - const reportRecord = await sc.createReport({ - reportedBy: sc.dids.carol, - reasonType: REASONOTHER, - reason: 'defamation', - subject: { - $type: 'com.atproto.repo.strongRef', - uri: sc.posts[sc.dids.alice][0].ref.uriStr, - cid: sc.posts[sc.dids.alice][0].ref.cidStr, - }, - }) - const flagRepo = await sc.takeModerationAction({ - action: FLAG, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.alice, - }, - }) - const takedownRecord = await sc.takeModerationAction({ - action: TAKEDOWN, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: sc.posts[sc.dids.alice][0].ref.uriStr, - cid: sc.posts[sc.dids.alice][0].ref.cidStr, - }, - }) - await sc.resolveReports({ - actionId: flagRepo.id, - reportIds: [reportRepo.id, reportRecord.id], - }) - await sc.resolveReports({ - actionId: takedownRecord.id, - reportIds: [reportRecord.id], - }) - await sc.reverseModerationAction({ id: flagRepo.id }) - }) - - it('gets moderation report for a repo.', async () => { - const result = await agent.api.com.atproto.admin.getModerationReport( - { id: 1 }, - { headers: network.pds.adminAuthHeaders() }, - ) - expect(forSnapshot(result.data)).toMatchSnapshot() - }) - - it('gets moderation report for a record.', async () => { - const result = await agent.api.com.atproto.admin.getModerationReport( - { id: 2 }, - { headers: network.pds.adminAuthHeaders() }, - ) - expect(forSnapshot(result.data)).toMatchSnapshot() - }) - - it('fails when moderation report does not exist.', async () => { - const promise = agent.api.com.atproto.admin.getModerationReport( - { id: 100 }, - { headers: network.pds.adminAuthHeaders() }, - ) - await expect(promise).rejects.toThrow('Report not found') - }) -}) diff --git a/packages/bsky/tests/admin/get-moderation-reports.test.ts b/packages/bsky/tests/admin/get-moderation-reports.test.ts deleted file mode 100644 index 64313130047..00000000000 --- a/packages/bsky/tests/admin/get-moderation-reports.test.ts +++ /dev/null @@ -1,332 +0,0 @@ -import { SeedClient, TestNetwork } from '@atproto/dev-env' -import AtpAgent from '@atproto/api' -import { - ACKNOWLEDGE, - FLAG, - TAKEDOWN, -} from '@atproto/api/src/client/types/com/atproto/admin/defs' -import { - REASONOTHER, - REASONSPAM, -} from '../../src/lexicon/types/com/atproto/moderation/defs' -import { forSnapshot, paginateAll } from '../_util' -import basicSeed from '../seeds/basic' - -describe('admin get moderation reports view', () => { - let network: TestNetwork - let agent: AtpAgent - let sc: SeedClient - - beforeAll(async () => { - network = await TestNetwork.create({ - dbPostgresSchema: 'views_admin_get_moderation_reports', - }) - agent = network.pds.getClient() - sc = network.getSeedClient() - await basicSeed(sc) - }) - - afterAll(async () => { - await network.close() - }) - - beforeAll(async () => { - const oneIn = (n) => (_, i) => i % n === 0 - const getAction = (i) => [FLAG, ACKNOWLEDGE, TAKEDOWN][i % 3] - const getReasonType = (i) => [REASONOTHER, REASONSPAM][i % 2] - const getReportedByDid = (i) => [sc.dids.alice, sc.dids.carol][i % 2] - const posts = Object.values(sc.posts) - .flatMap((x) => x) - .filter(oneIn(2)) - const dids = Object.values(sc.dids).filter(oneIn(2)) - const recordReports: Awaited>[] = [] - for (let i = 0; i < posts.length; ++i) { - const post = posts[i] - recordReports.push( - await sc.createReport({ - reasonType: getReasonType(i), - reportedBy: getReportedByDid(i), - subject: { - $type: 'com.atproto.repo.strongRef', - uri: post.ref.uriStr, - cid: post.ref.cidStr, - }, - }), - ) - } - const repoReports: Awaited>[] = [] - for (let i = 0; i < dids.length; ++i) { - const did = dids[i] - repoReports.push( - await sc.createReport({ - reasonType: getReasonType(i), - reportedBy: getReportedByDid(i), - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did, - }, - }), - ) - } - for (let i = 0; i < recordReports.length; ++i) { - const report = recordReports[i] - const ab = oneIn(2)(report, i) - const action = await sc.takeModerationAction({ - action: getAction(i), - subject: { - $type: 'com.atproto.repo.strongRef', - uri: report.subject.uri, - cid: report.subject.cid, - }, - createdBy: `did:example:admin${i}`, - }) - if (ab) { - await sc.resolveReports({ - actionId: action.id, - reportIds: [report.id], - }) - } else { - await sc.reverseModerationAction({ - id: action.id, - }) - } - } - for (let i = 0; i < repoReports.length; ++i) { - const report = repoReports[i] - const ab = oneIn(2)(report, i) - const action = await sc.takeModerationAction({ - action: getAction(i), - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: report.subject.did, - }, - }) - if (ab) { - await sc.resolveReports({ - actionId: action.id, - reportIds: [report.id], - }) - } else { - await sc.reverseModerationAction({ - id: action.id, - }) - } - } - }) - - it('ignores subjects when specified.', async () => { - // Get all reports and then make another request with a filter to ignore some subject dids - // and assert that the reports for those subject dids are ignored in the result set - const getDids = (reportsResponse) => - reportsResponse.data.reports - .map((report) => report.subject.did) - // Not all reports contain a did so we're discarding the undefined values in the mapped array - .filter(Boolean) - - const allReports = await agent.api.com.atproto.admin.getModerationReports( - {}, - { headers: network.pds.adminAuthHeaders() }, - ) - - const ignoreSubjects = getDids(allReports).slice(0, 2) - - const filteredReportsByDid = - await agent.api.com.atproto.admin.getModerationReports( - { ignoreSubjects }, - { headers: network.pds.adminAuthHeaders() }, - ) - - // Validate that when ignored by DID, all reports for that DID is ignored - getDids(filteredReportsByDid).forEach((resultDid) => - expect(ignoreSubjects).not.toContain(resultDid), - ) - - const ignoredAtUriSubjects: string[] = [ - `${ - allReports.data.reports.find(({ subject }) => !!subject.uri)?.subject - ?.uri - }`, - ] - const filteredReportsByAtUri = - await agent.api.com.atproto.admin.getModerationReports( - { - ignoreSubjects: ignoredAtUriSubjects, - }, - { headers: network.pds.adminAuthHeaders() }, - ) - - // Validate that when ignored by at uri, only the reports for that at uri is ignored - expect(filteredReportsByAtUri.data.reports.length).toEqual( - allReports.data.reports.length - 1, - ) - expect( - filteredReportsByAtUri.data.reports - .map(({ subject }) => subject.uri) - .filter(Boolean), - ).not.toContain(ignoredAtUriSubjects[0]) - }) - - it('gets all moderation reports.', async () => { - const result = await agent.api.com.atproto.admin.getModerationReports( - {}, - { headers: network.pds.adminAuthHeaders() }, - ) - expect(forSnapshot(result.data.reports)).toMatchSnapshot() - }) - - it('gets all moderation reports for a repo.', async () => { - const result = await agent.api.com.atproto.admin.getModerationReports( - { subject: Object.values(sc.dids)[0] }, - { headers: network.pds.adminAuthHeaders() }, - ) - expect(forSnapshot(result.data.reports)).toMatchSnapshot() - }) - - it('gets all moderation reports for a record.', async () => { - const result = await agent.api.com.atproto.admin.getModerationReports( - { subject: Object.values(sc.posts)[0][0].ref.uriStr }, - { headers: network.pds.adminAuthHeaders() }, - ) - expect(forSnapshot(result.data.reports)).toMatchSnapshot() - }) - - it('gets all resolved/unresolved moderation reports.', async () => { - const resolved = await agent.api.com.atproto.admin.getModerationReports( - { resolved: true }, - { headers: network.pds.adminAuthHeaders() }, - ) - expect(forSnapshot(resolved.data.reports)).toMatchSnapshot() - const unresolved = await agent.api.com.atproto.admin.getModerationReports( - { resolved: false }, - { headers: network.pds.adminAuthHeaders() }, - ) - expect(forSnapshot(unresolved.data.reports)).toMatchSnapshot() - }) - - it('allows reverting the order of reports.', async () => { - const [ - { - data: { reports: reverseList }, - }, - { - data: { reports: defaultList }, - }, - ] = await Promise.all([ - agent.api.com.atproto.admin.getModerationReports( - { reverse: true }, - { headers: network.pds.adminAuthHeaders() }, - ), - agent.api.com.atproto.admin.getModerationReports( - {}, - { headers: network.pds.adminAuthHeaders() }, - ), - ]) - - expect(defaultList[0].id).toEqual(reverseList[reverseList.length - 1].id) - expect(defaultList[defaultList.length - 1].id).toEqual(reverseList[0].id) - }) - - it('gets all moderation reports by active resolution action type.', async () => { - const reportsWithTakedown = - await agent.api.com.atproto.admin.getModerationReports( - { actionType: TAKEDOWN }, - { headers: network.pds.adminAuthHeaders() }, - ) - expect(forSnapshot(reportsWithTakedown.data.reports)).toMatchSnapshot() - }) - - it('gets all moderation reports actioned by a certain moderator.', async () => { - const adminDidOne = 'did:example:admin0' - const adminDidTwo = 'did:example:admin2' - const [actionedByAdminOne, actionedByAdminTwo] = await Promise.all([ - agent.api.com.atproto.admin.getModerationReports( - { actionedBy: adminDidOne }, - { headers: network.pds.adminAuthHeaders() }, - ), - agent.api.com.atproto.admin.getModerationReports( - { actionedBy: adminDidTwo }, - { headers: network.pds.adminAuthHeaders() }, - ), - ]) - const [fullReportOne, fullReportTwo] = await Promise.all([ - agent.api.com.atproto.admin.getModerationReport( - { id: actionedByAdminOne.data.reports[0].id }, - { headers: network.pds.adminAuthHeaders() }, - ), - agent.api.com.atproto.admin.getModerationReport( - { id: actionedByAdminTwo.data.reports[0].id }, - { headers: network.pds.adminAuthHeaders() }, - ), - ]) - - expect(forSnapshot(actionedByAdminOne.data.reports)).toMatchSnapshot() - expect(fullReportOne.data.resolvedByActions[0].createdBy).toEqual( - adminDidOne, - ) - expect(forSnapshot(actionedByAdminTwo.data.reports)).toMatchSnapshot() - expect(fullReportTwo.data.resolvedByActions[0].createdBy).toEqual( - adminDidTwo, - ) - }) - - it('paginates.', async () => { - const results = (results) => results.flatMap((res) => res.reports) - const paginator = async (cursor?: string) => { - const res = await agent.api.com.atproto.admin.getModerationReports( - { cursor, limit: 3 }, - { headers: network.pds.adminAuthHeaders() }, - ) - return res.data - } - - const paginatedAll = await paginateAll(paginator) - paginatedAll.forEach((res) => - expect(res.reports.length).toBeLessThanOrEqual(3), - ) - - const full = await agent.api.com.atproto.admin.getModerationReports( - {}, - { headers: network.pds.adminAuthHeaders() }, - ) - - expect(full.data.reports.length).toEqual(6) - expect(results(paginatedAll)).toEqual(results([full.data])) - }) - - it('paginates reverted list of reports.', async () => { - const paginator = - (reverse = false) => - async (cursor?: string) => { - const res = await agent.api.com.atproto.admin.getModerationReports( - { cursor, limit: 3, reverse }, - { headers: network.pds.adminAuthHeaders() }, - ) - return res.data - } - - const [reverseResponse, defaultResponse] = await Promise.all([ - paginateAll(paginator(true)), - paginateAll(paginator()), - ]) - - const reverseList = reverseResponse.flatMap((res) => res.reports) - const defaultList = defaultResponse.flatMap((res) => res.reports) - - expect(defaultList[0].id).toEqual(reverseList[reverseList.length - 1].id) - expect(defaultList[defaultList.length - 1].id).toEqual(reverseList[0].id) - }) - - it('filters reports by reporter DID.', async () => { - const result = await agent.api.com.atproto.admin.getModerationReports( - { reporters: [sc.dids.alice] }, - { headers: network.pds.adminAuthHeaders() }, - ) - - const reporterDidsFromReports = [ - ...new Set(result.data.reports.map(({ reportedBy }) => reportedBy)), - ] - - expect(reporterDidsFromReports.length).toEqual(1) - expect(reporterDidsFromReports[0]).toEqual(sc.dids.alice) - }) -}) diff --git a/packages/bsky/tests/admin/get-record.test.ts b/packages/bsky/tests/admin/get-record.test.ts index 94ae22b1694..3807724fa6c 100644 --- a/packages/bsky/tests/admin/get-record.test.ts +++ b/packages/bsky/tests/admin/get-record.test.ts @@ -1,10 +1,6 @@ import { SeedClient, TestNetwork } from '@atproto/dev-env' import AtpAgent from '@atproto/api' import { AtUri } from '@atproto/syntax' -import { - ACKNOWLEDGE, - TAKEDOWN, -} from '@atproto/api/src/client/types/com/atproto/admin/defs' import { REASONOTHER, REASONSPAM, @@ -24,6 +20,7 @@ describe('admin get record view', () => { agent = network.pds.getClient() sc = network.getSeedClient() await basicSeed(sc) + await network.processAll() }) afterAll(async () => { @@ -31,8 +28,8 @@ describe('admin get record view', () => { }) beforeAll(async () => { - const acknowledge = await sc.takeModerationAction({ - action: ACKNOWLEDGE, + await sc.emitModerationEvent({ + event: { $type: 'com.atproto.admin.defs#modEventFlag' }, subject: { $type: 'com.atproto.repo.strongRef', uri: sc.posts[sc.dids.alice][0].ref.uriStr, @@ -58,9 +55,8 @@ describe('admin get record view', () => { cid: sc.posts[sc.dids.alice][0].ref.cidStr, }, }) - await sc.reverseModerationAction({ id: acknowledge.id }) - await sc.takeModerationAction({ - action: TAKEDOWN, + await sc.emitModerationEvent({ + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, subject: { $type: 'com.atproto.repo.strongRef', uri: sc.posts[sc.dids.alice][0].ref.uriStr, diff --git a/packages/bsky/tests/admin/get-repo.test.ts b/packages/bsky/tests/admin/get-repo.test.ts index 9b4f6690ccd..dbd143bb2ac 100644 --- a/packages/bsky/tests/admin/get-repo.test.ts +++ b/packages/bsky/tests/admin/get-repo.test.ts @@ -1,9 +1,5 @@ import { SeedClient, TestNetwork } from '@atproto/dev-env' import AtpAgent from '@atproto/api' -import { - ACKNOWLEDGE, - TAKEDOWN, -} from '@atproto/api/src/client/types/com/atproto/admin/defs' import { REASONOTHER, REASONSPAM, @@ -23,6 +19,7 @@ describe('admin get repo view', () => { agent = network.pds.getClient() sc = network.getSeedClient() await basicSeed(sc) + await network.processAll() }) afterAll(async () => { @@ -30,8 +27,8 @@ describe('admin get repo view', () => { }) beforeAll(async () => { - const acknowledge = await sc.takeModerationAction({ - action: ACKNOWLEDGE, + await sc.emitModerationEvent({ + event: { $type: 'com.atproto.admin.defs#modEventAcknowledge' }, subject: { $type: 'com.atproto.admin.defs#repoRef', did: sc.dids.alice, @@ -54,9 +51,8 @@ describe('admin get repo view', () => { did: sc.dids.alice, }, }) - await sc.reverseModerationAction({ id: acknowledge.id }) - await sc.takeModerationAction({ - action: TAKEDOWN, + await sc.emitModerationEvent({ + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, subject: { $type: 'com.atproto.admin.defs#repoRef', did: sc.dids.alice, diff --git a/packages/bsky/tests/admin/moderation-events.test.ts b/packages/bsky/tests/admin/moderation-events.test.ts new file mode 100644 index 00000000000..174167034db --- /dev/null +++ b/packages/bsky/tests/admin/moderation-events.test.ts @@ -0,0 +1,221 @@ +import { TestNetwork, SeedClient } from '@atproto/dev-env' +import AtpAgent, { ComAtprotoAdminDefs } from '@atproto/api' +import { forSnapshot } from '../_util' +import basicSeed from '../seeds/basic' +import { + REASONMISLEADING, + REASONSPAM, +} from '../../src/lexicon/types/com/atproto/moderation/defs' + +describe('moderation-events', () => { + let network: TestNetwork + let agent: AtpAgent + let pdsAgent: AtpAgent + let sc: SeedClient + + const emitModerationEvent = async (eventData) => { + return pdsAgent.api.com.atproto.admin.emitModerationEvent(eventData, { + encoding: 'application/json', + headers: network.bsky.adminAuthHeaders('moderator'), + }) + } + + const queryModerationEvents = (eventQuery) => + agent.api.com.atproto.admin.queryModerationEvents(eventQuery, { + headers: network.bsky.adminAuthHeaders('moderator'), + }) + + const seedEvents = async () => { + const bobsAccount = { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.bob, + } + const alicesAccount = { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.alice, + } + const bobsPost = { + $type: 'com.atproto.repo.strongRef', + uri: sc.posts[sc.dids.bob][0].ref.uriStr, + cid: sc.posts[sc.dids.bob][0].ref.cidStr, + } + const alicesPost = { + $type: 'com.atproto.repo.strongRef', + uri: sc.posts[sc.dids.alice][0].ref.uriStr, + cid: sc.posts[sc.dids.alice][0].ref.cidStr, + } + + for (let i = 0; i < 4; i++) { + await emitModerationEvent({ + event: { + $type: 'com.atproto.admin.defs#modEventReport', + reportType: i % 2 ? REASONSPAM : REASONMISLEADING, + comment: 'X', + }, + // Report bob's account by alice and vice versa + subject: i % 2 ? bobsAccount : alicesAccount, + createdBy: i % 2 ? sc.dids.alice : sc.dids.bob, + }) + await emitModerationEvent({ + event: { + $type: 'com.atproto.admin.defs#modEventReport', + reportType: REASONSPAM, + comment: 'X', + }, + // Report bob's post by alice and vice versa + subject: i % 2 ? bobsPost : alicesPost, + createdBy: i % 2 ? sc.dids.alice : sc.dids.bob, + }) + } + } + + beforeAll(async () => { + network = await TestNetwork.create({ + dbPostgresSchema: 'bsky_moderation_events', + }) + agent = network.bsky.getClient() + pdsAgent = network.pds.getClient() + sc = network.getSeedClient() + await basicSeed(sc) + await network.processAll() + await seedEvents() + }) + + afterAll(async () => { + await network.close() + }) + + describe('query events', () => { + it('returns all events for record or repo', async () => { + const [bobsEvents, alicesPostEvents] = await Promise.all([ + queryModerationEvents({ + subject: sc.dids.bob, + }), + queryModerationEvents({ + subject: sc.posts[sc.dids.alice][0].ref.uriStr, + }), + ]) + + expect(forSnapshot(bobsEvents.data.events)).toMatchSnapshot() + expect(forSnapshot(alicesPostEvents.data.events)).toMatchSnapshot() + }) + + it('filters events by types', async () => { + const alicesAccount = { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.alice, + } + await Promise.all([ + emitModerationEvent({ + event: { + $type: 'com.atproto.admin.defs#modEventComment', + comment: 'X', + }, + subject: alicesAccount, + createdBy: 'did:plc:moderator', + }), + emitModerationEvent({ + event: { + $type: 'com.atproto.admin.defs#modEventEscalate', + comment: 'X', + }, + subject: alicesAccount, + createdBy: 'did:plc:moderator', + }), + ]) + const [allEvents, reportEvents] = await Promise.all([ + queryModerationEvents({ + subject: sc.dids.alice, + }), + queryModerationEvents({ + subject: sc.dids.alice, + types: ['com.atproto.admin.defs#modEventReport'], + }), + ]) + + expect(allEvents.data.events.length).toBeGreaterThan( + reportEvents.data.events.length, + ) + expect( + [...new Set(reportEvents.data.events.map((e) => e.event.$type))].length, + ).toEqual(1) + + expect( + [...new Set(allEvents.data.events.map((e) => e.event.$type))].length, + ).toEqual(3) + }) + + it('returns events for all content by user', async () => { + const [forAccount, forPost] = await Promise.all([ + queryModerationEvents({ + subject: sc.dids.bob, + includeAllUserRecords: true, + }), + queryModerationEvents({ + subject: sc.posts[sc.dids.bob][0].ref.uriStr, + includeAllUserRecords: true, + }), + ]) + + expect(forAccount.data.events.length).toEqual(forPost.data.events.length) + // Save events are returned from both requests + expect(forPost.data.events.map(({ id }) => id).sort()).toEqual( + forAccount.data.events.map(({ id }) => id).sort(), + ) + }) + + it('returns paginated list of events with cursor', async () => { + const allEvents = await queryModerationEvents({ + subject: sc.dids.bob, + includeAllUserRecords: true, + }) + + const getPaginatedEvents = async ( + sortDirection: 'asc' | 'desc' = 'desc', + ) => { + let defaultCursor: undefined | string = undefined + const events: ComAtprotoAdminDefs.ModEventView[] = [] + let count = 0 + do { + // get 1 event at a time and check we get all events + const { data } = await queryModerationEvents({ + limit: 1, + subject: sc.dids.bob, + includeAllUserRecords: true, + cursor: defaultCursor, + sortDirection, + }) + events.push(...data.events) + defaultCursor = data.cursor + count++ + // The count is a circuit breaker to prevent infinite loop in case of failing test + } while (defaultCursor && count < 10) + + return events + } + + const defaultEvents = await getPaginatedEvents() + const reversedEvents = await getPaginatedEvents('asc') + + expect(allEvents.data.events.length).toEqual(4) + expect(defaultEvents.length).toEqual(allEvents.data.events.length) + expect(reversedEvents.length).toEqual(allEvents.data.events.length) + expect(reversedEvents[0].id).toEqual(defaultEvents[3].id) + }) + }) + + describe('get event', () => { + it('gets an event by specific id', async () => { + const { data } = await pdsAgent.api.com.atproto.admin.getModerationEvent( + { + id: 1, + }, + { + headers: network.bsky.adminAuthHeaders('moderator'), + }, + ) + + expect(forSnapshot(data)).toMatchSnapshot() + }) + }) +}) diff --git a/packages/bsky/tests/admin/moderation-statuses.test.ts b/packages/bsky/tests/admin/moderation-statuses.test.ts new file mode 100644 index 00000000000..5109cc43b0e --- /dev/null +++ b/packages/bsky/tests/admin/moderation-statuses.test.ts @@ -0,0 +1,145 @@ +import { TestNetwork, SeedClient } from '@atproto/dev-env' +import AtpAgent, { + ComAtprotoAdminDefs, + ComAtprotoAdminQueryModerationStatuses, +} from '@atproto/api' +import { forSnapshot } from '../_util' +import basicSeed from '../seeds/basic' +import { + REASONMISLEADING, + REASONSPAM, +} from '../../src/lexicon/types/com/atproto/moderation/defs' + +describe('moderation-statuses', () => { + let network: TestNetwork + let agent: AtpAgent + let pdsAgent: AtpAgent + let sc: SeedClient + + const emitModerationEvent = async (eventData) => { + return pdsAgent.api.com.atproto.admin.emitModerationEvent(eventData, { + encoding: 'application/json', + headers: network.bsky.adminAuthHeaders('moderator'), + }) + } + + const queryModerationStatuses = (statusQuery) => + agent.api.com.atproto.admin.queryModerationStatuses(statusQuery, { + headers: network.bsky.adminAuthHeaders('moderator'), + }) + + const seedEvents = async () => { + const bobsAccount = { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.bob, + } + const carlasAccount = { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.alice, + } + const bobsPost = { + $type: 'com.atproto.repo.strongRef', + uri: sc.posts[sc.dids.bob][1].ref.uriStr, + cid: sc.posts[sc.dids.bob][1].ref.cidStr, + } + const alicesPost = { + $type: 'com.atproto.repo.strongRef', + uri: sc.posts[sc.dids.alice][1].ref.uriStr, + cid: sc.posts[sc.dids.alice][1].ref.cidStr, + } + + for (let i = 0; i < 4; i++) { + await emitModerationEvent({ + event: { + $type: 'com.atproto.admin.defs#modEventReport', + reportType: i % 2 ? REASONSPAM : REASONMISLEADING, + comment: 'X', + }, + // Report bob's account by alice and vice versa + subject: i % 2 ? bobsAccount : carlasAccount, + createdBy: i % 2 ? sc.dids.alice : sc.dids.bob, + }) + await emitModerationEvent({ + event: { + $type: 'com.atproto.admin.defs#modEventReport', + reportType: REASONSPAM, + comment: 'X', + }, + // Report bob's post by alice and vice versa + subject: i % 2 ? bobsPost : alicesPost, + createdBy: i % 2 ? sc.dids.alice : sc.dids.bob, + }) + } + } + + beforeAll(async () => { + network = await TestNetwork.create({ + dbPostgresSchema: 'bsky_moderation_statuses', + }) + agent = network.bsky.getClient() + pdsAgent = network.pds.getClient() + sc = network.getSeedClient() + await basicSeed(sc) + await network.processAll() + await seedEvents() + }) + + afterAll(async () => { + await network.close() + }) + + describe('query statuses', () => { + it('returns statuses for subjects that received moderation events', async () => { + const response = await queryModerationStatuses({}) + + expect(forSnapshot(response.data.subjectStatuses)).toMatchSnapshot() + }) + + it('returns paginated statuses', async () => { + // We know there will be exactly 4 statuses in db + const getPaginatedStatuses = async ( + params: ComAtprotoAdminQueryModerationStatuses.QueryParams, + ) => { + let cursor: string | undefined = '' + const statuses: ComAtprotoAdminDefs.SubjectStatusView[] = [] + let count = 0 + do { + const results = await queryModerationStatuses({ + limit: 1, + cursor, + ...params, + }) + cursor = results.data.cursor + statuses.push(...results.data.subjectStatuses) + count++ + // The count is just a brake-check to prevent infinite loop + } while (cursor && count < 10) + + return statuses + } + + const list = await getPaginatedStatuses({}) + expect(list[0].id).toEqual(4) + expect(list[list.length - 1].id).toEqual(1) + + await emitModerationEvent({ + subject: list[1].subject, + event: { + $type: 'com.atproto.admin.defs#modEventAcknowledge', + comment: 'X', + }, + createdBy: sc.dids.bob, + }) + + const listReviewedFirst = await getPaginatedStatuses({ + sortDirection: 'desc', + sortField: 'lastReviewedAt', + }) + + // Verify that the item that was recently reviewed comes up first when sorted descendingly + // while the result set always contains same number of items regardless of sorting + expect(listReviewedFirst[0].id).toEqual(list[1].id) + expect(listReviewedFirst.length).toEqual(list.length) + }) + }) +}) diff --git a/packages/bsky/tests/admin/moderation.test.ts b/packages/bsky/tests/admin/moderation.test.ts index 05200087e3c..5f7fea32c3a 100644 --- a/packages/bsky/tests/admin/moderation.test.ts +++ b/packages/bsky/tests/admin/moderation.test.ts @@ -1,20 +1,34 @@ import { TestNetwork, ImageRef, RecordRef, SeedClient } from '@atproto/dev-env' -import { TID, cidForCbor } from '@atproto/common' -import AtpAgent, { ComAtprotoAdminTakeModerationAction } from '@atproto/api' +import AtpAgent, { + ComAtprotoAdminEmitModerationEvent, + ComAtprotoAdminQueryModerationStatuses, + ComAtprotoModerationCreateReport, +} from '@atproto/api' import { AtUri } from '@atproto/syntax' import { forSnapshot } from '../_util' import basicSeed from '../seeds/basic' import { - ACKNOWLEDGE, - ESCALATE, - FLAG, - TAKEDOWN, -} from '../../src/lexicon/types/com/atproto/admin/defs' -import { + REASONMISLEADING, REASONOTHER, REASONSPAM, } from '../../src/lexicon/types/com/atproto/moderation/defs' -import { PeriodicModerationActionReversal } from '../../src' +import { + ModEventLabel, + ModEventTakedown, + REVIEWCLOSED, + REVIEWESCALATED, +} from '../../src/lexicon/types/com/atproto/admin/defs' +import { PeriodicModerationEventReversal } from '../../src' + +type BaseCreateReportParams = + | { account: string } + | { content: { uri: string; cid: string } } +type CreateReportParams = BaseCreateReportParams & { + author: string +} & Omit + +type TakedownParams = BaseCreateReportParams & + Omit describe('moderation', () => { let network: TestNetwork @@ -22,6 +36,99 @@ describe('moderation', () => { let pdsAgent: AtpAgent let sc: SeedClient + const createReport = async (params: CreateReportParams) => { + const { author, ...rest } = params + return agent.api.com.atproto.moderation.createReport( + { + // Set default type to spam + reasonType: REASONSPAM, + ...rest, + subject: + 'account' in params + ? { + $type: 'com.atproto.admin.defs#repoRef', + did: params.account, + } + : { + $type: 'com.atproto.repo.strongRef', + uri: params.content.uri, + cid: params.content.cid, + }, + }, + { + headers: await network.serviceHeaders(author), + encoding: 'application/json', + }, + ) + } + + const performTakedown = async ({ + durationInHours, + ...rest + }: TakedownParams & Pick) => + agent.api.com.atproto.admin.emitModerationEvent( + { + event: { + $type: 'com.atproto.admin.defs#modEventTakedown', + durationInHours, + }, + subject: + 'account' in rest + ? { + $type: 'com.atproto.admin.defs#repoRef', + did: rest.account, + } + : { + $type: 'com.atproto.repo.strongRef', + uri: rest.content.uri, + cid: rest.content.cid, + }, + createdBy: 'did:example:admin', + ...rest, + }, + { + encoding: 'application/json', + headers: network.bsky.adminAuthHeaders(), + }, + ) + + const performReverseTakedown = async (params: TakedownParams) => + agent.api.com.atproto.admin.emitModerationEvent( + { + event: { + $type: 'com.atproto.admin.defs#modEventReverseTakedown', + }, + subject: + 'account' in params + ? { + $type: 'com.atproto.admin.defs#repoRef', + did: params.account, + } + : { + $type: 'com.atproto.repo.strongRef', + uri: params.content.uri, + cid: params.content.cid, + }, + createdBy: 'did:example:admin', + ...params, + }, + { + encoding: 'application/json', + headers: network.bsky.adminAuthHeaders(), + }, + ) + + const getStatuses = async ( + params: ComAtprotoAdminQueryModerationStatuses.QueryParams, + ) => { + const { data } = await agent.api.com.atproto.admin.queryModerationStatuses( + params, + { headers: network.bsky.adminAuthHeaders() }, + ) + + return data + } + beforeAll(async () => { network = await TestNetwork.create({ dbPostgresSchema: 'bsky_moderation', @@ -39,89 +146,51 @@ describe('moderation', () => { describe('reporting', () => { it('creates reports of a repo.', async () => { - const { data: reportA } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONSPAM, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, - }, - { - headers: await network.serviceHeaders(sc.dids.alice), - encoding: 'application/json', - }, - ) - const { data: reportB } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONOTHER, - reason: 'impersonation', - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, - }, - { - headers: await network.serviceHeaders(sc.dids.carol), - encoding: 'application/json', - }, - ) + const { data: reportA } = await createReport({ + reasonType: REASONSPAM, + account: sc.dids.bob, + author: sc.dids.alice, + }) + const { data: reportB } = await createReport({ + reasonType: REASONOTHER, + reason: 'impersonation', + account: sc.dids.bob, + author: sc.dids.carol, + }) expect(forSnapshot([reportA, reportB])).toMatchSnapshot() }) it("allows reporting a repo that doesn't exist.", async () => { - const promise = agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONSPAM, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: 'did:plc:unknown', - }, - }, - { - headers: await network.serviceHeaders(sc.dids.alice), - encoding: 'application/json', - }, - ) + const promise = createReport({ + reasonType: REASONSPAM, + account: 'did:plc:unknown', + author: sc.dids.alice, + }) await expect(promise).resolves.toBeDefined() }) it('creates reports of a record.', async () => { const postA = sc.posts[sc.dids.bob][0].ref const postB = sc.posts[sc.dids.bob][1].ref - const { data: reportA } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONSPAM, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postA.uriStr, - cid: postA.cidStr, - }, - }, - { - headers: await network.serviceHeaders(sc.dids.alice), - encoding: 'application/json', - }, - ) - const { data: reportB } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONOTHER, - reason: 'defamation', - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postB.uriStr, - cid: postB.cidStr, - }, - }, - { - headers: await network.serviceHeaders(sc.dids.carol), - encoding: 'application/json', - }, - ) + const { data: reportA } = await createReport({ + author: sc.dids.alice, + reasonType: REASONSPAM, + content: { + $type: 'com.atproto.repo.strongRef', + uri: postA.uriStr, + cid: postA.cidStr, + }, + }) + const { data: reportB } = await createReport({ + reasonType: REASONOTHER, + reason: 'defamation', + content: { + $type: 'com.atproto.repo.strongRef', + uri: postB.uriStr, + cid: postB.cidStr, + }, + author: sc.dids.carol, + }) expect(forSnapshot([reportA, reportB])).toMatchSnapshot() }) @@ -131,682 +200,238 @@ describe('moderation', () => { const postUriBad = new AtUri(postA.uriStr) postUriBad.rkey = 'badrkey' - const promiseA = agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONSPAM, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postUriBad.toString(), - cid: postA.cidStr, - }, - }, - { - headers: await network.serviceHeaders(sc.dids.alice), - encoding: 'application/json', + const promiseA = createReport({ + reasonType: REASONSPAM, + content: { + $type: 'com.atproto.repo.strongRef', + uri: postUriBad.toString(), + cid: postA.cidStr, }, - ) + author: sc.dids.alice, + }) await expect(promiseA).resolves.toBeDefined() - const promiseB = agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONOTHER, - reason: 'defamation', - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postB.uri.toString(), - cid: postA.cidStr, // bad cid - }, - }, - { - headers: await network.serviceHeaders(sc.dids.carol), - encoding: 'application/json', + const promiseB = createReport({ + reasonType: REASONOTHER, + reason: 'defamation', + content: { + $type: 'com.atproto.repo.strongRef', + uri: postB.uri.toString(), + cid: postA.cidStr, // bad cid }, - ) + author: sc.dids.carol, + }) await expect(promiseB).resolves.toBeDefined() }) }) describe('actioning', () => { it('resolves reports on repos and records.', async () => { - const { data: reportA } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONSPAM, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, - }, - { - headers: await network.serviceHeaders(sc.dids.alice), - encoding: 'application/json', - }, - ) const post = sc.posts[sc.dids.bob][1].ref - const { data: reportB } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONOTHER, - reason: 'defamation', - subject: { - $type: 'com.atproto.repo.strongRef', - uri: post.uri.toString(), - cid: post.cid.toString(), - }, - }, - { - headers: await network.serviceHeaders(sc.dids.carol), - encoding: 'application/json', - }, - ) - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - const { data: actionResolvedReports } = - await agent.api.com.atproto.admin.resolveModerationReports( - { - actionId: action.id, - reportIds: [reportB.id, reportA.id], - createdBy: 'did:example:admin', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - - expect(forSnapshot(actionResolvedReports)).toMatchSnapshot() - - // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: action.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - }) - it('resolves reports on missing repos and records.', async () => { - const unknownDid = 'did:plc:unknown' - const unknownPostUri = `at://did:plc:unknown/app.bsky.feed.post/${TID.nextStr()}` - const unknownPostCid = (await cidForCbor({})).toString() - // Report user and post unknown to bsky - const { data: reportA } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONSPAM, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: unknownDid, - }, - }, - { - headers: await network.serviceHeaders(sc.dids.alice), - encoding: 'application/json', - }, - ) - const { data: reportB } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONOTHER, - reason: 'defamation', - subject: { - $type: 'com.atproto.repo.strongRef', - uri: unknownPostUri, - cid: unknownPostCid, - }, - }, - { - headers: await network.serviceHeaders(sc.dids.carol), - encoding: 'application/json', - }, - ) - // Take action on deleted content - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: FLAG, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: unknownPostUri, - cid: unknownPostCid, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - await agent.api.com.atproto.admin.resolveModerationReports( - { - actionId: action.id, - reportIds: [reportB.id, reportA.id], - createdBy: 'did:example:admin', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - // Check report and action details - const { data: recordActionDetail } = - await agent.api.com.atproto.admin.getModerationAction( - { id: action.id }, - { headers: network.bsky.adminAuthHeaders() }, - ) - const { data: reportADetail } = - await agent.api.com.atproto.admin.getModerationReport( - { id: reportA.id }, - { headers: network.bsky.adminAuthHeaders() }, - ) - const { data: reportBDetail } = - await agent.api.com.atproto.admin.getModerationReport( - { id: reportB.id }, - { headers: network.bsky.adminAuthHeaders() }, - ) - expect( - forSnapshot({ - recordActionDetail, - reportADetail, - reportBDetail, - }), - ).toMatchSnapshot() - // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: action.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - }) - - it('does not resolve report for mismatching repo.', async () => { - const { data: report } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONSPAM, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, - }, - { - headers: await network.serviceHeaders(sc.dids.alice), - encoding: 'application/json', - }, - ) - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.carol, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - - const promise = agent.api.com.atproto.admin.resolveModerationReports( - { - actionId: action.id, - reportIds: [report.id], - createdBy: 'did:example:admin', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - - await expect(promise).rejects.toThrow( - `Report ${report.id} cannot be resolved by action`, - ) - - // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: action.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - }) - - it('does not resolve report for mismatching record.', async () => { - const postRef1 = sc.posts[sc.dids.alice][0].ref - const postRef2 = sc.posts[sc.dids.bob][0].ref - const { data: report } = - await agent.api.com.atproto.moderation.createReport( - { - reasonType: REASONSPAM, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postRef1.uriStr, - cid: postRef1.cidStr, - }, - }, - { - headers: await network.serviceHeaders(sc.dids.alice), - encoding: 'application/json', - }, - ) - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postRef2.uriStr, - cid: postRef2.cidStr, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - - const promise = agent.api.com.atproto.admin.resolveModerationReports( - { - actionId: action.id, - reportIds: [report.id], - createdBy: 'did:example:admin', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - - await expect(promise).rejects.toThrow( - `Report ${report.id} cannot be resolved by action`, - ) - - // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: action.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - }) - - it('supports escalating and acknowledging for triage.', async () => { - const postRef1 = sc.posts[sc.dids.alice][0].ref - const postRef2 = sc.posts[sc.dids.bob][0].ref - const { data: action1 } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: ESCALATE, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postRef1.uri.toString(), - cid: postRef1.cid.toString(), - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders('triage'), - }, - ) - expect(action1).toEqual( - expect.objectContaining({ - action: ESCALATE, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postRef1.uriStr, - cid: postRef1.cidStr, - }, + await Promise.all([ + createReport({ + reasonType: REASONSPAM, + account: sc.dids.bob, + author: sc.dids.alice, }), - ) - const { data: action2 } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: ACKNOWLEDGE, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postRef2.uri.toString(), - cid: postRef2.cid.toString(), - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders('triage'), - }, - ) - expect(action2).toEqual( - expect.objectContaining({ - action: ACKNOWLEDGE, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postRef2.uriStr, - cid: postRef2.cidStr, + createReport({ + reasonType: REASONOTHER, + reason: 'defamation', + content: { + uri: post.uri.toString(), + cid: post.cid.toString(), }, + author: sc.dids.carol, }), - ) - // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: action1.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders('triage'), - }, - ) - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: action2.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders('triage'), - }, - ) - }) + ]) - it('only allows record to have one current action.', async () => { - const postRef = sc.posts[sc.dids.alice][0].ref - const { data: acknowledge } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: ACKNOWLEDGE, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postRef.uriStr, - cid: postRef.cidStr, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - const flagPromise = agent.api.com.atproto.admin.takeModerationAction( - { - action: FLAG, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postRef.uriStr, - cid: postRef.cidStr, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - await expect(flagPromise).rejects.toThrow( - 'Subject already has an active action:', - ) + await performTakedown({ + account: sc.dids.bob, + }) - // Reverse current then retry - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: acknowledge.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - const { data: flag } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: FLAG, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postRef.uriStr, - cid: postRef.cidStr, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) + const moderationStatusOnBobsAccount = await getStatuses({ + subject: sc.dids.bob, + }) - // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: flag.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), + // Validate that subject status is set to review closed and takendown flag is on + expect(moderationStatusOnBobsAccount.subjectStatuses[0]).toMatchObject({ + reviewState: REVIEWCLOSED, + takendown: true, + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.bob, }, - ) + }) + + // Cleanup + await performReverseTakedown({ + account: sc.dids.bob, + }) }) - it('only allows repo to have one current action.', async () => { - const { data: acknowledge } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: ACKNOWLEDGE, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.alice, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - const flagPromise = agent.api.com.atproto.admin.takeModerationAction( + it('supports escalating a subject', async () => { + const alicesPostRef = sc.posts[sc.dids.alice][0].ref + const alicesPostSubject = { + $type: 'com.atproto.repo.strongRef', + uri: alicesPostRef.uri.toString(), + cid: alicesPostRef.cid.toString(), + } + await agent.api.com.atproto.admin.emitModerationEvent( { - action: FLAG, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.alice, + event: { + $type: 'com.atproto.admin.defs#modEventEscalate', + comment: 'Y', }, + subject: alicesPostSubject, createdBy: 'did:example:admin', - reason: 'Y', }, { encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), + headers: network.bsky.adminAuthHeaders('triage'), }, ) - await expect(flagPromise).rejects.toThrow( - 'Subject already has an active action:', - ) - // Reverse current then retry - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: acknowledge.id, + const alicesPostStatus = await getStatuses({ + subject: alicesPostRef.uri.toString(), + }) + + expect(alicesPostStatus.subjectStatuses[0]).toMatchObject({ + reviewState: REVIEWESCALATED, + takendown: false, + subject: alicesPostSubject, + }) + }) + + it('adds persistent comment on subject through comment event', async () => { + const alicesPostRef = sc.posts[sc.dids.alice][0].ref + const alicesPostSubject = { + $type: 'com.atproto.repo.strongRef', + uri: alicesPostRef.uri.toString(), + cid: alicesPostRef.cid.toString(), + } + await agent.api.com.atproto.admin.emitModerationEvent( + { + event: { + $type: 'com.atproto.admin.defs#modEventComment', + sticky: true, + comment: 'This is a persistent note', + }, + subject: alicesPostSubject, createdBy: 'did:example:admin', - reason: 'Y', }, { encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), + headers: network.bsky.adminAuthHeaders('triage'), }, ) - const { data: flag } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: FLAG, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.alice, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: flag.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, + const alicesPostStatus = await getStatuses({ + subject: alicesPostRef.uri.toString(), + }) + + expect(alicesPostStatus.subjectStatuses[0].comment).toEqual( + 'This is a persistent note', ) }) - it('only allows blob to have one current action.', async () => { - const img = sc.posts[sc.dids.carol][0].images[0] - const postA = await sc.post(sc.dids.carol, 'image A', undefined, [img]) - const postB = await sc.post(sc.dids.carol, 'image B', undefined, [img]) - const { data: acknowledge } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: ACKNOWLEDGE, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postA.ref.uriStr, - cid: postA.ref.cidStr, - }, - subjectBlobCids: [img.image.ref.toString()], - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - const flagPromise = agent.api.com.atproto.admin.takeModerationAction( - { - action: FLAG, + it('reverses status when revert event is triggered.', async () => { + const alicesPostRef = sc.posts[sc.dids.alice][0].ref + const emitModEvent = async ( + event: ComAtprotoAdminEmitModerationEvent.InputSchema['event'], + overwrites: Partial = {}, + ) => { + const baseAction = { subject: { $type: 'com.atproto.repo.strongRef', - uri: postB.ref.uriStr, - cid: postB.ref.cidStr, + uri: alicesPostRef.uriStr, + cid: alicesPostRef.cidStr, }, - subjectBlobCids: [img.image.ref.toString()], - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - await expect(flagPromise).rejects.toThrow( - 'Blob already has an active action:', - ) - // Reverse current then retry - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: acknowledge.id, createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - const { data: flag } = - await agent.api.com.atproto.admin.takeModerationAction( + } + return agent.api.com.atproto.admin.emitModerationEvent( { - action: FLAG, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: postB.ref.uriStr, - cid: postB.ref.cidStr, - }, - subjectBlobCids: [img.image.ref.toString()], - createdBy: 'did:example:admin', - reason: 'Y', + event, + ...baseAction, + ...overwrites, }, { encoding: 'application/json', headers: network.bsky.adminAuthHeaders(), }, ) + } + // Validate that subject status is marked as escalated + await emitModEvent({ + $type: 'com.atproto.admin.defs#modEventReport', + reportType: REASONSPAM, + }) + await emitModEvent({ + $type: 'com.atproto.admin.defs#modEventReport', + reportType: REASONMISLEADING, + }) + await emitModEvent({ + $type: 'com.atproto.admin.defs#modEventEscalate', + }) + const alicesPostStatusAfterEscalation = await getStatuses({ + subject: alicesPostRef.uriStr, + }) + expect( + alicesPostStatusAfterEscalation.subjectStatuses[0].reviewState, + ).toEqual(REVIEWESCALATED) - // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: flag.id, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) + // Validate that subject status is marked as takendown + + await emitModEvent({ + $type: 'com.atproto.admin.defs#modEventLabel', + createLabelVals: ['nsfw'], + negateLabelVals: [], + }) + const { data: takedownAction } = await emitModEvent({ + $type: 'com.atproto.admin.defs#modEventTakedown', + }) + + const alicesPostStatusAfterTakedown = await getStatuses({ + subject: alicesPostRef.uriStr, + }) + expect(alicesPostStatusAfterTakedown.subjectStatuses[0]).toMatchObject({ + reviewState: REVIEWCLOSED, + takendown: true, + }) + + await emitModEvent({ + $type: 'com.atproto.admin.defs#modEventReverseTakedown', + }) + const alicesPostStatusAfterRevert = await getStatuses({ + subject: alicesPostRef.uriStr, + }) + // Validate that after reverting, the status of the subject is reverted to the last status changing event + expect(alicesPostStatusAfterRevert.subjectStatuses[0]).toMatchObject({ + reviewState: REVIEWCLOSED, + takendown: false, + }) + // Validate that after reverting, the last review date of the subject + // DOES NOT update to the the last status changing event + expect( + new Date( + alicesPostStatusAfterEscalation.subjectStatuses[0] + .lastReviewedAt as string, + ) < + new Date( + alicesPostStatusAfterRevert.subjectStatuses[0] + .lastReviewedAt as string, + ), + ).toBeTruthy() }) - it('negates an existing label and reverses.', async () => { + it('negates an existing label.', async () => { const { ctx } = network.bsky const post = sc.posts[sc.dids.bob][0].ref + const bobsPostSubject = { + $type: 'com.atproto.repo.strongRef', + uri: post.uriStr, + cid: post.cidStr, + } const labelingService = ctx.services.label(ctx.db.getPrimary()) await labelingService.formatAndCreate( ctx.cfg.labelerDid, @@ -814,16 +439,18 @@ describe('moderation', () => { post.cidStr, { create: ['kittens'] }, ) - const action = await actionWithLabels({ + await emitLabelEvent({ negateLabelVals: ['kittens'], - subject: { - $type: 'com.atproto.repo.strongRef', - uri: post.uriStr, - cid: post.cidStr, - }, + createLabelVals: [], + subject: bobsPostSubject, }) await expect(getRecordLabels(post.uriStr)).resolves.toEqual([]) - await reverse(action.id) + + await emitLabelEvent({ + createLabelVals: ['kittens'], + negateLabelVals: [], + subject: bobsPostSubject, + }) await expect(getRecordLabels(post.uriStr)).resolves.toEqual(['kittens']) // Cleanup await labelingService.formatAndCreate( @@ -838,8 +465,9 @@ describe('moderation', () => { const { ctx } = network.bsky const post = sc.posts[sc.dids.bob][0].ref const labelingService = ctx.services.label(ctx.db.getPrimary()) - const action = await actionWithLabels({ + await emitLabelEvent({ negateLabelVals: ['bears'], + createLabelVals: [], subject: { $type: 'com.atproto.repo.strongRef', uri: post.uriStr, @@ -847,7 +475,15 @@ describe('moderation', () => { }, }) await expect(getRecordLabels(post.uriStr)).resolves.toEqual([]) - await reverse(action.id) + await emitLabelEvent({ + createLabelVals: ['bears'], + negateLabelVals: [], + subject: { + $type: 'com.atproto.repo.strongRef', + uri: post.uriStr, + cid: post.cidStr, + }, + }) await expect(getRecordLabels(post.uriStr)).resolves.toEqual(['bears']) // Cleanup await labelingService.formatAndCreate( @@ -860,7 +496,7 @@ describe('moderation', () => { it('creates non-existing labels and reverses.', async () => { const post = sc.posts[sc.dids.bob][0].ref - const action = await actionWithLabels({ + await emitLabelEvent({ createLabelVals: ['puppies', 'doggies'], negateLabelVals: [], subject: { @@ -873,35 +509,20 @@ describe('moderation', () => { 'puppies', 'doggies', ]) - await reverse(action.id) - await expect(getRecordLabels(post.uriStr)).resolves.toEqual([]) - }) - - it('no-ops when creating an existing label and reverses.', async () => { - const { ctx } = network.bsky - const post = sc.posts[sc.dids.bob][0].ref - const labelingService = ctx.services.label(ctx.db.getPrimary()) - await labelingService.formatAndCreate( - ctx.cfg.labelerDid, - post.uriStr, - post.cidStr, - { create: ['birds'] }, - ) - const action = await actionWithLabels({ - createLabelVals: ['birds'], + await emitLabelEvent({ + negateLabelVals: ['puppies', 'doggies'], + createLabelVals: [], subject: { $type: 'com.atproto.repo.strongRef', uri: post.uriStr, cid: post.cidStr, }, }) - await expect(getRecordLabels(post.uriStr)).resolves.toEqual(['birds']) - await reverse(action.id) await expect(getRecordLabels(post.uriStr)).resolves.toEqual([]) }) it('creates labels on a repo and reverses.', async () => { - const action = await actionWithLabels({ + await emitLabelEvent({ createLabelVals: ['puppies', 'doggies'], negateLabelVals: [], subject: { @@ -913,7 +534,14 @@ describe('moderation', () => { 'puppies', 'doggies', ]) - await reverse(action.id) + await emitLabelEvent({ + negateLabelVals: ['puppies', 'doggies'], + createLabelVals: [], + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.bob, + }, + }) await expect(getRepoLabels(sc.dids.bob)).resolves.toEqual([]) }) @@ -926,7 +554,7 @@ describe('moderation', () => { null, { create: ['kittens'] }, ) - const action = await actionWithLabels({ + await emitLabelEvent({ createLabelVals: ['puppies'], negateLabelVals: ['kittens'], subject: { @@ -935,22 +563,32 @@ describe('moderation', () => { }, }) await expect(getRepoLabels(sc.dids.bob)).resolves.toEqual(['puppies']) - await reverse(action.id) + + await emitLabelEvent({ + negateLabelVals: ['puppies'], + createLabelVals: ['kittens'], + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.bob, + }, + }) await expect(getRepoLabels(sc.dids.bob)).resolves.toEqual(['kittens']) }) it('does not allow triage moderators to label.', async () => { - const attemptLabel = agent.api.com.atproto.admin.takeModerationAction( + const attemptLabel = agent.api.com.atproto.admin.emitModerationEvent( { - action: ACKNOWLEDGE, + event: { + $type: 'com.atproto.admin.defs#modEventLabel', + negateLabelVals: ['a'], + createLabelVals: ['b', 'c'], + }, createdBy: 'did:example:moderator', reason: 'Y', subject: { $type: 'com.atproto.admin.defs#repoRef', did: sc.dids.bob, }, - negateLabelVals: ['a'], - createLabelVals: ['b', 'c'], }, { encoding: 'application/json', @@ -962,23 +600,30 @@ describe('moderation', () => { ) }) + it('does not allow take down event on takendown post or reverse takedown on available post.', async () => { + await performTakedown({ + account: sc.dids.bob, + }) + await expect( + performTakedown({ + account: sc.dids.bob, + }), + ).rejects.toThrow('Subject is already taken down') + + // Cleanup + await performReverseTakedown({ + account: sc.dids.bob, + }) + await expect( + performReverseTakedown({ + account: sc.dids.bob, + }), + ).rejects.toThrow('Subject is not taken down') + }) it('fans out repo takedowns to pds', async () => { - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - createdBy: 'did:example:moderator', - reason: 'Y', - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) + await performTakedown({ + account: sc.dids.bob, + }) const res1 = await pdsAgent.api.com.atproto.admin.getSubjectStatus( { @@ -989,7 +634,7 @@ describe('moderation', () => { expect(res1.data.takedown?.applied).toBe(true) // cleanup - await reverse(action.id) + await performReverseTakedown({ account: sc.dids.bob }) const res2 = await pdsAgent.api.com.atproto.admin.getSubjectStatus( { @@ -1004,24 +649,9 @@ describe('moderation', () => { const post = sc.posts[sc.dids.bob][0] const uri = post.ref.uriStr const cid = post.ref.cidStr - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - createdBy: 'did:example:moderator', - reason: 'Y', - subject: { - $type: 'com.atproto.repo.strongRef', - uri, - cid, - }, - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), - }, - ) - + await performTakedown({ + content: { uri, cid }, + }) const res1 = await pdsAgent.api.com.atproto.admin.getSubjectStatus( { uri }, { headers: network.pds.adminAuthHeaders() }, @@ -1029,7 +659,7 @@ describe('moderation', () => { expect(res1.data.takedown?.applied).toBe(true) // cleanup - await reverse(action.id) + await performReverseTakedown({ content: { uri, cid } }) const res2 = await pdsAgent.api.com.atproto.admin.getSubjectStatus( { uri }, @@ -1039,33 +669,39 @@ describe('moderation', () => { }) it('allows full moderators to takedown.', async () => { - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - createdBy: 'did:example:moderator', - reason: 'Y', - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, + await agent.api.com.atproto.admin.emitModerationEvent( + { + event: { + $type: 'com.atproto.admin.defs#modEventTakedown', }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders('moderator'), + createdBy: 'did:example:moderator', + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.bob, }, - ) + }, + { + encoding: 'application/json', + headers: network.bsky.adminAuthHeaders('moderator'), + }, + ) // cleanup - await reverse(action.id) + await reverse({ + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.bob, + }, + }) }) it('does not allow non-full moderators to takedown.', async () => { const attemptTakedownTriage = - agent.api.com.atproto.admin.takeModerationAction( + agent.api.com.atproto.admin.emitModerationEvent( { - action: TAKEDOWN, + event: { + $type: 'com.atproto.admin.defs#modEventTakedown', + }, createdBy: 'did:example:moderator', - reason: 'Y', subject: { $type: 'com.atproto.admin.defs#repoRef', did: sc.dids.bob, @@ -1081,61 +717,76 @@ describe('moderation', () => { ) }) it('automatically reverses actions marked with duration', async () => { - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - createdBy: 'did:example:moderator', - reason: 'Y', - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, - createLabelVals: ['takendown'], - // Use negative value to set the expiry time in the past so that the action is automatically reversed - // right away without having to wait n number of hours for a successful assertion - durationInHours: -1, - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders('moderator'), - }, + await createReport({ + reasonType: REASONSPAM, + account: sc.dids.bob, + author: sc.dids.alice, + }) + const { data: action } = await performTakedown({ + account: sc.dids.bob, + // Use negative value to set the expiry time in the past so that the action is automatically reversed + // right away without having to wait n number of hours for a successful assertion + durationInHours: -1, + }) + + const { data: statusesAfterTakedown } = + await agent.api.com.atproto.admin.queryModerationStatuses( + { subject: sc.dids.bob }, + { headers: network.bsky.adminAuthHeaders('moderator') }, ) - const labelsAfterTakedown = await getRepoLabels(sc.dids.bob) - expect(labelsAfterTakedown).toContain('takendown') + expect(statusesAfterTakedown.subjectStatuses[0]).toMatchObject({ + takendown: true, + }) + // In the actual app, this will be instantiated and run on server startup - const periodicReversal = new PeriodicModerationActionReversal( + const periodicReversal = new PeriodicModerationEventReversal( network.bsky.ctx, ) await periodicReversal.findAndRevertDueActions() - const { data: reversedAction } = - await agent.api.com.atproto.admin.getModerationAction( - { id: action.id }, + const [{ data: eventList }, { data: statuses }] = await Promise.all([ + agent.api.com.atproto.admin.queryModerationEvents( + { subject: sc.dids.bob }, { headers: network.bsky.adminAuthHeaders('moderator') }, - ) + ), + agent.api.com.atproto.admin.queryModerationStatuses( + { subject: sc.dids.bob }, + { headers: network.bsky.adminAuthHeaders('moderator') }, + ), + ]) + expect(statuses.subjectStatuses[0]).toMatchObject({ + takendown: false, + reviewState: REVIEWCLOSED, + }) // Verify that the automatic reversal is attributed to the original moderator of the temporary action // and that the reason is set to indicate that the action was automatically reversed. - expect(reversedAction.reversal).toMatchObject({ + expect(eventList.events[0]).toMatchObject({ createdBy: action.createdBy, - reason: '[SCHEDULED_REVERSAL] Reverting action as originally scheduled', + event: { + $type: 'com.atproto.admin.defs#modEventReverseTakedown', + comment: + '[SCHEDULED_REVERSAL] Reverting action as originally scheduled', + }, }) - - // Verify that labels are also reversed when takedown action is reversed - const labelsAfterReversal = await getRepoLabels(sc.dids.bob) - expect(labelsAfterReversal).not.toContain('takendown') }) - async function actionWithLabels( - opts: Partial & { - subject: ComAtprotoAdminTakeModerationAction.InputSchema['subject'] + async function emitLabelEvent( + opts: Partial & { + subject: ComAtprotoAdminEmitModerationEvent.InputSchema['subject'] + createLabelVals: ModEventLabel['createLabelVals'] + negateLabelVals: ModEventLabel['negateLabelVals'] }, ) { - const result = await agent.api.com.atproto.admin.takeModerationAction( + const { createLabelVals, negateLabelVals, ...rest } = opts + const result = await agent.api.com.atproto.admin.emitModerationEvent( { - action: FLAG, + event: { + $type: 'com.atproto.admin.defs#modEventLabel', + createLabelVals, + negateLabelVals, + }, createdBy: 'did:example:admin', reason: 'Y', ...opts, @@ -1148,12 +799,19 @@ describe('moderation', () => { return result.data } - async function reverse(actionId: number) { - await agent.api.com.atproto.admin.reverseModerationAction( + async function reverse( + opts: Partial & { + subject: ComAtprotoAdminEmitModerationEvent.InputSchema['subject'] + }, + ) { + await agent.api.com.atproto.admin.emitModerationEvent( { - id: actionId, + event: { + $type: 'com.atproto.admin.defs#modEventReverseTakedown', + }, createdBy: 'did:example:admin', reason: 'Y', + ...opts, }, { encoding: 'application/json', @@ -1185,7 +843,6 @@ describe('moderation', () => { let post: { ref: RecordRef; images: ImageRef[] } let blob: ImageRef let imageUri: string - let actionId: number beforeAll(async () => { const { ctx } = network.bsky post = sc.posts[sc.dids.carol][0] @@ -1201,24 +858,23 @@ describe('moderation', () => { await fetch(imageUri) const cached = await fetch(imageUri) expect(cached.headers.get('x-cache')).toEqual('hit') - const takeAction = await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: post.ref.uriStr, - cid: post.ref.cidStr, - }, - subjectBlobCids: [blob.image.ref.toString()], - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), + await performTakedown({ + content: { + uri: post.ref.uriStr, + cid: post.ref.cidStr, }, - ) - actionId = takeAction.data.id + subjectBlobCids: [blob.image.ref.toString()], + }) + }) + + it('sets blobCids in moderation status', async () => { + const { subjectStatuses } = await getStatuses({ + subject: post.ref.uriStr, + }) + + expect(subjectStatuses[0].subjectBlobCids).toEqual([ + blob.image.ref.toString(), + ]) }) it('prevents resolution of blob', async () => { @@ -1249,17 +905,13 @@ describe('moderation', () => { }) it('restores blob when action is reversed.', async () => { - await agent.api.com.atproto.admin.reverseModerationAction( - { - id: actionId, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.bsky.adminAuthHeaders(), + await performReverseTakedown({ + content: { + uri: post.ref.uriStr, + cid: post.ref.cidStr, }, - ) + subjectBlobCids: [blob.image.ref.toString()], + }) // Can resolve blob const blobPath = `/blob/${sc.dids.carol}/${blob.image.ref.toString()}` diff --git a/packages/bsky/tests/admin/repo-search.test.ts b/packages/bsky/tests/admin/repo-search.test.ts index fab63257147..837c4b2154a 100644 --- a/packages/bsky/tests/admin/repo-search.test.ts +++ b/packages/bsky/tests/admin/repo-search.test.ts @@ -1,6 +1,5 @@ import { SeedClient, TestNetwork } from '@atproto/dev-env' import AtpAgent from '@atproto/api' -import { TAKEDOWN } from '@atproto/api/src/client/types/com/atproto/admin/defs' import { paginateAll } from '../_util' import usersBulkSeed from '../seeds/users-bulk' @@ -25,8 +24,8 @@ describe('admin repo search view', () => { }) beforeAll(async () => { - await sc.takeModerationAction({ - action: TAKEDOWN, + await sc.emitModerationEvent({ + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, subject: { $type: 'com.atproto.admin.defs#repoRef', did: sc.dids['cara-wiegand69.test'], diff --git a/packages/bsky/tests/auto-moderator/fuzzy-matcher.test.ts b/packages/bsky/tests/auto-moderator/fuzzy-matcher.test.ts index 09422cd8d6e..60fe50d582d 100644 --- a/packages/bsky/tests/auto-moderator/fuzzy-matcher.test.ts +++ b/packages/bsky/tests/auto-moderator/fuzzy-matcher.test.ts @@ -37,7 +37,8 @@ describe('fuzzy matcher', () => { const getAllReports = () => { return network.bsky.ctx.db .getPrimary() - .db.selectFrom('moderation_report') + .db.selectFrom('moderation_event') + .where('action', '=', 'com.atproto.admin.defs#modEventReport') .selectAll() .orderBy('id', 'asc') .execute() diff --git a/packages/bsky/tests/auto-moderator/takedowns.test.ts b/packages/bsky/tests/auto-moderator/takedowns.test.ts index d2bc8d4a2a2..43a0fe5c00a 100644 --- a/packages/bsky/tests/auto-moderator/takedowns.test.ts +++ b/packages/bsky/tests/auto-moderator/takedowns.test.ts @@ -77,28 +77,41 @@ describe('takedowner', () => { const post = await sc.post(alice, 'blah', undefined, [goodBlob, badBlob1]) await network.processAll() await autoMod.processAll() - const modAction = await ctx.db.db - .selectFrom('moderation_action') - .where('subjectUri', '=', post.ref.uriStr) - .select(['action', 'id']) - .executeTakeFirst() - if (!modAction) { + const [modStatus, takedownEvent] = await Promise.all([ + ctx.db.db + .selectFrom('moderation_subject_status') + .where('did', '=', alice) + .where( + 'recordPath', + '=', + `${post.ref.uri.collection}/${post.ref.uri.rkey}`, + ) + .select(['takendown', 'id']) + .executeTakeFirst(), + ctx.db.db + .selectFrom('moderation_event') + .where('subjectDid', '=', alice) + .where('action', '=', 'com.atproto.admin.defs#modEventTakedown') + .selectAll() + .executeTakeFirst(), + ]) + if (!modStatus || !takedownEvent) { throw new Error('expected mod action') } - expect(modAction.action).toEqual('com.atproto.admin.defs#takedown') + expect(modStatus.takendown).toEqual(true) const record = await ctx.db.db .selectFrom('record') .where('uri', '=', post.ref.uriStr) .select('takedownId') .executeTakeFirst() - expect(record?.takedownId).toEqual(modAction.id) + expect(record?.takedownId).toBeGreaterThan(0) const recordPds = await network.pds.ctx.db.db .selectFrom('record') .where('uri', '=', post.ref.uriStr) .select('takedownRef') .executeTakeFirst() - expect(recordPds?.takedownRef).toEqual(modAction.id.toString()) + expect(recordPds?.takedownRef).toEqual(takedownEvent.id.toString()) expect(testInvalidator.invalidated.length).toBe(1) expect(testInvalidator.invalidated[0].subject).toBe( @@ -119,28 +132,42 @@ describe('takedowner', () => { { headers: sc.getHeaders(alice), encoding: 'application/json' }, ) await network.processAll() - const modAction = await ctx.db.db - .selectFrom('moderation_action') - .where('subjectUri', '=', res.data.uri) - .select(['action', 'id']) - .executeTakeFirst() - if (!modAction) { + const [modStatus, takedownEvent] = await Promise.all([ + ctx.db.db + .selectFrom('moderation_subject_status') + .where('did', '=', alice) + .where('recordPath', '=', `${ids.AppBskyActorProfile}/self`) + .select(['takendown', 'id']) + .executeTakeFirst(), + ctx.db.db + .selectFrom('moderation_event') + .where('subjectDid', '=', alice) + .where( + 'subjectUri', + '=', + AtUri.make(alice, ids.AppBskyActorProfile, 'self').toString(), + ) + .where('action', '=', 'com.atproto.admin.defs#modEventTakedown') + .selectAll() + .executeTakeFirst(), + ]) + if (!modStatus || !takedownEvent) { throw new Error('expected mod action') } - expect(modAction.action).toEqual('com.atproto.admin.defs#takedown') + expect(modStatus.takendown).toEqual(true) const record = await ctx.db.db .selectFrom('record') .where('uri', '=', res.data.uri) .select('takedownId') .executeTakeFirst() - expect(record?.takedownId).toEqual(modAction.id) + expect(record?.takedownId).toBeGreaterThan(0) const recordPds = await network.pds.ctx.db.db .selectFrom('record') .where('uri', '=', res.data.uri) .select('takedownRef') .executeTakeFirst() - expect(recordPds?.takedownRef).toEqual(modAction.id.toString()) + expect(recordPds?.takedownRef).toEqual(takedownEvent.id.toString()) expect(testInvalidator.invalidated.length).toBe(2) expect(testInvalidator.invalidated[1].subject).toBe( diff --git a/packages/bsky/tests/feed-generation.test.ts b/packages/bsky/tests/feed-generation.test.ts index 4970c13b31c..aceecec3204 100644 --- a/packages/bsky/tests/feed-generation.test.ts +++ b/packages/bsky/tests/feed-generation.test.ts @@ -9,7 +9,6 @@ import { import { Handler as SkeletonHandler } from '../src/lexicon/types/app/bsky/feed/getFeedSkeleton' import { GeneratorView } from '@atproto/api/src/client/types/app/bsky/feed/defs' import { UnknownFeedError } from '@atproto/api/src/client/types/app/bsky/feed/getFeed' -import { TAKEDOWN } from '@atproto/api/src/client/types/com/atproto/admin/defs' import { ids } from '../src/lexicon/lexicons' import { FeedViewPost, @@ -158,9 +157,9 @@ describe('feed generation', () => { sc.getHeaders(alice), ) await network.processAll() - await agent.api.com.atproto.admin.takeModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - action: TAKEDOWN, + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, subject: { $type: 'com.atproto.repo.strongRef', uri: prime.uri, diff --git a/packages/bsky/tests/views/actor-search.test.ts b/packages/bsky/tests/views/actor-search.test.ts index 0f22eff0513..70f8862f7d7 100644 --- a/packages/bsky/tests/views/actor-search.test.ts +++ b/packages/bsky/tests/views/actor-search.test.ts @@ -1,7 +1,6 @@ import AtpAgent from '@atproto/api' import { wait } from '@atproto/common' import { TestNetwork, SeedClient } from '@atproto/dev-env' -import { TAKEDOWN } from '@atproto/api/src/client/types/com/atproto/admin/defs' import { forSnapshot, paginateAll, stripViewer } from '../_util' import usersBulkSeed from '../seeds/users-bulk' @@ -240,9 +239,9 @@ describe.skip('pds actor search views', () => { }) it('search blocks by actor takedown', async () => { - await agent.api.com.atproto.admin.takeModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - action: TAKEDOWN, + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, subject: { $type: 'com.atproto.admin.defs#repoRef', did: sc.dids['cara-wiegand69.test'], diff --git a/packages/bsky/tests/views/author-feed.test.ts b/packages/bsky/tests/views/author-feed.test.ts index 3d764335282..b8fade87c54 100644 --- a/packages/bsky/tests/views/author-feed.test.ts +++ b/packages/bsky/tests/views/author-feed.test.ts @@ -2,7 +2,6 @@ import AtpAgent from '@atproto/api' import { TestNetwork, SeedClient } from '@atproto/dev-env' import { forSnapshot, paginateAll, stripViewerFromPost } from '../_util' import basicSeed from '../seeds/basic' -import { TAKEDOWN } from '@atproto/api/src/client/types/com/atproto/admin/defs' import { isRecord } from '../../src/lexicon/types/app/bsky/feed/post' import { isView as isEmbedRecordWithMedia } from '../../src/lexicon/types/app/bsky/embed/recordWithMedia' import { isView as isImageEmbed } from '../../src/lexicon/types/app/bsky/embed/images' @@ -146,22 +145,21 @@ describe('pds author feed views', () => { expect(preBlock.feed.length).toBeGreaterThan(0) - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: alice, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), + await agent.api.com.atproto.admin.emitModerationEvent( + { + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: alice, }, - ) + createdBy: 'did:example:admin', + reason: 'Y', + }, + { + encoding: 'application/json', + headers: network.pds.adminAuthHeaders(), + }, + ) const attempt = agent.api.app.bsky.feed.getAuthorFeed( { actor: alice }, @@ -170,9 +168,13 @@ describe('pds author feed views', () => { await expect(attempt).rejects.toThrow('Profile not found') // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - id: action.id, + event: { $type: 'com.atproto.admin.defs#modEventReverseTakedown' }, + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: alice, + }, createdBy: 'did:example:admin', reason: 'Y', }, @@ -193,23 +195,22 @@ describe('pds author feed views', () => { const post = preBlock.feed[0].post - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: post.uri, - cid: post.cid, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), + await agent.api.com.atproto.admin.emitModerationEvent( + { + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, + subject: { + $type: 'com.atproto.repo.strongRef', + uri: post.uri, + cid: post.cid, }, - ) + createdBy: 'did:example:admin', + reason: 'Y', + }, + { + encoding: 'application/json', + headers: network.pds.adminAuthHeaders(), + }, + ) const { data: postBlock } = await agent.api.app.bsky.feed.getAuthorFeed( { actor: alice }, @@ -220,9 +221,14 @@ describe('pds author feed views', () => { expect(postBlock.feed.map((item) => item.post.uri)).not.toContain(post.uri) // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - id: action.id, + event: { $type: 'com.atproto.admin.defs#modEventReverseTakedown' }, + subject: { + $type: 'com.atproto.repo.strongRef', + uri: post.uri, + cid: post.cid, + }, createdBy: 'did:example:admin', reason: 'Y', }, diff --git a/packages/bsky/tests/views/follows.test.ts b/packages/bsky/tests/views/follows.test.ts index 3bf89ff965e..f290ec622d5 100644 --- a/packages/bsky/tests/views/follows.test.ts +++ b/packages/bsky/tests/views/follows.test.ts @@ -2,7 +2,6 @@ import AtpAgent from '@atproto/api' import { TestNetwork, SeedClient } from '@atproto/dev-env' import { forSnapshot, paginateAll, stripViewer } from '../_util' import followsSeed from '../seeds/follows' -import { TAKEDOWN } from '@atproto/api/src/client/types/com/atproto/admin/defs' describe('pds follow views', () => { let agent: AtpAgent @@ -121,22 +120,21 @@ describe('pds follow views', () => { }) it('blocks followers by actor takedown', async () => { - const { data: modAction } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.dan, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), + await agent.api.com.atproto.admin.emitModerationEvent( + { + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.dan, }, - ) + createdBy: 'did:example:admin', + reason: 'Y', + }, + { + encoding: 'application/json', + headers: network.pds.adminAuthHeaders(), + }, + ) const aliceFollowers = await agent.api.app.bsky.graph.getFollowers( { actor: sc.dids.alice }, @@ -147,9 +145,13 @@ describe('pds follow views', () => { sc.dids.dan, ) - await agent.api.com.atproto.admin.reverseModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - id: modAction.id, + event: { $type: 'com.atproto.admin.defs#modEventReverseTakedown' }, + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.dan, + }, createdBy: 'did:example:admin', reason: 'Y', }, @@ -250,22 +252,21 @@ describe('pds follow views', () => { }) it('blocks follows by actor takedown', async () => { - const { data: modAction } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.dan, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), + await agent.api.com.atproto.admin.emitModerationEvent( + { + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.dan, }, - ) + createdBy: 'did:example:admin', + reason: 'Y', + }, + { + encoding: 'application/json', + headers: network.pds.adminAuthHeaders(), + }, + ) const aliceFollows = await agent.api.app.bsky.graph.getFollows( { actor: sc.dids.alice }, @@ -276,9 +277,13 @@ describe('pds follow views', () => { sc.dids.dan, ) - await agent.api.com.atproto.admin.reverseModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - id: modAction.id, + event: { $type: 'com.atproto.admin.defs#modEventReverseTakedown' }, + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.dan, + }, createdBy: 'did:example:admin', reason: 'Y', }, diff --git a/packages/bsky/tests/views/list-feed.test.ts b/packages/bsky/tests/views/list-feed.test.ts index baef857f437..b8cd977922b 100644 --- a/packages/bsky/tests/views/list-feed.test.ts +++ b/packages/bsky/tests/views/list-feed.test.ts @@ -2,7 +2,6 @@ import AtpAgent from '@atproto/api' import { TestNetwork, SeedClient, RecordRef } from '@atproto/dev-env' import { forSnapshot, paginateAll, stripViewerFromPost } from '../_util' import basicSeed from '../seeds/basic' -import { TAKEDOWN } from '@atproto/api/src/client/types/com/atproto/admin/defs' describe('list feed views', () => { let network: TestNetwork @@ -113,9 +112,9 @@ describe('list feed views', () => { }) it('blocks posts by actor takedown', async () => { - const actionRes = await agent.api.com.atproto.admin.takeModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - action: TAKEDOWN, + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, subject: { $type: 'com.atproto.admin.defs#repoRef', did: bob, @@ -136,9 +135,13 @@ describe('list feed views', () => { expect(hasBob).toBe(false) // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - id: actionRes.data.id, + event: { $type: 'com.atproto.admin.defs#modEventReverseTakedown' }, + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: bob, + }, createdBy: 'did:example:admin', reason: 'Y', }, @@ -151,9 +154,9 @@ describe('list feed views', () => { it('blocks posts by record takedown.', async () => { const postRef = sc.replies[bob][0].ref // Post and reply parent - const actionRes = await agent.api.com.atproto.admin.takeModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - action: TAKEDOWN, + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, subject: { $type: 'com.atproto.repo.strongRef', uri: postRef.uriStr, @@ -177,9 +180,14 @@ describe('list feed views', () => { expect(hasPost).toBe(false) // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - id: actionRes.data.id, + event: { $type: 'com.atproto.admin.defs#modEventReverseTakedown' }, + subject: { + $type: 'com.atproto.repo.strongRef', + uri: postRef.uriStr, + cid: postRef.cidStr, + }, createdBy: 'did:example:admin', reason: 'Y', }, diff --git a/packages/bsky/tests/views/notifications.test.ts b/packages/bsky/tests/views/notifications.test.ts index 7bdd5d5f933..7449d764671 100644 --- a/packages/bsky/tests/views/notifications.test.ts +++ b/packages/bsky/tests/views/notifications.test.ts @@ -1,6 +1,5 @@ import AtpAgent from '@atproto/api' import { TestNetwork, SeedClient } from '@atproto/dev-env' -import { TAKEDOWN } from '@atproto/api/src/client/types/com/atproto/admin/defs' import { forSnapshot, paginateAll } from '../_util' import basicSeed from '../seeds/basic' import { Notification } from '../../src/lexicon/types/app/bsky/notification/listNotifications' @@ -61,7 +60,7 @@ describe('notification views', () => { { headers: await network.serviceHeaders(sc.dids.bob) }, ) - expect(notifCountBob.data.count).toBe(4) + expect(notifCountBob.data.count).toBeGreaterThanOrEqual(3) }) it('generates notifications for all reply ancestors', async () => { @@ -89,7 +88,7 @@ describe('notification views', () => { { headers: await network.serviceHeaders(sc.dids.bob) }, ) - expect(notifCountBob.data.count).toBe(5) + expect(notifCountBob.data.count).toBeGreaterThanOrEqual(4) }) it('does not give notifs for a deleted subject', async () => { @@ -233,11 +232,11 @@ describe('notification views', () => { it('fetches notifications omitting mentions and replies for taken-down posts', async () => { const postRef1 = sc.replies[sc.dids.carol][0].ref // Reply const postRef2 = sc.posts[sc.dids.dan][1].ref // Mention - const actionResults = await Promise.all( + await Promise.all( [postRef1, postRef2].map((postRef) => - agent.api.com.atproto.admin.takeModerationAction( + agent.api.com.atproto.admin.emitModerationEvent( { - action: TAKEDOWN, + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, subject: { $type: 'com.atproto.repo.strongRef', uri: postRef.uriStr, @@ -270,10 +269,15 @@ describe('notification views', () => { // Cleanup await Promise.all( - actionResults.map((result) => - agent.api.com.atproto.admin.reverseModerationAction( + [postRef1, postRef2].map((postRef) => + agent.api.com.atproto.admin.emitModerationEvent( { - id: result.data.id, + event: { $type: 'com.atproto.admin.defs#modEventReverseTakedown' }, + subject: { + $type: 'com.atproto.repo.strongRef', + uri: postRef.uriStr, + cid: postRef.cidStr, + }, createdBy: 'did:example:admin', reason: 'Y', }, diff --git a/packages/bsky/tests/views/profile.test.ts b/packages/bsky/tests/views/profile.test.ts index fd3bde6d0ef..d4e0c718bed 100644 --- a/packages/bsky/tests/views/profile.test.ts +++ b/packages/bsky/tests/views/profile.test.ts @@ -1,7 +1,6 @@ import fs from 'fs/promises' import AtpAgent from '@atproto/api' import { TestNetwork, SeedClient } from '@atproto/dev-env' -import { TAKEDOWN } from '@atproto/api/src/client/types/com/atproto/admin/defs' import { forSnapshot, stripViewer } from '../_util' import { ids } from '../../src/lexicon/lexicons' import basicSeed from '../seeds/basic' @@ -186,22 +185,21 @@ describe('pds profile views', () => { }) it('blocked by actor takedown', async () => { - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: alice, - }, - createdBy: 'did:example:admin', - reason: 'Y', - }, - { - encoding: 'application/json', - headers: network.pds.adminAuthHeaders(), + await agent.api.com.atproto.admin.emitModerationEvent( + { + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: alice, }, - ) + createdBy: 'did:example:admin', + reason: 'Y', + }, + { + encoding: 'application/json', + headers: network.pds.adminAuthHeaders(), + }, + ) const promise = agent.api.app.bsky.actor.getProfile( { actor: alice }, { headers: await network.serviceHeaders(bob) }, @@ -210,9 +208,13 @@ describe('pds profile views', () => { await expect(promise).rejects.toThrow('Account has been taken down') // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - id: action.id, + event: { $type: 'com.atproto.admin.defs#modEventReverseTakedown' }, + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: alice, + }, createdBy: 'did:example:admin', reason: 'Y', }, diff --git a/packages/bsky/tests/views/thread.test.ts b/packages/bsky/tests/views/thread.test.ts index bee609f197b..f13be284a30 100644 --- a/packages/bsky/tests/views/thread.test.ts +++ b/packages/bsky/tests/views/thread.test.ts @@ -1,6 +1,5 @@ import AtpAgent, { AppBskyFeedGetPostThread } from '@atproto/api' import { TestNetwork, SeedClient } from '@atproto/dev-env' -import { TAKEDOWN } from '@atproto/api/src/client/types/com/atproto/admin/defs' import { forSnapshot, stripViewerFromThread } from '../_util' import basicSeed from '../seeds/basic' import assert from 'assert' @@ -167,9 +166,9 @@ describe('pds thread views', () => { describe('takedown', () => { it('blocks post by actor', async () => { const { data: modAction } = - await agent.api.com.atproto.admin.takeModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - action: TAKEDOWN, + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, subject: { $type: 'com.atproto.admin.defs#repoRef', did: alice, @@ -194,9 +193,13 @@ describe('pds thread views', () => { ) // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - id: modAction.id, + event: { $type: 'com.atproto.admin.defs#modEventReverseTakedown' }, + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: alice, + }, createdBy: 'did:example:admin', reason: 'Y', }, @@ -209,9 +212,9 @@ describe('pds thread views', () => { it('blocks replies by actor', async () => { const { data: modAction } = - await agent.api.com.atproto.admin.takeModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - action: TAKEDOWN, + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, subject: { $type: 'com.atproto.admin.defs#repoRef', did: carol, @@ -234,9 +237,13 @@ describe('pds thread views', () => { expect(forSnapshot(thread.data.thread)).toMatchSnapshot() // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - id: modAction.id, + event: { $type: 'com.atproto.admin.defs#modEventReverseTakedown' }, + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: carol, + }, createdBy: 'did:example:admin', reason: 'Y', }, @@ -249,9 +256,9 @@ describe('pds thread views', () => { it('blocks ancestors by actor', async () => { const { data: modAction } = - await agent.api.com.atproto.admin.takeModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - action: TAKEDOWN, + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, subject: { $type: 'com.atproto.admin.defs#repoRef', did: bob, @@ -274,9 +281,13 @@ describe('pds thread views', () => { expect(forSnapshot(thread.data.thread)).toMatchSnapshot() // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - id: modAction.id, + event: { $type: 'com.atproto.admin.defs#modEventReverseTakedown' }, + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: bob, + }, createdBy: 'did:example:admin', reason: 'Y', }, @@ -290,9 +301,9 @@ describe('pds thread views', () => { it('blocks post by record', async () => { const postRef = sc.posts[alice][1].ref const { data: modAction } = - await agent.api.com.atproto.admin.takeModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - action: TAKEDOWN, + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, subject: { $type: 'com.atproto.repo.strongRef', uri: postRef.uriStr, @@ -317,9 +328,14 @@ describe('pds thread views', () => { ) // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - id: modAction.id, + event: { $type: 'com.atproto.admin.defs#modEventReverseTakedown' }, + subject: { + $type: 'com.atproto.repo.strongRef', + uri: postRef.uriStr, + cid: postRef.cidStr, + }, createdBy: 'did:example:admin', reason: 'Y', }, @@ -339,9 +355,9 @@ describe('pds thread views', () => { const parent = threadPreTakedown.data.thread.parent?.['post'] const { data: modAction } = - await agent.api.com.atproto.admin.takeModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - action: TAKEDOWN, + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, subject: { $type: 'com.atproto.repo.strongRef', uri: parent.uri, @@ -365,9 +381,14 @@ describe('pds thread views', () => { expect(forSnapshot(thread.data.thread)).toMatchSnapshot() // Cleanup - await agent.api.com.atproto.admin.reverseModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - id: modAction.id, + event: { $type: 'com.atproto.admin.defs#modEventReverseTakedown' }, + subject: { + $type: 'com.atproto.repo.strongRef', + uri: parent.uri, + cid: parent.cid, + }, createdBy: 'did:example:admin', reason: 'Y', }, @@ -388,9 +409,9 @@ describe('pds thread views', () => { const actionResults = await Promise.all( [post1, post2].map((post) => - agent.api.com.atproto.admin.takeModerationAction( + agent.api.com.atproto.admin.emitModerationEvent( { - action: TAKEDOWN, + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, subject: { $type: 'com.atproto.repo.strongRef', uri: post.uri, @@ -417,10 +438,17 @@ describe('pds thread views', () => { // Cleanup await Promise.all( - actionResults.map((result) => - agent.api.com.atproto.admin.reverseModerationAction( + [post1, post2].map((post) => + agent.api.com.atproto.admin.emitModerationEvent( { - id: result.data.id, + event: { + $type: 'com.atproto.admin.defs#modEventReverseTakedown', + }, + subject: { + $type: 'com.atproto.repo.strongRef', + uri: post.uri, + cid: post.cid, + }, createdBy: 'did:example:admin', reason: 'Y', }, diff --git a/packages/bsky/tests/views/timeline.test.ts b/packages/bsky/tests/views/timeline.test.ts index 9cd3f688e33..5410d792a1f 100644 --- a/packages/bsky/tests/views/timeline.test.ts +++ b/packages/bsky/tests/views/timeline.test.ts @@ -1,7 +1,6 @@ import assert from 'assert' import AtpAgent from '@atproto/api' import { TestNetwork, SeedClient } from '@atproto/dev-env' -import { TAKEDOWN } from '@atproto/api/src/client/types/com/atproto/admin/defs' import { forSnapshot, getOriginator, paginateAll } from '../_util' import basicSeed from '../seeds/basic' import { FeedAlgorithm } from '../../src/api/app/bsky/util/feed' @@ -182,11 +181,11 @@ describe('timeline views', () => { }) it('blocks posts, reposts, replies by actor takedown', async () => { - const actionResults = await Promise.all( + await Promise.all( [bob, carol].map((did) => - agent.api.com.atproto.admin.takeModerationAction( + agent.api.com.atproto.admin.emitModerationEvent( { - action: TAKEDOWN, + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, subject: { $type: 'com.atproto.admin.defs#repoRef', did, @@ -211,10 +210,14 @@ describe('timeline views', () => { // Cleanup await Promise.all( - actionResults.map((result) => - agent.api.com.atproto.admin.reverseModerationAction( + [bob, carol].map((did) => + agent.api.com.atproto.admin.emitModerationEvent( { - id: result.data.id, + event: { $type: 'com.atproto.admin.defs#modEventReverseTakedown' }, + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did, + }, createdBy: 'did:example:admin', reason: 'Y', }, @@ -230,11 +233,11 @@ describe('timeline views', () => { it('blocks posts, reposts, replies by record takedown.', async () => { const postRef1 = sc.posts[dan][1].ref // Repost const postRef2 = sc.replies[bob][0].ref // Post and reply parent - const actionResults = await Promise.all( + await Promise.all( [postRef1, postRef2].map((postRef) => - agent.api.com.atproto.admin.takeModerationAction( + agent.api.com.atproto.admin.emitModerationEvent( { - action: TAKEDOWN, + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, subject: { $type: 'com.atproto.repo.strongRef', uri: postRef.uriStr, @@ -260,10 +263,15 @@ describe('timeline views', () => { // Cleanup await Promise.all( - actionResults.map((result) => - agent.api.com.atproto.admin.reverseModerationAction( + [postRef1, postRef2].map((postRef) => + agent.api.com.atproto.admin.emitModerationEvent( { - id: result.data.id, + event: { $type: 'com.atproto.admin.defs#modEventReverseTakedown' }, + subject: { + $type: 'com.atproto.repo.strongRef', + uri: postRef.uriStr, + cid: postRef.cidStr, + }, createdBy: 'did:example:admin', reason: 'Y', }, diff --git a/packages/dev-env/src/seed-client.ts b/packages/dev-env/src/seed-client.ts index b9b1eded96a..71dfebd53c0 100644 --- a/packages/dev-env/src/seed-client.ts +++ b/packages/dev-env/src/seed-client.ts @@ -2,7 +2,7 @@ import fs from 'fs/promises' import { CID } from 'multiformats/cid' import AtpAgent from '@atproto/api' import { Main as Facet } from '@atproto/api/src/client/types/app/bsky/richtext/facet' -import { InputSchema as TakeActionInput } from '@atproto/api/src/client/types/com/atproto/admin/takeModerationAction' +import { InputSchema as TakeActionInput } from '@atproto/api/src/client/types/com/atproto/admin/emitModerationEvent' import { InputSchema as CreateReportInput } from '@atproto/api/src/client/types/com/atproto/moderation/createReport' import { Record as PostRecord } from '@atproto/api/src/client/types/app/bsky/feed/post' import { Record as LikeRecord } from '@atproto/api/src/client/types/app/bsky/feed/like' @@ -419,20 +419,21 @@ export class SeedClient { delete foundList.items[subject] } - async takeModerationAction(opts: { - action: TakeActionInput['action'] + async emitModerationEvent(opts: { + event: TakeActionInput['event'] subject: TakeActionInput['subject'] reason?: string createdBy?: string + meta?: TakeActionInput['meta'] }) { const { - action, + event, subject, reason = 'X', createdBy = 'did:example:admin', } = opts - const result = await this.agent.api.com.atproto.admin.takeModerationAction( - { action, subject, createdBy, reason }, + const result = await this.agent.api.com.atproto.admin.emitModerationEvent( + { event, subject, createdBy, reason }, { encoding: 'application/json', headers: this.adminAuthHeaders(), @@ -443,35 +444,25 @@ export class SeedClient { async reverseModerationAction(opts: { id: number + subject: TakeActionInput['subject'] reason?: string createdBy?: string }) { - const { id, reason = 'X', createdBy = 'did:example:admin' } = opts - const result = - await this.agent.api.com.atproto.admin.reverseModerationAction( - { id, reason, createdBy }, - { - encoding: 'application/json', - headers: this.adminAuthHeaders(), - }, - ) - return result.data - } - - async resolveReports(opts: { - actionId: number - reportIds: number[] - createdBy?: string - }) { - const { actionId, reportIds, createdBy = 'did:example:admin' } = opts - const result = - await this.agent.api.com.atproto.admin.resolveModerationReports( - { actionId, createdBy, reportIds }, - { - encoding: 'application/json', - headers: this.adminAuthHeaders(), + const { id, subject, reason = 'X', createdBy = 'did:example:admin' } = opts + const result = await this.agent.api.com.atproto.admin.emitModerationEvent( + { + subject, + event: { + $type: 'com.atproto.admin.defs#modEventReverseTakedown', + comment: reason, }, - ) + createdBy, + }, + { + encoding: 'application/json', + headers: this.adminAuthHeaders(), + }, + ) return result.data } diff --git a/packages/pds/src/api/com/atproto/admin/takeModerationAction.ts b/packages/pds/src/api/com/atproto/admin/emitModerationEvent.ts similarity index 79% rename from packages/pds/src/api/com/atproto/admin/takeModerationAction.ts rename to packages/pds/src/api/com/atproto/admin/emitModerationEvent.ts index c07e8d04f08..d615dd0766d 100644 --- a/packages/pds/src/api/com/atproto/admin/takeModerationAction.ts +++ b/packages/pds/src/api/com/atproto/admin/emitModerationEvent.ts @@ -3,11 +3,11 @@ import AppContext from '../../../../context' import { authPassthru } from './util' export default function (server: Server, ctx: AppContext) { - server.com.atproto.admin.takeModerationAction({ + server.com.atproto.admin.emitModerationEvent({ auth: ctx.authVerifier.role, handler: async ({ req, input }) => { const { data: result } = - await ctx.appViewAgent.com.atproto.admin.takeModerationAction( + await ctx.appViewAgent.com.atproto.admin.emitModerationEvent( input.body, authPassthru(req, true), ) diff --git a/packages/pds/src/api/com/atproto/admin/getModerationAction.ts b/packages/pds/src/api/com/atproto/admin/getModerationAction.ts deleted file mode 100644 index 50b9fcde5ad..00000000000 --- a/packages/pds/src/api/com/atproto/admin/getModerationAction.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' -import { authPassthru } from './util' - -export default function (server: Server, ctx: AppContext) { - server.com.atproto.admin.getModerationAction({ - auth: ctx.authVerifier.role, - handler: async ({ req, params }) => { - const { data: resultAppview } = - await ctx.appViewAgent.com.atproto.admin.getModerationAction( - params, - authPassthru(req), - ) - return { - encoding: 'application/json', - body: resultAppview, - } - }, - }) -} diff --git a/packages/pds/src/api/com/atproto/admin/getModerationActions.ts b/packages/pds/src/api/com/atproto/admin/getModerationEvent.ts similarity index 69% rename from packages/pds/src/api/com/atproto/admin/getModerationActions.ts rename to packages/pds/src/api/com/atproto/admin/getModerationEvent.ts index d9cf61ba1ee..6984a2acbec 100644 --- a/packages/pds/src/api/com/atproto/admin/getModerationActions.ts +++ b/packages/pds/src/api/com/atproto/admin/getModerationEvent.ts @@ -3,17 +3,17 @@ import AppContext from '../../../../context' import { authPassthru } from './util' export default function (server: Server, ctx: AppContext) { - server.com.atproto.admin.getModerationActions({ + server.com.atproto.admin.getModerationEvent({ auth: ctx.authVerifier.role, handler: async ({ req, params }) => { - const { data: result } = - await ctx.appViewAgent.com.atproto.admin.getModerationActions( + const { data } = + await ctx.appViewAgent.com.atproto.admin.getModerationEvent( params, authPassthru(req), ) return { encoding: 'application/json', - body: result, + body: data, } }, }) diff --git a/packages/pds/src/api/com/atproto/admin/index.ts b/packages/pds/src/api/com/atproto/admin/index.ts index 29fdec10efe..3ff1bcdb517 100644 --- a/packages/pds/src/api/com/atproto/admin/index.ts +++ b/packages/pds/src/api/com/atproto/admin/index.ts @@ -1,18 +1,14 @@ import AppContext from '../../../../context' import { Server } from '../../../../lexicon' -import resolveModerationReports from './resolveModerationReports' -import reverseModerationAction from './reverseModerationAction' -import takeModerationAction from './takeModerationAction' +import emitModerationEvent from './emitModerationEvent' import updateSubjectStatus from './updateSubjectStatus' import getSubjectStatus from './getSubjectStatus' import getAccountInfo from './getAccountInfo' import searchRepos from './searchRepos' import getRecord from './getRecord' import getRepo from './getRepo' -import getModerationAction from './getModerationAction' -import getModerationActions from './getModerationActions' -import getModerationReport from './getModerationReport' -import getModerationReports from './getModerationReports' +import getModerationEvent from './getModerationEvent' +import queryModerationEvents from './queryModerationEvents' import enableAccountInvites from './enableAccountInvites' import disableAccountInvites from './disableAccountInvites' import disableInviteCodes from './disableInviteCodes' @@ -20,21 +16,19 @@ import getInviteCodes from './getInviteCodes' import updateAccountHandle from './updateAccountHandle' import updateAccountEmail from './updateAccountEmail' import sendEmail from './sendEmail' +import queryModerationStatuses from './queryModerationStatuses' export default function (server: Server, ctx: AppContext) { - resolveModerationReports(server, ctx) - reverseModerationAction(server, ctx) - takeModerationAction(server, ctx) + emitModerationEvent(server, ctx) updateSubjectStatus(server, ctx) getSubjectStatus(server, ctx) getAccountInfo(server, ctx) searchRepos(server, ctx) getRecord(server, ctx) getRepo(server, ctx) - getModerationAction(server, ctx) - getModerationActions(server, ctx) - getModerationReport(server, ctx) - getModerationReports(server, ctx) + getModerationEvent(server, ctx) + queryModerationEvents(server, ctx) + queryModerationStatuses(server, ctx) enableAccountInvites(server, ctx) disableAccountInvites(server, ctx) disableInviteCodes(server, ctx) diff --git a/packages/pds/src/api/com/atproto/admin/getModerationReports.ts b/packages/pds/src/api/com/atproto/admin/queryModerationEvents.ts similarity index 78% rename from packages/pds/src/api/com/atproto/admin/getModerationReports.ts rename to packages/pds/src/api/com/atproto/admin/queryModerationEvents.ts index a213504d840..55fee1d61b1 100644 --- a/packages/pds/src/api/com/atproto/admin/getModerationReports.ts +++ b/packages/pds/src/api/com/atproto/admin/queryModerationEvents.ts @@ -3,11 +3,11 @@ import AppContext from '../../../../context' import { authPassthru } from './util' export default function (server: Server, ctx: AppContext) { - server.com.atproto.admin.getModerationReports({ + server.com.atproto.admin.queryModerationEvents({ auth: ctx.authVerifier.role, handler: async ({ req, params }) => { const { data: result } = - await ctx.appViewAgent.com.atproto.admin.getModerationReports( + await ctx.appViewAgent.com.atproto.admin.queryModerationEvents( params, authPassthru(req), ) diff --git a/packages/pds/src/api/com/atproto/admin/getModerationReport.ts b/packages/pds/src/api/com/atproto/admin/queryModerationStatuses.ts similarity index 68% rename from packages/pds/src/api/com/atproto/admin/getModerationReport.ts rename to packages/pds/src/api/com/atproto/admin/queryModerationStatuses.ts index 681679c87db..e9f068018f4 100644 --- a/packages/pds/src/api/com/atproto/admin/getModerationReport.ts +++ b/packages/pds/src/api/com/atproto/admin/queryModerationStatuses.ts @@ -3,17 +3,17 @@ import AppContext from '../../../../context' import { authPassthru } from './util' export default function (server: Server, ctx: AppContext) { - server.com.atproto.admin.getModerationReport({ + server.com.atproto.admin.queryModerationStatuses({ auth: ctx.authVerifier.role, handler: async ({ req, params }) => { - const { data: resultAppview } = - await ctx.appViewAgent.com.atproto.admin.getModerationReport( + const { data } = + await ctx.appViewAgent.com.atproto.admin.queryModerationStatuses( params, authPassthru(req), ) return { encoding: 'application/json', - body: resultAppview, + body: data, } }, }) diff --git a/packages/pds/src/api/com/atproto/admin/resolveModerationReports.ts b/packages/pds/src/api/com/atproto/admin/resolveModerationReports.ts deleted file mode 100644 index 1246a2364b1..00000000000 --- a/packages/pds/src/api/com/atproto/admin/resolveModerationReports.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' -import { authPassthru } from './util' - -export default function (server: Server, ctx: AppContext) { - server.com.atproto.admin.resolveModerationReports({ - auth: ctx.authVerifier.role, - handler: async ({ req, input }) => { - const { data: result } = - await ctx.appViewAgent.com.atproto.admin.resolveModerationReports( - input.body, - authPassthru(req, true), - ) - return { - encoding: 'application/json', - body: result, - } - }, - }) -} diff --git a/packages/pds/src/api/com/atproto/admin/reverseModerationAction.ts b/packages/pds/src/api/com/atproto/admin/reverseModerationAction.ts deleted file mode 100644 index ec52e2c36c6..00000000000 --- a/packages/pds/src/api/com/atproto/admin/reverseModerationAction.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' -import { authPassthru } from './util' - -export default function (server: Server, ctx: AppContext) { - server.com.atproto.admin.reverseModerationAction({ - auth: ctx.authVerifier.role, - handler: async ({ req, input }) => { - const { data: result } = - await ctx.appViewAgent.com.atproto.admin.reverseModerationAction( - input.body, - authPassthru(req, true), - ) - - return { - encoding: 'application/json', - body: result, - } - }, - }) -} diff --git a/packages/pds/src/api/com/atproto/admin/sendEmail.ts b/packages/pds/src/api/com/atproto/admin/sendEmail.ts index b1d53c9db44..e4defa466a8 100644 --- a/packages/pds/src/api/com/atproto/admin/sendEmail.ts +++ b/packages/pds/src/api/com/atproto/admin/sendEmail.ts @@ -1,11 +1,12 @@ import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' +import { authPassthru } from './util' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.sendEmail({ auth: ctx.authVerifier.role, - handler: async ({ input, auth }) => { + handler: async ({ req, input, auth }) => { if (!auth.credentials.admin && !auth.credentials.moderator) { throw new AuthRequiredError('Insufficient privileges') } @@ -13,6 +14,7 @@ export default function (server: Server, ctx: AppContext) { const { content, recipientDid, + senderDid, subject = 'Message from Bluesky moderator', } = input.body const userInfo = await ctx.db.db @@ -29,6 +31,20 @@ export default function (server: Server, ctx: AppContext) { { content }, { subject, to: userInfo.email }, ) + await ctx.appViewAgent.api.com.atproto.admin.emitModerationEvent( + { + event: { + $type: 'com.atproto.admin.defs#modEventEmail', + subjectLine: subject, + }, + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: recipientDid, + }, + createdBy: senderDid, + }, + { ...authPassthru(req), encoding: 'application/json' }, + ) return { encoding: 'application/json', body: { sent: true }, diff --git a/packages/pds/src/api/com/atproto/moderation/util.ts b/packages/pds/src/api/com/atproto/moderation/util.ts index 89ee2f1ac92..4de1e8cd4bc 100644 --- a/packages/pds/src/api/com/atproto/moderation/util.ts +++ b/packages/pds/src/api/com/atproto/moderation/util.ts @@ -1,15 +1,8 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import { AtUri } from '@atproto/syntax' -import { ModerationAction } from '../../../../db/tables/moderation' import { ModerationReport } from '../../../../db/tables/moderation' import { InputSchema as ReportInput } from '../../../../lexicon/types/com/atproto/moderation/createReport' -import { InputSchema as ActionInput } from '../../../../lexicon/types/com/atproto/admin/takeModerationAction' -import { - ACKNOWLEDGE, - FLAG, - TAKEDOWN, - ESCALATE, -} from '../../../../lexicon/types/com/atproto/admin/defs' +import { InputSchema as ActionInput } from '../../../../lexicon/types/com/atproto/admin/emitModerationEvent' import { REASONOTHER, REASONSPAM, @@ -49,18 +42,6 @@ export const getReasonType = (reasonType: ReportInput['reasonType']) => { throw new InvalidRequestError('Invalid reason type') } -export const getAction = (action: ActionInput['action']) => { - if ( - action === TAKEDOWN || - action === FLAG || - action === ACKNOWLEDGE || - action === ESCALATE - ) { - return action as ModerationAction['action'] - } - throw new InvalidRequestError('Invalid action') -} - const reasonTypes = new Set([ REASONOTHER, REASONSPAM, diff --git a/packages/pds/src/db/tables/moderation.ts b/packages/pds/src/db/tables/moderation.ts index 061b3981634..d6e5458735e 100644 --- a/packages/pds/src/db/tables/moderation.ts +++ b/packages/pds/src/db/tables/moderation.ts @@ -1,10 +1,4 @@ import { Generated } from 'kysely' -import { - ACKNOWLEDGE, - FLAG, - TAKEDOWN, - ESCALATE, -} from '../../lexicon/types/com/atproto/admin/defs' import { REASONOTHER, REASONSPAM, @@ -21,21 +15,27 @@ export const reportResolutionTableName = 'moderation_report_resolution' export interface ModerationAction { id: Generated - action: typeof TAKEDOWN | typeof FLAG | typeof ACKNOWLEDGE | typeof ESCALATE + action: + | 'com.atproto.admin.defs#modEventTakedown' + | 'com.atproto.admin.defs#modEventAcknowledge' + | 'com.atproto.admin.defs#modEventEscalate' + | 'com.atproto.admin.defs#modEventComment' + | 'com.atproto.admin.defs#modEventLabel' + | 'com.atproto.admin.defs#modEventReport' + | 'com.atproto.admin.defs#modEventMute' + | 'com.atproto.admin.defs#modEventReverseTakedown' subjectType: 'com.atproto.admin.defs#repoRef' | 'com.atproto.repo.strongRef' subjectDid: string subjectUri: string | null subjectCid: string | null createLabelVals: string | null negateLabelVals: string | null - reason: string + comment: string | null createdAt: string createdBy: string - reversedAt: string | null - reversedBy: string | null - reversedReason: string | null durationInHours: number | null expiresAt: string | null + meta: Record | null } export interface ModerationActionSubjectBlob { diff --git a/packages/pds/src/lexicon/index.ts b/packages/pds/src/lexicon/index.ts index 6474220edad..e4a075bee9f 100644 --- a/packages/pds/src/lexicon/index.ts +++ b/packages/pds/src/lexicon/index.ts @@ -11,21 +11,18 @@ import { import { schemas } from './lexicons' import * as ComAtprotoAdminDisableAccountInvites from './types/com/atproto/admin/disableAccountInvites' import * as ComAtprotoAdminDisableInviteCodes from './types/com/atproto/admin/disableInviteCodes' +import * as ComAtprotoAdminEmitModerationEvent from './types/com/atproto/admin/emitModerationEvent' import * as ComAtprotoAdminEnableAccountInvites from './types/com/atproto/admin/enableAccountInvites' import * as ComAtprotoAdminGetAccountInfo from './types/com/atproto/admin/getAccountInfo' import * as ComAtprotoAdminGetInviteCodes from './types/com/atproto/admin/getInviteCodes' -import * as ComAtprotoAdminGetModerationAction from './types/com/atproto/admin/getModerationAction' -import * as ComAtprotoAdminGetModerationActions from './types/com/atproto/admin/getModerationActions' -import * as ComAtprotoAdminGetModerationReport from './types/com/atproto/admin/getModerationReport' -import * as ComAtprotoAdminGetModerationReports from './types/com/atproto/admin/getModerationReports' +import * as ComAtprotoAdminGetModerationEvent from './types/com/atproto/admin/getModerationEvent' import * as ComAtprotoAdminGetRecord from './types/com/atproto/admin/getRecord' import * as ComAtprotoAdminGetRepo from './types/com/atproto/admin/getRepo' import * as ComAtprotoAdminGetSubjectStatus from './types/com/atproto/admin/getSubjectStatus' -import * as ComAtprotoAdminResolveModerationReports from './types/com/atproto/admin/resolveModerationReports' -import * as ComAtprotoAdminReverseModerationAction from './types/com/atproto/admin/reverseModerationAction' +import * as ComAtprotoAdminQueryModerationEvents from './types/com/atproto/admin/queryModerationEvents' +import * as ComAtprotoAdminQueryModerationStatuses from './types/com/atproto/admin/queryModerationStatuses' import * as ComAtprotoAdminSearchRepos from './types/com/atproto/admin/searchRepos' import * as ComAtprotoAdminSendEmail from './types/com/atproto/admin/sendEmail' -import * as ComAtprotoAdminTakeModerationAction from './types/com/atproto/admin/takeModerationAction' import * as ComAtprotoAdminUpdateAccountEmail from './types/com/atproto/admin/updateAccountEmail' import * as ComAtprotoAdminUpdateAccountHandle from './types/com/atproto/admin/updateAccountHandle' import * as ComAtprotoAdminUpdateSubjectStatus from './types/com/atproto/admin/updateSubjectStatus' @@ -123,10 +120,9 @@ import * as AppBskyUnspeccedSearchActorsSkeleton from './types/app/bsky/unspecce import * as AppBskyUnspeccedSearchPostsSkeleton from './types/app/bsky/unspecced/searchPostsSkeleton' export const COM_ATPROTO_ADMIN = { - DefsTakedown: 'com.atproto.admin.defs#takedown', - DefsFlag: 'com.atproto.admin.defs#flag', - DefsAcknowledge: 'com.atproto.admin.defs#acknowledge', - DefsEscalate: 'com.atproto.admin.defs#escalate', + DefsReviewOpen: 'com.atproto.admin.defs#reviewOpen', + DefsReviewEscalated: 'com.atproto.admin.defs#reviewEscalated', + DefsReviewClosed: 'com.atproto.admin.defs#reviewClosed', } export const COM_ATPROTO_MODERATION = { DefsReasonSpam: 'com.atproto.moderation.defs#reasonSpam', @@ -220,6 +216,17 @@ export class AdminNS { return this._server.xrpc.method(nsid, cfg) } + emitModerationEvent( + cfg: ConfigOf< + AV, + ComAtprotoAdminEmitModerationEvent.Handler>, + ComAtprotoAdminEmitModerationEvent.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.admin.emitModerationEvent' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } + enableAccountInvites( cfg: ConfigOf< AV, @@ -253,47 +260,14 @@ export class AdminNS { return this._server.xrpc.method(nsid, cfg) } - getModerationAction( - cfg: ConfigOf< - AV, - ComAtprotoAdminGetModerationAction.Handler>, - ComAtprotoAdminGetModerationAction.HandlerReqCtx> - >, - ) { - const nsid = 'com.atproto.admin.getModerationAction' // @ts-ignore - return this._server.xrpc.method(nsid, cfg) - } - - getModerationActions( + getModerationEvent( cfg: ConfigOf< AV, - ComAtprotoAdminGetModerationActions.Handler>, - ComAtprotoAdminGetModerationActions.HandlerReqCtx> + ComAtprotoAdminGetModerationEvent.Handler>, + ComAtprotoAdminGetModerationEvent.HandlerReqCtx> >, ) { - const nsid = 'com.atproto.admin.getModerationActions' // @ts-ignore - return this._server.xrpc.method(nsid, cfg) - } - - getModerationReport( - cfg: ConfigOf< - AV, - ComAtprotoAdminGetModerationReport.Handler>, - ComAtprotoAdminGetModerationReport.HandlerReqCtx> - >, - ) { - const nsid = 'com.atproto.admin.getModerationReport' // @ts-ignore - return this._server.xrpc.method(nsid, cfg) - } - - getModerationReports( - cfg: ConfigOf< - AV, - ComAtprotoAdminGetModerationReports.Handler>, - ComAtprotoAdminGetModerationReports.HandlerReqCtx> - >, - ) { - const nsid = 'com.atproto.admin.getModerationReports' // @ts-ignore + const nsid = 'com.atproto.admin.getModerationEvent' // @ts-ignore return this._server.xrpc.method(nsid, cfg) } @@ -330,25 +304,25 @@ export class AdminNS { return this._server.xrpc.method(nsid, cfg) } - resolveModerationReports( + queryModerationEvents( cfg: ConfigOf< AV, - ComAtprotoAdminResolveModerationReports.Handler>, - ComAtprotoAdminResolveModerationReports.HandlerReqCtx> + ComAtprotoAdminQueryModerationEvents.Handler>, + ComAtprotoAdminQueryModerationEvents.HandlerReqCtx> >, ) { - const nsid = 'com.atproto.admin.resolveModerationReports' // @ts-ignore + const nsid = 'com.atproto.admin.queryModerationEvents' // @ts-ignore return this._server.xrpc.method(nsid, cfg) } - reverseModerationAction( + queryModerationStatuses( cfg: ConfigOf< AV, - ComAtprotoAdminReverseModerationAction.Handler>, - ComAtprotoAdminReverseModerationAction.HandlerReqCtx> + ComAtprotoAdminQueryModerationStatuses.Handler>, + ComAtprotoAdminQueryModerationStatuses.HandlerReqCtx> >, ) { - const nsid = 'com.atproto.admin.reverseModerationAction' // @ts-ignore + const nsid = 'com.atproto.admin.queryModerationStatuses' // @ts-ignore return this._server.xrpc.method(nsid, cfg) } @@ -374,17 +348,6 @@ export class AdminNS { return this._server.xrpc.method(nsid, cfg) } - takeModerationAction( - cfg: ConfigOf< - AV, - ComAtprotoAdminTakeModerationAction.Handler>, - ComAtprotoAdminTakeModerationAction.HandlerReqCtx> - >, - ) { - const nsid = 'com.atproto.admin.takeModerationAction' // @ts-ignore - return this._server.xrpc.method(nsid, cfg) - } - updateAccountEmail( cfg: ConfigOf< AV, diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index cc3f09f0c4e..25a60f90054 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -20,30 +20,33 @@ export const schemaDict = { }, }, }, - actionView: { + modEventView: { type: 'object', required: [ 'id', - 'action', + 'event', 'subject', 'subjectBlobCids', - 'reason', 'createdBy', 'createdAt', - 'resolvedReportIds', ], properties: { id: { type: 'integer', }, - action: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionType', - }, - durationInHours: { - type: 'integer', - description: - 'Indicates how long this action is meant to be in effect before automatically expiring.', + event: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#modEventTakedown', + 'lex:com.atproto.admin.defs#modEventReverseTakedown', + 'lex:com.atproto.admin.defs#modEventComment', + 'lex:com.atproto.admin.defs#modEventReport', + 'lex:com.atproto.admin.defs#modEventLabel', + 'lex:com.atproto.admin.defs#modEventAcknowledge', + 'lex:com.atproto.admin.defs#modEventEscalate', + 'lex:com.atproto.admin.defs#modEventMute', + 'lex:com.atproto.admin.defs#modEventEmail', + ], }, subject: { type: 'union', @@ -58,21 +61,6 @@ export const schemaDict = { type: 'string', }, }, - createLabelVals: { - type: 'array', - items: { - type: 'string', - }, - }, - negateLabelVals: { - type: 'array', - items: { - type: 'string', - }, - }, - reason: { - type: 'string', - }, createdBy: { type: 'string', format: 'did', @@ -81,42 +69,40 @@ export const schemaDict = { type: 'string', format: 'datetime', }, - reversal: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionReversal', + creatorHandle: { + type: 'string', }, - resolvedReportIds: { - type: 'array', - items: { - type: 'integer', - }, + subjectHandle: { + type: 'string', }, }, }, - actionViewDetail: { + modEventViewDetail: { type: 'object', required: [ 'id', - 'action', + 'event', 'subject', 'subjectBlobs', - 'reason', 'createdBy', 'createdAt', - 'resolvedReports', ], properties: { id: { type: 'integer', }, - action: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionType', - }, - durationInHours: { - type: 'integer', - description: - 'Indicates how long this action is meant to be in effect before automatically expiring.', + event: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#modEventTakedown', + 'lex:com.atproto.admin.defs#modEventReverseTakedown', + 'lex:com.atproto.admin.defs#modEventComment', + 'lex:com.atproto.admin.defs#modEventReport', + 'lex:com.atproto.admin.defs#modEventLabel', + 'lex:com.atproto.admin.defs#modEventAcknowledge', + 'lex:com.atproto.admin.defs#modEventEscalate', + 'lex:com.atproto.admin.defs#modEventMute', + ], }, subject: { type: 'union', @@ -134,67 +120,6 @@ export const schemaDict = { ref: 'lex:com.atproto.admin.defs#blobView', }, }, - createLabelVals: { - type: 'array', - items: { - type: 'string', - }, - }, - negateLabelVals: { - type: 'array', - items: { - type: 'string', - }, - }, - reason: { - type: 'string', - }, - createdBy: { - type: 'string', - format: 'did', - }, - createdAt: { - type: 'string', - format: 'datetime', - }, - reversal: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionReversal', - }, - resolvedReports: { - type: 'array', - items: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#reportView', - }, - }, - }, - }, - actionViewCurrent: { - type: 'object', - required: ['id', 'action'], - properties: { - id: { - type: 'integer', - }, - action: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionType', - }, - durationInHours: { - type: 'integer', - description: - 'Indicates how long this action is meant to be in effect before automatically expiring.', - }, - }, - }, - actionReversal: { - type: 'object', - required: ['reason', 'createdBy', 'createdAt'], - properties: { - reason: { - type: 'string', - }, createdBy: { type: 'string', format: 'did', @@ -205,35 +130,6 @@ export const schemaDict = { }, }, }, - actionType: { - type: 'string', - knownValues: [ - 'lex:com.atproto.admin.defs#takedown', - 'lex:com.atproto.admin.defs#flag', - 'lex:com.atproto.admin.defs#acknowledge', - 'lex:com.atproto.admin.defs#escalate', - ], - }, - takedown: { - type: 'token', - description: - 'Moderation action type: Takedown. Indicates that content should not be served by the PDS.', - }, - flag: { - type: 'token', - description: - 'Moderation action type: Flag. Indicates that the content was reviewed and considered to violate PDS rules, but may still be served.', - }, - acknowledge: { - type: 'token', - description: - 'Moderation action type: Acknowledge. Indicates that the content was reviewed and not considered to violate PDS rules.', - }, - escalate: { - type: 'token', - description: - 'Moderation action type: Escalate. Indicates that the content has been flagged for additional review.', - }, reportView: { type: 'object', required: [ @@ -252,7 +148,7 @@ export const schemaDict = { type: 'ref', ref: 'lex:com.atproto.moderation.defs#reasonType', }, - reason: { + comment: { type: 'string', }, subjectRepoHandle: { @@ -281,6 +177,75 @@ export const schemaDict = { }, }, }, + subjectStatusView: { + type: 'object', + required: ['id', 'subject', 'createdAt', 'updatedAt', 'reviewState'], + properties: { + id: { + type: 'integer', + }, + subject: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#repoRef', + 'lex:com.atproto.repo.strongRef', + ], + }, + subjectBlobCids: { + type: 'array', + items: { + type: 'string', + format: 'cid', + }, + }, + subjectRepoHandle: { + type: 'string', + }, + updatedAt: { + type: 'string', + format: 'datetime', + description: + 'Timestamp referencing when the last update was made to the moderation status of the subject', + }, + createdAt: { + type: 'string', + format: 'datetime', + description: + 'Timestamp referencing the first moderation status impacting event was emitted on the subject', + }, + reviewState: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#subjectReviewState', + }, + comment: { + type: 'string', + description: 'Sticky comment on the subject.', + }, + muteUntil: { + type: 'string', + format: 'datetime', + }, + lastReviewedBy: { + type: 'string', + format: 'did', + }, + lastReviewedAt: { + type: 'string', + format: 'datetime', + }, + lastReportedAt: { + type: 'string', + format: 'datetime', + }, + takendown: { + type: 'boolean', + }, + suspendUntil: { + type: 'string', + format: 'datetime', + }, + }, + }, reportViewDetail: { type: 'object', required: [ @@ -299,7 +264,7 @@ export const schemaDict = { type: 'ref', ref: 'lex:com.atproto.moderation.defs#reasonType', }, - reason: { + comment: { type: 'string', }, subject: { @@ -311,6 +276,10 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#recordViewNotFound', ], }, + subjectStatus: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#subjectStatusView', + }, reportedBy: { type: 'string', format: 'did', @@ -323,7 +292,7 @@ export const schemaDict = { type: 'array', items: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionView', + ref: 'lex:com.atproto.admin.defs#modEventView', }, }, }, @@ -628,33 +597,18 @@ export const schemaDict = { moderation: { type: 'object', properties: { - currentAction: { + subjectStatus: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionViewCurrent', + ref: 'lex:com.atproto.admin.defs#subjectStatusView', }, }, }, moderationDetail: { type: 'object', - required: ['actions', 'reports'], properties: { - currentAction: { + subjectStatus: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionViewCurrent', - }, - actions: { - type: 'array', - items: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionView', - }, - }, - reports: { - type: 'array', - items: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#reportView', - }, + ref: 'lex:com.atproto.admin.defs#subjectStatusView', }, }, }, @@ -716,74 +670,164 @@ export const schemaDict = { }, }, }, - }, - }, - ComAtprotoAdminDisableAccountInvites: { - lexicon: 1, - id: 'com.atproto.admin.disableAccountInvites', - defs: { - main: { - type: 'procedure', + subjectReviewState: { + type: 'string', + knownValues: [ + 'lex:com.atproto.admin.defs#reviewOpen', + 'lex:com.atproto.admin.defs#reviewEscalated', + 'lex:com.atproto.admin.defs#reviewClosed', + ], + }, + reviewOpen: { + type: 'token', description: - 'Disable an account from receiving new invite codes, but does not invalidate existing codes.', - input: { - encoding: 'application/json', - schema: { - type: 'object', - required: ['account'], - properties: { - account: { - type: 'string', - format: 'did', - }, - note: { - type: 'string', - description: 'Optional reason for disabled invites.', - }, - }, + 'Moderator review status of a subject: Open. Indicates that the subject needs to be reviewed by a moderator', + }, + reviewEscalated: { + type: 'token', + description: + 'Moderator review status of a subject: Escalated. Indicates that the subject was escalated for review by a moderator', + }, + reviewClosed: { + type: 'token', + description: + 'Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator', + }, + modEventTakedown: { + type: 'object', + description: 'Take down a subject permanently or temporarily', + properties: { + comment: { + type: 'string', + }, + durationInHours: { + type: 'integer', + description: + 'Indicates how long the takedown should be in effect before automatically expiring.', }, }, }, - }, - }, - ComAtprotoAdminDisableInviteCodes: { - lexicon: 1, - id: 'com.atproto.admin.disableInviteCodes', - defs: { - main: { - type: 'procedure', - description: - 'Disable some set of codes and/or all codes associated with a set of users.', - input: { - encoding: 'application/json', - schema: { - type: 'object', - properties: { - codes: { - type: 'array', - items: { - type: 'string', - }, - }, - accounts: { - type: 'array', - items: { - type: 'string', - }, - }, - }, + modEventReverseTakedown: { + type: 'object', + description: 'Revert take down action on a subject', + properties: { + comment: { + type: 'string', + description: 'Describe reasoning behind the reversal.', }, }, }, - }, - }, - ComAtprotoAdminEnableAccountInvites: { - lexicon: 1, - id: 'com.atproto.admin.enableAccountInvites', + modEventComment: { + type: 'object', + description: 'Add a comment to a subject', + required: ['comment'], + properties: { + comment: { + type: 'string', + }, + sticky: { + type: 'boolean', + description: 'Make the comment persistent on the subject', + }, + }, + }, + modEventReport: { + type: 'object', + description: 'Report a subject', + required: ['reportType'], + properties: { + comment: { + type: 'string', + }, + reportType: { + type: 'ref', + ref: 'lex:com.atproto.moderation.defs#reasonType', + }, + }, + }, + modEventLabel: { + type: 'object', + description: 'Apply/Negate labels on a subject', + required: ['createLabelVals', 'negateLabelVals'], + properties: { + comment: { + type: 'string', + }, + createLabelVals: { + type: 'array', + items: { + type: 'string', + }, + }, + negateLabelVals: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + modEventAcknowledge: { + type: 'object', + properties: { + comment: { + type: 'string', + }, + }, + }, + modEventEscalate: { + type: 'object', + properties: { + comment: { + type: 'string', + }, + }, + }, + modEventMute: { + type: 'object', + description: 'Mute incoming reports on a subject', + required: ['durationInHours'], + properties: { + comment: { + type: 'string', + }, + durationInHours: { + type: 'integer', + description: 'Indicates how long the subject should remain muted.', + }, + }, + }, + modEventUnmute: { + type: 'object', + description: 'Unmute action on a subject', + properties: { + comment: { + type: 'string', + description: 'Describe reasoning behind the reversal.', + }, + }, + }, + modEventEmail: { + type: 'object', + description: 'Keep a log of outgoing email to a user', + required: ['subjectLine'], + properties: { + subjectLine: { + type: 'string', + description: 'The subject line of the email sent to the user.', + }, + }, + }, + }, + }, + ComAtprotoAdminDisableAccountInvites: { + lexicon: 1, + id: 'com.atproto.admin.disableAccountInvites', defs: { main: { type: 'procedure', - description: "Re-enable an account's ability to receive invite codes.", + description: + 'Disable an account from receiving new invite codes, but does not invalidate existing codes.', input: { encoding: 'application/json', schema: { @@ -796,7 +840,7 @@ export const schemaDict = { }, note: { type: 'string', - description: 'Optional reason for enabled invites.', + description: 'Optional reason for disabled invites.', }, }, }, @@ -804,20 +848,83 @@ export const schemaDict = { }, }, }, - ComAtprotoAdminGetAccountInfo: { + ComAtprotoAdminDisableInviteCodes: { lexicon: 1, - id: 'com.atproto.admin.getAccountInfo', + id: 'com.atproto.admin.disableInviteCodes', defs: { main: { - type: 'query', - description: 'Get details about an account.', - parameters: { - type: 'params', - required: ['did'], - properties: { - did: { - type: 'string', - format: 'did', + type: 'procedure', + description: + 'Disable some set of codes and/or all codes associated with a set of users.', + input: { + encoding: 'application/json', + schema: { + type: 'object', + properties: { + codes: { + type: 'array', + items: { + type: 'string', + }, + }, + accounts: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + }, + }, + }, + }, + ComAtprotoAdminEmitModerationEvent: { + lexicon: 1, + id: 'com.atproto.admin.emitModerationEvent', + defs: { + main: { + type: 'procedure', + description: 'Take a moderation action on an actor.', + input: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['event', 'subject', 'createdBy'], + properties: { + event: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#modEventTakedown', + 'lex:com.atproto.admin.defs#modEventAcknowledge', + 'lex:com.atproto.admin.defs#modEventEscalate', + 'lex:com.atproto.admin.defs#modEventComment', + 'lex:com.atproto.admin.defs#modEventLabel', + 'lex:com.atproto.admin.defs#modEventReport', + 'lex:com.atproto.admin.defs#modEventMute', + 'lex:com.atproto.admin.defs#modEventReverseTakedown', + 'lex:com.atproto.admin.defs#modEventUnmute', + 'lex:com.atproto.admin.defs#modEventEmail', + ], + }, + subject: { + type: 'union', + refs: [ + 'lex:com.atproto.admin.defs#repoRef', + 'lex:com.atproto.repo.strongRef', + ], + }, + subjectBlobCids: { + type: 'array', + items: { + type: 'string', + format: 'cid', + }, + }, + createdBy: { + type: 'string', + format: 'did', + }, }, }, }, @@ -825,53 +932,37 @@ export const schemaDict = { encoding: 'application/json', schema: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#accountView', + ref: 'lex:com.atproto.admin.defs#modEventView', }, }, + errors: [ + { + name: 'SubjectHasAction', + }, + ], }, }, }, - ComAtprotoAdminGetInviteCodes: { + ComAtprotoAdminEnableAccountInvites: { lexicon: 1, - id: 'com.atproto.admin.getInviteCodes', + id: 'com.atproto.admin.enableAccountInvites', defs: { main: { - type: 'query', - description: 'Get an admin view of invite codes.', - parameters: { - type: 'params', - properties: { - sort: { - type: 'string', - knownValues: ['recent', 'usage'], - default: 'recent', - }, - limit: { - type: 'integer', - minimum: 1, - maximum: 500, - default: 100, - }, - cursor: { - type: 'string', - }, - }, - }, - output: { + type: 'procedure', + description: "Re-enable an account's ability to receive invite codes.", + input: { encoding: 'application/json', schema: { type: 'object', - required: ['codes'], + required: ['account'], properties: { - cursor: { + account: { type: 'string', + format: 'did', }, - codes: { - type: 'array', - items: { - type: 'ref', - ref: 'lex:com.atproto.server.defs#inviteCode', - }, + note: { + type: 'string', + description: 'Optional reason for enabled invites.', }, }, }, @@ -879,19 +970,20 @@ export const schemaDict = { }, }, }, - ComAtprotoAdminGetModerationAction: { + ComAtprotoAdminGetAccountInfo: { lexicon: 1, - id: 'com.atproto.admin.getModerationAction', + id: 'com.atproto.admin.getAccountInfo', defs: { main: { type: 'query', - description: 'Get details about a moderation action.', + description: 'Get details about an account.', parameters: { type: 'params', - required: ['id'], + required: ['did'], properties: { - id: { - type: 'integer', + did: { + type: 'string', + format: 'did', }, }, }, @@ -899,30 +991,32 @@ export const schemaDict = { encoding: 'application/json', schema: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionViewDetail', + ref: 'lex:com.atproto.admin.defs#accountView', }, }, }, }, }, - ComAtprotoAdminGetModerationActions: { + ComAtprotoAdminGetInviteCodes: { lexicon: 1, - id: 'com.atproto.admin.getModerationActions', + id: 'com.atproto.admin.getInviteCodes', defs: { main: { type: 'query', - description: 'Get a list of moderation actions related to a subject.', + description: 'Get an admin view of invite codes.', parameters: { type: 'params', properties: { - subject: { + sort: { type: 'string', + knownValues: ['recent', 'usage'], + default: 'recent', }, limit: { type: 'integer', minimum: 1, - maximum: 100, - default: 50, + maximum: 500, + default: 100, }, cursor: { type: 'string', @@ -933,16 +1027,16 @@ export const schemaDict = { encoding: 'application/json', schema: { type: 'object', - required: ['actions'], + required: ['codes'], properties: { cursor: { type: 'string', }, - actions: { + codes: { type: 'array', items: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionView', + ref: 'lex:com.atproto.server.defs#inviteCode', }, }, }, @@ -951,13 +1045,13 @@ export const schemaDict = { }, }, }, - ComAtprotoAdminGetModerationReport: { + ComAtprotoAdminGetModerationEvent: { lexicon: 1, - id: 'com.atproto.admin.getModerationReport', + id: 'com.atproto.admin.getModerationEvent', defs: { main: { type: 'query', - description: 'Get details about a moderation report.', + description: 'Get details about a moderation event.', parameters: { type: 'params', required: ['id'], @@ -971,89 +1065,7 @@ export const schemaDict = { encoding: 'application/json', schema: { type: 'ref', - ref: 'lex:com.atproto.admin.defs#reportViewDetail', - }, - }, - }, - }, - }, - ComAtprotoAdminGetModerationReports: { - lexicon: 1, - id: 'com.atproto.admin.getModerationReports', - defs: { - main: { - type: 'query', - description: 'Get moderation reports related to a subject.', - parameters: { - type: 'params', - properties: { - subject: { - type: 'string', - }, - ignoreSubjects: { - type: 'array', - items: { - type: 'string', - }, - }, - actionedBy: { - type: 'string', - format: 'did', - description: - 'Get all reports that were actioned by a specific moderator.', - }, - reporters: { - type: 'array', - items: { - type: 'string', - }, - description: 'Filter reports made by one or more DIDs.', - }, - resolved: { - type: 'boolean', - }, - actionType: { - type: 'string', - knownValues: [ - 'com.atproto.admin.defs#takedown', - 'com.atproto.admin.defs#flag', - 'com.atproto.admin.defs#acknowledge', - 'com.atproto.admin.defs#escalate', - ], - }, - limit: { - type: 'integer', - minimum: 1, - maximum: 100, - default: 50, - }, - cursor: { - type: 'string', - }, - reverse: { - type: 'boolean', - description: - 'Reverse the order of the returned records. When true, returns reports in chronological order.', - }, - }, - }, - output: { - encoding: 'application/json', - schema: { - type: 'object', - required: ['reports'], - properties: { - cursor: { - type: 'string', - }, - reports: { - type: 'array', - items: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#reportView', - }, - }, - }, + ref: 'lex:com.atproto.admin.defs#modEventViewDetail', }, }, }, @@ -1176,76 +1188,180 @@ export const schemaDict = { }, }, }, - ComAtprotoAdminResolveModerationReports: { + ComAtprotoAdminQueryModerationEvents: { lexicon: 1, - id: 'com.atproto.admin.resolveModerationReports', + id: 'com.atproto.admin.queryModerationEvents', defs: { main: { - type: 'procedure', - description: 'Resolve moderation reports by an action.', - input: { + type: 'query', + description: 'List moderation events related to a subject.', + parameters: { + type: 'params', + properties: { + types: { + type: 'array', + items: { + type: 'string', + }, + description: + 'The types of events (fully qualified string in the format of com.atproto.admin#modEvent) to filter by. If not specified, all events are returned.', + }, + createdBy: { + type: 'string', + format: 'did', + }, + sortDirection: { + type: 'string', + default: 'desc', + enum: ['asc', 'desc'], + description: + 'Sort direction for the events. Defaults to descending order of created at timestamp.', + }, + subject: { + type: 'string', + format: 'uri', + }, + includeAllUserRecords: { + type: 'boolean', + default: false, + description: + 'If true, events on all record types (posts, lists, profile etc.) owned by the did are returned', + }, + limit: { + type: 'integer', + minimum: 1, + maximum: 100, + default: 50, + }, + cursor: { + type: 'string', + }, + }, + }, + output: { encoding: 'application/json', schema: { type: 'object', - required: ['actionId', 'reportIds', 'createdBy'], + required: ['events'], properties: { - actionId: { - type: 'integer', + cursor: { + type: 'string', }, - reportIds: { + events: { type: 'array', items: { - type: 'integer', + type: 'ref', + ref: 'lex:com.atproto.admin.defs#modEventView', }, }, - createdBy: { - type: 'string', - format: 'did', - }, }, }, }, - output: { - encoding: 'application/json', - schema: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionView', - }, - }, }, }, }, - ComAtprotoAdminReverseModerationAction: { + ComAtprotoAdminQueryModerationStatuses: { lexicon: 1, - id: 'com.atproto.admin.reverseModerationAction', + id: 'com.atproto.admin.queryModerationStatuses', defs: { main: { - type: 'procedure', - description: 'Reverse a moderation action.', - input: { - encoding: 'application/json', - schema: { - type: 'object', - required: ['id', 'reason', 'createdBy'], - properties: { - id: { - type: 'integer', - }, - reason: { - type: 'string', - }, - createdBy: { + type: 'query', + description: 'View moderation statuses of subjects (record or repo).', + parameters: { + type: 'params', + properties: { + subject: { + type: 'string', + format: 'uri', + }, + comment: { + type: 'string', + description: 'Search subjects by keyword from comments', + }, + reportedAfter: { + type: 'string', + format: 'datetime', + description: 'Search subjects reported after a given timestamp', + }, + reportedBefore: { + type: 'string', + format: 'datetime', + description: 'Search subjects reported before a given timestamp', + }, + reviewedAfter: { + type: 'string', + format: 'datetime', + description: 'Search subjects reviewed after a given timestamp', + }, + reviewedBefore: { + type: 'string', + format: 'datetime', + description: 'Search subjects reviewed before a given timestamp', + }, + includeMuted: { + type: 'boolean', + description: + "By default, we don't include muted subjects in the results. Set this to true to include them.", + }, + reviewState: { + type: 'string', + description: 'Specify when fetching subjects in a certain state', + }, + ignoreSubjects: { + type: 'array', + items: { type: 'string', - format: 'did', + format: 'uri', }, }, + lastReviewedBy: { + type: 'string', + format: 'did', + description: + 'Get all subject statuses that were reviewed by a specific moderator', + }, + sortField: { + type: 'string', + default: 'lastReportedAt', + enum: ['lastReviewedAt', 'lastReportedAt'], + }, + sortDirection: { + type: 'string', + default: 'desc', + enum: ['asc', 'desc'], + }, + takendown: { + type: 'boolean', + description: 'Get subjects that were taken down', + }, + limit: { + type: 'integer', + minimum: 1, + maximum: 100, + default: 50, + }, + cursor: { + type: 'string', + }, }, }, output: { encoding: 'application/json', schema: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionView', + type: 'object', + required: ['subjectStatuses'], + properties: { + cursor: { + type: 'string', + }, + subjectStatuses: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.admin.defs#subjectStatusView', + }, + }, + }, }, }, }, @@ -1312,7 +1428,7 @@ export const schemaDict = { encoding: 'application/json', schema: { type: 'object', - required: ['recipientDid', 'content'], + required: ['recipientDid', 'content', 'senderDid'], properties: { recipientDid: { type: 'string', @@ -1324,6 +1440,10 @@ export const schemaDict = { subject: { type: 'string', }, + senderDid: { + type: 'string', + format: 'did', + }, }, }, }, @@ -1342,83 +1462,6 @@ export const schemaDict = { }, }, }, - ComAtprotoAdminTakeModerationAction: { - lexicon: 1, - id: 'com.atproto.admin.takeModerationAction', - defs: { - main: { - type: 'procedure', - description: 'Take a moderation action on an actor.', - input: { - encoding: 'application/json', - schema: { - type: 'object', - required: ['action', 'subject', 'reason', 'createdBy'], - properties: { - action: { - type: 'string', - knownValues: [ - 'com.atproto.admin.defs#takedown', - 'com.atproto.admin.defs#flag', - 'com.atproto.admin.defs#acknowledge', - ], - }, - subject: { - type: 'union', - refs: [ - 'lex:com.atproto.admin.defs#repoRef', - 'lex:com.atproto.repo.strongRef', - ], - }, - subjectBlobCids: { - type: 'array', - items: { - type: 'string', - format: 'cid', - }, - }, - createLabelVals: { - type: 'array', - items: { - type: 'string', - }, - }, - negateLabelVals: { - type: 'array', - items: { - type: 'string', - }, - }, - reason: { - type: 'string', - }, - durationInHours: { - type: 'integer', - description: - 'Indicates how long this action is meant to be in effect before automatically expiring.', - }, - createdBy: { - type: 'string', - format: 'did', - }, - }, - }, - }, - output: { - encoding: 'application/json', - schema: { - type: 'ref', - ref: 'lex:com.atproto.admin.defs#actionView', - }, - }, - errors: [ - { - name: 'SubjectHasAction', - }, - ], - }, - }, - }, ComAtprotoAdminUpdateAccountEmail: { lexicon: 1, id: 'com.atproto.admin.updateAccountEmail', @@ -7627,23 +7670,20 @@ export const ids = { ComAtprotoAdminDisableAccountInvites: 'com.atproto.admin.disableAccountInvites', ComAtprotoAdminDisableInviteCodes: 'com.atproto.admin.disableInviteCodes', + ComAtprotoAdminEmitModerationEvent: 'com.atproto.admin.emitModerationEvent', ComAtprotoAdminEnableAccountInvites: 'com.atproto.admin.enableAccountInvites', ComAtprotoAdminGetAccountInfo: 'com.atproto.admin.getAccountInfo', ComAtprotoAdminGetInviteCodes: 'com.atproto.admin.getInviteCodes', - ComAtprotoAdminGetModerationAction: 'com.atproto.admin.getModerationAction', - ComAtprotoAdminGetModerationActions: 'com.atproto.admin.getModerationActions', - ComAtprotoAdminGetModerationReport: 'com.atproto.admin.getModerationReport', - ComAtprotoAdminGetModerationReports: 'com.atproto.admin.getModerationReports', + ComAtprotoAdminGetModerationEvent: 'com.atproto.admin.getModerationEvent', ComAtprotoAdminGetRecord: 'com.atproto.admin.getRecord', ComAtprotoAdminGetRepo: 'com.atproto.admin.getRepo', ComAtprotoAdminGetSubjectStatus: 'com.atproto.admin.getSubjectStatus', - ComAtprotoAdminResolveModerationReports: - 'com.atproto.admin.resolveModerationReports', - ComAtprotoAdminReverseModerationAction: - 'com.atproto.admin.reverseModerationAction', + ComAtprotoAdminQueryModerationEvents: + 'com.atproto.admin.queryModerationEvents', + ComAtprotoAdminQueryModerationStatuses: + 'com.atproto.admin.queryModerationStatuses', ComAtprotoAdminSearchRepos: 'com.atproto.admin.searchRepos', ComAtprotoAdminSendEmail: 'com.atproto.admin.sendEmail', - ComAtprotoAdminTakeModerationAction: 'com.atproto.admin.takeModerationAction', ComAtprotoAdminUpdateAccountEmail: 'com.atproto.admin.updateAccountEmail', ComAtprotoAdminUpdateAccountHandle: 'com.atproto.admin.updateAccountHandle', ComAtprotoAdminUpdateSubjectStatus: 'com.atproto.admin.updateSubjectStatus', diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts index 8a21c42119e..27f080cbe31 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts @@ -28,43 +28,55 @@ export function validateStatusAttr(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#statusAttr', v) } -export interface ActionView { +export interface ModEventView { id: number - action: ActionType - /** Indicates how long this action is meant to be in effect before automatically expiring. */ - durationInHours?: number + event: + | ModEventTakedown + | ModEventReverseTakedown + | ModEventComment + | ModEventReport + | ModEventLabel + | ModEventAcknowledge + | ModEventEscalate + | ModEventMute + | ModEventEmail + | { $type: string; [k: string]: unknown } subject: | RepoRef | ComAtprotoRepoStrongRef.Main | { $type: string; [k: string]: unknown } subjectBlobCids: string[] - createLabelVals?: string[] - negateLabelVals?: string[] - reason: string createdBy: string createdAt: string - reversal?: ActionReversal - resolvedReportIds: number[] + creatorHandle?: string + subjectHandle?: string [k: string]: unknown } -export function isActionView(v: unknown): v is ActionView { +export function isModEventView(v: unknown): v is ModEventView { return ( isObj(v) && hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#actionView' + v.$type === 'com.atproto.admin.defs#modEventView' ) } -export function validateActionView(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#actionView', v) +export function validateModEventView(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventView', v) } -export interface ActionViewDetail { +export interface ModEventViewDetail { id: number - action: ActionType - /** Indicates how long this action is meant to be in effect before automatically expiring. */ - durationInHours?: number + event: + | ModEventTakedown + | ModEventReverseTakedown + | ModEventComment + | ModEventReport + | ModEventLabel + | ModEventAcknowledge + | ModEventEscalate + | ModEventMute + | { $type: string; [k: string]: unknown } subject: | RepoView | RepoViewNotFound @@ -72,123 +84,100 @@ export interface ActionViewDetail { | RecordViewNotFound | { $type: string; [k: string]: unknown } subjectBlobs: BlobView[] - createLabelVals?: string[] - negateLabelVals?: string[] - reason: string createdBy: string createdAt: string - reversal?: ActionReversal - resolvedReports: ReportView[] [k: string]: unknown } -export function isActionViewDetail(v: unknown): v is ActionViewDetail { +export function isModEventViewDetail(v: unknown): v is ModEventViewDetail { return ( isObj(v) && hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#actionViewDetail' + v.$type === 'com.atproto.admin.defs#modEventViewDetail' ) } -export function validateActionViewDetail(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#actionViewDetail', v) +export function validateModEventViewDetail(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventViewDetail', v) } -export interface ActionViewCurrent { +export interface ReportView { id: number - action: ActionType - /** Indicates how long this action is meant to be in effect before automatically expiring. */ - durationInHours?: number - [k: string]: unknown -} - -export function isActionViewCurrent(v: unknown): v is ActionViewCurrent { - return ( - isObj(v) && - hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#actionViewCurrent' - ) -} - -export function validateActionViewCurrent(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#actionViewCurrent', v) -} - -export interface ActionReversal { - reason: string - createdBy: string + reasonType: ComAtprotoModerationDefs.ReasonType + comment?: string + subjectRepoHandle?: string + subject: + | RepoRef + | ComAtprotoRepoStrongRef.Main + | { $type: string; [k: string]: unknown } + reportedBy: string createdAt: string + resolvedByActionIds: number[] [k: string]: unknown } -export function isActionReversal(v: unknown): v is ActionReversal { +export function isReportView(v: unknown): v is ReportView { return ( isObj(v) && hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#actionReversal' + v.$type === 'com.atproto.admin.defs#reportView' ) } -export function validateActionReversal(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#actionReversal', v) +export function validateReportView(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#reportView', v) } -export type ActionType = - | 'lex:com.atproto.admin.defs#takedown' - | 'lex:com.atproto.admin.defs#flag' - | 'lex:com.atproto.admin.defs#acknowledge' - | 'lex:com.atproto.admin.defs#escalate' - | (string & {}) - -/** Moderation action type: Takedown. Indicates that content should not be served by the PDS. */ -export const TAKEDOWN = 'com.atproto.admin.defs#takedown' -/** Moderation action type: Flag. Indicates that the content was reviewed and considered to violate PDS rules, but may still be served. */ -export const FLAG = 'com.atproto.admin.defs#flag' -/** Moderation action type: Acknowledge. Indicates that the content was reviewed and not considered to violate PDS rules. */ -export const ACKNOWLEDGE = 'com.atproto.admin.defs#acknowledge' -/** Moderation action type: Escalate. Indicates that the content has been flagged for additional review. */ -export const ESCALATE = 'com.atproto.admin.defs#escalate' - -export interface ReportView { +export interface SubjectStatusView { id: number - reasonType: ComAtprotoModerationDefs.ReasonType - reason?: string - subjectRepoHandle?: string subject: | RepoRef | ComAtprotoRepoStrongRef.Main | { $type: string; [k: string]: unknown } - reportedBy: string + subjectBlobCids?: string[] + subjectRepoHandle?: string + /** Timestamp referencing when the last update was made to the moderation status of the subject */ + updatedAt: string + /** Timestamp referencing the first moderation status impacting event was emitted on the subject */ createdAt: string - resolvedByActionIds: number[] + reviewState: SubjectReviewState + /** Sticky comment on the subject. */ + comment?: string + muteUntil?: string + lastReviewedBy?: string + lastReviewedAt?: string + lastReportedAt?: string + takendown?: boolean + suspendUntil?: string [k: string]: unknown } -export function isReportView(v: unknown): v is ReportView { +export function isSubjectStatusView(v: unknown): v is SubjectStatusView { return ( isObj(v) && hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#reportView' + v.$type === 'com.atproto.admin.defs#subjectStatusView' ) } -export function validateReportView(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#reportView', v) +export function validateSubjectStatusView(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#subjectStatusView', v) } export interface ReportViewDetail { id: number reasonType: ComAtprotoModerationDefs.ReasonType - reason?: string + comment?: string subject: | RepoView | RepoViewNotFound | RecordView | RecordViewNotFound | { $type: string; [k: string]: unknown } + subjectStatus?: SubjectStatusView reportedBy: string createdAt: string - resolvedByActions: ActionView[] + resolvedByActions: ModEventView[] [k: string]: unknown } @@ -400,7 +389,7 @@ export function validateRecordViewNotFound(v: unknown): ValidationResult { } export interface Moderation { - currentAction?: ActionViewCurrent + subjectStatus?: SubjectStatusView [k: string]: unknown } @@ -417,9 +406,7 @@ export function validateModeration(v: unknown): ValidationResult { } export interface ModerationDetail { - currentAction?: ActionViewCurrent - actions: ActionView[] - reports: ReportView[] + subjectStatus?: SubjectStatusView [k: string]: unknown } @@ -496,3 +483,208 @@ export function isVideoDetails(v: unknown): v is VideoDetails { export function validateVideoDetails(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#videoDetails', v) } + +export type SubjectReviewState = + | 'lex:com.atproto.admin.defs#reviewOpen' + | 'lex:com.atproto.admin.defs#reviewEscalated' + | 'lex:com.atproto.admin.defs#reviewClosed' + | (string & {}) + +/** Moderator review status of a subject: Open. Indicates that the subject needs to be reviewed by a moderator */ +export const REVIEWOPEN = 'com.atproto.admin.defs#reviewOpen' +/** Moderator review status of a subject: Escalated. Indicates that the subject was escalated for review by a moderator */ +export const REVIEWESCALATED = 'com.atproto.admin.defs#reviewEscalated' +/** Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator */ +export const REVIEWCLOSED = 'com.atproto.admin.defs#reviewClosed' + +/** Take down a subject permanently or temporarily */ +export interface ModEventTakedown { + comment?: string + /** Indicates how long the takedown should be in effect before automatically expiring. */ + durationInHours?: number + [k: string]: unknown +} + +export function isModEventTakedown(v: unknown): v is ModEventTakedown { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventTakedown' + ) +} + +export function validateModEventTakedown(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventTakedown', v) +} + +/** Revert take down action on a subject */ +export interface ModEventReverseTakedown { + /** Describe reasoning behind the reversal. */ + comment?: string + [k: string]: unknown +} + +export function isModEventReverseTakedown( + v: unknown, +): v is ModEventReverseTakedown { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventReverseTakedown' + ) +} + +export function validateModEventReverseTakedown(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventReverseTakedown', v) +} + +/** Add a comment to a subject */ +export interface ModEventComment { + comment: string + /** Make the comment persistent on the subject */ + sticky?: boolean + [k: string]: unknown +} + +export function isModEventComment(v: unknown): v is ModEventComment { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventComment' + ) +} + +export function validateModEventComment(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventComment', v) +} + +/** Report a subject */ +export interface ModEventReport { + comment?: string + reportType: ComAtprotoModerationDefs.ReasonType + [k: string]: unknown +} + +export function isModEventReport(v: unknown): v is ModEventReport { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventReport' + ) +} + +export function validateModEventReport(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventReport', v) +} + +/** Apply/Negate labels on a subject */ +export interface ModEventLabel { + comment?: string + createLabelVals: string[] + negateLabelVals: string[] + [k: string]: unknown +} + +export function isModEventLabel(v: unknown): v is ModEventLabel { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventLabel' + ) +} + +export function validateModEventLabel(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventLabel', v) +} + +export interface ModEventAcknowledge { + comment?: string + [k: string]: unknown +} + +export function isModEventAcknowledge(v: unknown): v is ModEventAcknowledge { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventAcknowledge' + ) +} + +export function validateModEventAcknowledge(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventAcknowledge', v) +} + +export interface ModEventEscalate { + comment?: string + [k: string]: unknown +} + +export function isModEventEscalate(v: unknown): v is ModEventEscalate { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventEscalate' + ) +} + +export function validateModEventEscalate(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventEscalate', v) +} + +/** Mute incoming reports on a subject */ +export interface ModEventMute { + comment?: string + /** Indicates how long the subject should remain muted. */ + durationInHours: number + [k: string]: unknown +} + +export function isModEventMute(v: unknown): v is ModEventMute { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventMute' + ) +} + +export function validateModEventMute(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventMute', v) +} + +/** Unmute action on a subject */ +export interface ModEventUnmute { + /** Describe reasoning behind the reversal. */ + comment?: string + [k: string]: unknown +} + +export function isModEventUnmute(v: unknown): v is ModEventUnmute { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventUnmute' + ) +} + +export function validateModEventUnmute(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventUnmute', v) +} + +/** Keep a log of outgoing email to a user */ +export interface ModEventEmail { + /** The subject line of the email sent to the user. */ + subjectLine: string + [k: string]: unknown +} + +export function isModEventEmail(v: unknown): v is ModEventEmail { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventEmail' + ) +} + +export function validateModEventEmail(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventEmail', v) +} diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/takeModerationAction.ts b/packages/pds/src/lexicon/types/com/atproto/admin/emitModerationEvent.ts similarity index 71% rename from packages/bsky/src/lexicon/types/com/atproto/admin/takeModerationAction.ts rename to packages/pds/src/lexicon/types/com/atproto/admin/emitModerationEvent.ts index 33877d90d11..df44702b51c 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/takeModerationAction.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/emitModerationEvent.ts @@ -13,26 +13,28 @@ import * as ComAtprotoRepoStrongRef from '../repo/strongRef' export interface QueryParams {} export interface InputSchema { - action: - | 'com.atproto.admin.defs#takedown' - | 'com.atproto.admin.defs#flag' - | 'com.atproto.admin.defs#acknowledge' - | (string & {}) + event: + | ComAtprotoAdminDefs.ModEventTakedown + | ComAtprotoAdminDefs.ModEventAcknowledge + | ComAtprotoAdminDefs.ModEventEscalate + | ComAtprotoAdminDefs.ModEventComment + | ComAtprotoAdminDefs.ModEventLabel + | ComAtprotoAdminDefs.ModEventReport + | ComAtprotoAdminDefs.ModEventMute + | ComAtprotoAdminDefs.ModEventReverseTakedown + | ComAtprotoAdminDefs.ModEventUnmute + | ComAtprotoAdminDefs.ModEventEmail + | { $type: string; [k: string]: unknown } subject: | ComAtprotoAdminDefs.RepoRef | ComAtprotoRepoStrongRef.Main | { $type: string; [k: string]: unknown } subjectBlobCids?: string[] - createLabelVals?: string[] - negateLabelVals?: string[] - reason: string - /** Indicates how long this action is meant to be in effect before automatically expiring. */ - durationInHours?: number createdBy: string [k: string]: unknown } -export type OutputSchema = ComAtprotoAdminDefs.ActionView +export type OutputSchema = ComAtprotoAdminDefs.ModEventView export interface HandlerInput { encoding: 'application/json' diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/getModerationAction.ts b/packages/pds/src/lexicon/types/com/atproto/admin/getModerationAction.ts deleted file mode 100644 index 2ab52f237cc..00000000000 --- a/packages/pds/src/lexicon/types/com/atproto/admin/getModerationAction.ts +++ /dev/null @@ -1,41 +0,0 @@ -/** - * GENERATED CODE - DO NOT MODIFY - */ -import express from 'express' -import { ValidationResult, BlobRef } from '@atproto/lexicon' -import { lexicons } from '../../../../lexicons' -import { isObj, hasProp } from '../../../../util' -import { CID } from 'multiformats/cid' -import { HandlerAuth } from '@atproto/xrpc-server' -import * as ComAtprotoAdminDefs from './defs' - -export interface QueryParams { - id: number -} - -export type InputSchema = undefined -export type OutputSchema = ComAtprotoAdminDefs.ActionViewDetail -export type HandlerInput = undefined - -export interface HandlerSuccess { - encoding: 'application/json' - body: OutputSchema - headers?: { [key: string]: string } -} - -export interface HandlerError { - status: number - message?: string -} - -export type HandlerOutput = HandlerError | HandlerSuccess -export type HandlerReqCtx = { - auth: HA - params: QueryParams - input: HandlerInput - req: express.Request - res: express.Response -} -export type Handler = ( - ctx: HandlerReqCtx, -) => Promise | HandlerOutput diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/getModerationReport.ts b/packages/pds/src/lexicon/types/com/atproto/admin/getModerationEvent.ts similarity index 94% rename from packages/pds/src/lexicon/types/com/atproto/admin/getModerationReport.ts rename to packages/pds/src/lexicon/types/com/atproto/admin/getModerationEvent.ts index 28d714453f2..7de567a73db 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/getModerationReport.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/getModerationEvent.ts @@ -14,7 +14,7 @@ export interface QueryParams { } export type InputSchema = undefined -export type OutputSchema = ComAtprotoAdminDefs.ReportViewDetail +export type OutputSchema = ComAtprotoAdminDefs.ModEventViewDetail export type HandlerInput = undefined export interface HandlerSuccess { diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/getModerationActions.ts b/packages/pds/src/lexicon/types/com/atproto/admin/queryModerationEvents.ts similarity index 69% rename from packages/pds/src/lexicon/types/com/atproto/admin/getModerationActions.ts rename to packages/pds/src/lexicon/types/com/atproto/admin/queryModerationEvents.ts index 4c29f965df6..f3c4f1fbb95 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/getModerationActions.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/queryModerationEvents.ts @@ -10,7 +10,14 @@ import { HandlerAuth } from '@atproto/xrpc-server' import * as ComAtprotoAdminDefs from './defs' export interface QueryParams { + /** The types of events (fully qualified string in the format of com.atproto.admin#modEvent) to filter by. If not specified, all events are returned. */ + types?: string[] + createdBy?: string + /** Sort direction for the events. Defaults to descending order of created at timestamp. */ + sortDirection: 'asc' | 'desc' subject?: string + /** If true, events on all record types (posts, lists, profile etc.) owned by the did are returned */ + includeAllUserRecords: boolean limit: number cursor?: string } @@ -19,7 +26,7 @@ export type InputSchema = undefined export interface OutputSchema { cursor?: string - actions: ComAtprotoAdminDefs.ActionView[] + events: ComAtprotoAdminDefs.ModEventView[] [k: string]: unknown } diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/getModerationReports.ts b/packages/pds/src/lexicon/types/com/atproto/admin/queryModerationStatuses.ts similarity index 56% rename from packages/pds/src/lexicon/types/com/atproto/admin/getModerationReports.ts rename to packages/pds/src/lexicon/types/com/atproto/admin/queryModerationStatuses.ts index b80811cf213..d4e55aff386 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/getModerationReports.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/queryModerationStatuses.ts @@ -11,29 +11,36 @@ import * as ComAtprotoAdminDefs from './defs' export interface QueryParams { subject?: string + /** Search subjects by keyword from comments */ + comment?: string + /** Search subjects reported after a given timestamp */ + reportedAfter?: string + /** Search subjects reported before a given timestamp */ + reportedBefore?: string + /** Search subjects reviewed after a given timestamp */ + reviewedAfter?: string + /** Search subjects reviewed before a given timestamp */ + reviewedBefore?: string + /** By default, we don't include muted subjects in the results. Set this to true to include them. */ + includeMuted?: boolean + /** Specify when fetching subjects in a certain state */ + reviewState?: string ignoreSubjects?: string[] - /** Get all reports that were actioned by a specific moderator. */ - actionedBy?: string - /** Filter reports made by one or more DIDs. */ - reporters?: string[] - resolved?: boolean - actionType?: - | 'com.atproto.admin.defs#takedown' - | 'com.atproto.admin.defs#flag' - | 'com.atproto.admin.defs#acknowledge' - | 'com.atproto.admin.defs#escalate' - | (string & {}) + /** Get all subject statuses that were reviewed by a specific moderator */ + lastReviewedBy?: string + sortField: 'lastReviewedAt' | 'lastReportedAt' + sortDirection: 'asc' | 'desc' + /** Get subjects that were taken down */ + takendown?: boolean limit: number cursor?: string - /** Reverse the order of the returned records. When true, returns reports in chronological order. */ - reverse?: boolean } export type InputSchema = undefined export interface OutputSchema { cursor?: string - reports: ComAtprotoAdminDefs.ReportView[] + subjectStatuses: ComAtprotoAdminDefs.SubjectStatusView[] [k: string]: unknown } diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/resolveModerationReports.ts b/packages/pds/src/lexicon/types/com/atproto/admin/resolveModerationReports.ts deleted file mode 100644 index e3f4d028202..00000000000 --- a/packages/pds/src/lexicon/types/com/atproto/admin/resolveModerationReports.ts +++ /dev/null @@ -1,49 +0,0 @@ -/** - * GENERATED CODE - DO NOT MODIFY - */ -import express from 'express' -import { ValidationResult, BlobRef } from '@atproto/lexicon' -import { lexicons } from '../../../../lexicons' -import { isObj, hasProp } from '../../../../util' -import { CID } from 'multiformats/cid' -import { HandlerAuth } from '@atproto/xrpc-server' -import * as ComAtprotoAdminDefs from './defs' - -export interface QueryParams {} - -export interface InputSchema { - actionId: number - reportIds: number[] - createdBy: string - [k: string]: unknown -} - -export type OutputSchema = ComAtprotoAdminDefs.ActionView - -export interface HandlerInput { - encoding: 'application/json' - body: InputSchema -} - -export interface HandlerSuccess { - encoding: 'application/json' - body: OutputSchema - headers?: { [key: string]: string } -} - -export interface HandlerError { - status: number - message?: string -} - -export type HandlerOutput = HandlerError | HandlerSuccess -export type HandlerReqCtx = { - auth: HA - params: QueryParams - input: HandlerInput - req: express.Request - res: express.Response -} -export type Handler = ( - ctx: HandlerReqCtx, -) => Promise | HandlerOutput diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/reverseModerationAction.ts b/packages/pds/src/lexicon/types/com/atproto/admin/reverseModerationAction.ts deleted file mode 100644 index 17dcb5085de..00000000000 --- a/packages/pds/src/lexicon/types/com/atproto/admin/reverseModerationAction.ts +++ /dev/null @@ -1,49 +0,0 @@ -/** - * GENERATED CODE - DO NOT MODIFY - */ -import express from 'express' -import { ValidationResult, BlobRef } from '@atproto/lexicon' -import { lexicons } from '../../../../lexicons' -import { isObj, hasProp } from '../../../../util' -import { CID } from 'multiformats/cid' -import { HandlerAuth } from '@atproto/xrpc-server' -import * as ComAtprotoAdminDefs from './defs' - -export interface QueryParams {} - -export interface InputSchema { - id: number - reason: string - createdBy: string - [k: string]: unknown -} - -export type OutputSchema = ComAtprotoAdminDefs.ActionView - -export interface HandlerInput { - encoding: 'application/json' - body: InputSchema -} - -export interface HandlerSuccess { - encoding: 'application/json' - body: OutputSchema - headers?: { [key: string]: string } -} - -export interface HandlerError { - status: number - message?: string -} - -export type HandlerOutput = HandlerError | HandlerSuccess -export type HandlerReqCtx = { - auth: HA - params: QueryParams - input: HandlerInput - req: express.Request - res: express.Response -} -export type Handler = ( - ctx: HandlerReqCtx, -) => Promise | HandlerOutput diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/sendEmail.ts b/packages/pds/src/lexicon/types/com/atproto/admin/sendEmail.ts index 87e7ceec172..91b53d9be81 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/sendEmail.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/sendEmail.ts @@ -14,6 +14,7 @@ export interface InputSchema { recipientDid: string content: string subject?: string + senderDid: string [k: string]: unknown } diff --git a/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap b/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap index 498fbe4a77f..15c63498ac1 100644 --- a/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap +++ b/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap @@ -4,7 +4,7 @@ exports[`proxies admin requests creates reports of a repo. 1`] = ` Array [ Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, + "id": 2, "reasonType": "com.atproto.moderation.defs#reasonSpam", "reportedBy": "user(0)", "subject": Object { @@ -14,7 +14,7 @@ Array [ }, Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 2, + "id": 3, "reason": "impersonation", "reasonType": "com.atproto.moderation.defs#reasonOther", "reportedBy": "user(2)", @@ -26,133 +26,90 @@ Array [ ] `; -exports[`proxies admin requests fetches a list of actions. 1`] = ` -Object { - "actions": Array [ - Object { - "action": "com.atproto.admin.defs#acknowledge", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 3, - "reason": "Y", - "resolvedReportIds": Array [], - "reversal": Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "reason": "X", - }, - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - "subjectBlobCids": Array [], +exports[`proxies admin requests fetches a list of events. 1`] = ` +Array [ + Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "createdBy": "did:example:admin", + "event": Object { + "$type": "com.atproto.admin.defs#modEventAcknowledge", }, - Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 2, - "reason": "Y", - "resolvedReportIds": Array [ - 2, - 1, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectBlobCids": Array [], + "id": 5, + "subject": Object { + "$type": "com.atproto.admin.defs#repoRef", + "did": "user(0)", }, - ], - "cursor": "2", -} -`; - -exports[`proxies admin requests fetches a list of reports. 1`] = ` -Object { - "cursor": "2", - "reports": Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(0)", - "resolvedByActionIds": Array [ - 2, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(1)", - }, - "subjectRepoHandle": "bob.test", + "subjectBlobCids": Array [], + "subjectHandle": "bob.test", + }, + Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "createdBy": "user(1)", + "creatorHandle": "carol.test", + "event": Object { + "$type": "com.atproto.admin.defs#modEventReport", + "comment": "impersonation", + "reportType": "com.atproto.moderation.defs#reasonOther", }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 2, - "reason": "impersonation", - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(2)", - "resolvedByActionIds": Array [ - 2, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(1)", - }, - "subjectRepoHandle": "bob.test", + "id": 3, + "subject": Object { + "$type": "com.atproto.admin.defs#repoRef", + "did": "user(0)", }, - ], -} + "subjectBlobCids": Array [], + "subjectHandle": "bob.test", + }, + Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "createdBy": "user(2)", + "creatorHandle": "alice.test", + "event": Object { + "$type": "com.atproto.admin.defs#modEventReport", + "reportType": "com.atproto.moderation.defs#reasonSpam", + }, + "id": 2, + "subject": Object { + "$type": "com.atproto.admin.defs#repoRef", + "did": "user(0)", + }, + "subjectBlobCids": Array [], + "subjectHandle": "bob.test", + }, +] `; -exports[`proxies admin requests fetches action details. 1`] = ` +exports[`proxies admin requests fetches event details. 1`] = ` Object { - "action": "com.atproto.admin.defs#acknowledge", "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 3, - "reason": "Y", - "resolvedReports": Array [], - "reversal": Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "reason": "X", + "createdBy": "user(1)", + "event": Object { + "$type": "com.atproto.admin.defs#modEventReport", + "reportType": "com.atproto.moderation.defs#reasonSpam", }, + "id": 2, "subject": Object { "$type": "com.atproto.admin.defs#repoView", "did": "user(0)", - "email": "bob@test.com", "handle": "bob.test", "indexedAt": "1970-01-01T00:00:00.000Z", - "invitedBy": Object { - "available": 10, - "code": "invite-code", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "admin", - "disabled": false, - "forAccount": "admin", - "uses": Array [ - Object { - "usedAt": "1970-01-01T00:00:00.000Z", - "usedBy": "user(1)", - }, - Object { - "usedAt": "1970-01-01T00:00:00.000Z", - "usedBy": "user(0)", - }, - Object { - "usedAt": "1970-01-01T00:00:00.000Z", - "usedBy": "user(2)", - }, - Object { - "usedAt": "1970-01-01T00:00:00.000Z", - "usedBy": "user(3)", + "moderation": Object { + "subjectStatus": Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "id": 1, + "lastReportedAt": "1970-01-01T00:00:00.000Z", + "lastReviewedAt": "1970-01-01T00:00:00.000Z", + "lastReviewedBy": "did:example:admin", + "reviewState": "com.atproto.admin.defs#reviewClosed", + "subject": Object { + "$type": "com.atproto.admin.defs#repoRef", + "did": "user(0)", }, - ], + "subjectBlobCids": Array [], + "subjectRepoHandle": "bob.test", + "takendown": false, + "updatedAt": "1970-01-01T00:00:00.000Z", + }, }, - "invitesDisabled": true, - "moderation": Object {}, "relatedRecords": Array [ Object { "$type": "app.bsky.actor.profile", @@ -169,10 +126,31 @@ Object { }, ], }, + "subjectBlobCids": Array [], "subjectBlobs": Array [], } `; +exports[`proxies admin requests fetches moderation events. 1`] = ` +Array [ + Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "createdBy": "did:example:admin", + "event": Object { + "$type": "com.atproto.admin.defs#modEventAcknowledge", + }, + "id": 4, + "subject": Object { + "$type": "com.atproto.repo.strongRef", + "cid": "cids(0)", + "uri": "record(0)", + }, + "subjectBlobCids": Array [], + "subjectHandle": "bob.test", + }, +] +`; + exports[`proxies admin requests fetches record details. 1`] = ` Object { "blobCids": Array [], @@ -181,30 +159,22 @@ Object { "indexedAt": "1970-01-01T00:00:00.000Z", "labels": Array [], "moderation": Object { - "actions": Array [ - Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 2, - "reason": "Y", - "resolvedReportIds": Array [ - 2, - 1, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectBlobCids": Array [], + "subjectStatus": Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "id": 3, + "lastReviewedAt": "1970-01-01T00:00:00.000Z", + "lastReviewedBy": "did:example:admin", + "reviewState": "com.atproto.admin.defs#reviewClosed", + "subject": Object { + "$type": "com.atproto.repo.strongRef", + "cid": "cids(0)", + "uri": "record(0)", }, - ], - "currentAction": Object { - "action": "com.atproto.admin.defs#flag", - "id": 2, + "subjectBlobCids": Array [], + "subjectRepoHandle": "bob.test", + "takendown": false, + "updatedAt": "1970-01-01T00:00:00.000Z", }, - "reports": Array [], }, "repo": Object { "did": "user(0)", @@ -239,9 +209,21 @@ Object { }, "invitesDisabled": true, "moderation": Object { - "currentAction": Object { - "action": "com.atproto.admin.defs#acknowledge", - "id": 3, + "subjectStatus": Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "id": 1, + "lastReportedAt": "1970-01-01T00:00:00.000Z", + "lastReviewedAt": "1970-01-01T00:00:00.000Z", + "lastReviewedBy": "did:example:admin", + "reviewState": "com.atproto.admin.defs#reviewClosed", + "subject": Object { + "$type": "com.atproto.admin.defs#repoRef", + "did": "user(0)", + }, + "subjectBlobCids": Array [], + "subjectRepoHandle": "bob.test", + "takendown": false, + "updatedAt": "1970-01-01T00:00:00.000Z", }, }, "relatedRecords": Array [ @@ -292,118 +274,11 @@ Object { "invites": Array [], "invitesDisabled": false, "labels": Array [], - "moderation": Object { - "actions": Array [], - "reports": Array [], - }, + "moderation": Object {}, "relatedRecords": Array [], } `; -exports[`proxies admin requests fetches report details. 1`] = ` -Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(0)", - "resolvedByActions": Array [ - Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 2, - "reason": "Y", - "resolvedReportIds": Array [ - 2, - 1, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(1)", - "uri": "record(0)", - }, - "subjectBlobCids": Array [], - }, - ], - "subject": Object { - "$type": "com.atproto.admin.defs#repoView", - "did": "user(1)", - "email": "bob@test.com", - "handle": "bob.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitedBy": Object { - "available": 10, - "code": "invite-code", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "admin", - "disabled": false, - "forAccount": "admin", - "uses": Array [ - Object { - "usedAt": "1970-01-01T00:00:00.000Z", - "usedBy": "user(0)", - }, - Object { - "usedAt": "1970-01-01T00:00:00.000Z", - "usedBy": "user(1)", - }, - Object { - "usedAt": "1970-01-01T00:00:00.000Z", - "usedBy": "user(2)", - }, - Object { - "usedAt": "1970-01-01T00:00:00.000Z", - "usedBy": "user(3)", - }, - ], - }, - "invitesDisabled": true, - "moderation": Object { - "currentAction": Object { - "action": "com.atproto.admin.defs#acknowledge", - "id": 3, - }, - }, - "relatedRecords": Array [ - Object { - "$type": "app.bsky.actor.profile", - "avatar": Object { - "$type": "blob", - "mimeType": "image/jpeg", - "ref": Object { - "$link": "cids(0)", - }, - "size": 3976, - }, - "description": "hi im bob label_me", - "displayName": "bobby", - }, - ], - }, -} -`; - -exports[`proxies admin requests reverses action. 1`] = ` -Object { - "action": "com.atproto.admin.defs#acknowledge", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 3, - "reason": "Y", - "resolvedReportIds": Array [], - "reversal": Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "reason": "X", - }, - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - "subjectBlobCids": Array [], -} -`; - exports[`proxies admin requests searches repos. 1`] = ` Array [ Object { @@ -443,12 +318,12 @@ Array [ exports[`proxies admin requests takes actions and resolves reports 1`] = ` Object { - "action": "com.atproto.admin.defs#flag", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 2, - "reason": "Y", - "resolvedReportIds": Array [], + "event": Object { + "$type": "com.atproto.admin.defs#modEventAcknowledge", + }, + "id": 4, "subject": Object { "$type": "com.atproto.repo.strongRef", "cid": "cids(0)", @@ -460,12 +335,12 @@ Object { exports[`proxies admin requests takes actions and resolves reports 2`] = ` Object { - "action": "com.atproto.admin.defs#acknowledge", "createdAt": "1970-01-01T00:00:00.000Z", "createdBy": "did:example:admin", - "id": 3, - "reason": "Y", - "resolvedReportIds": Array [], + "event": Object { + "$type": "com.atproto.admin.defs#modEventAcknowledge", + }, + "id": 5, "subject": Object { "$type": "com.atproto.admin.defs#repoRef", "did": "user(0)", @@ -473,23 +348,3 @@ Object { "subjectBlobCids": Array [], } `; - -exports[`proxies admin requests takes actions and resolves reports 3`] = ` -Object { - "action": "com.atproto.admin.defs#flag", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 2, - "reason": "Y", - "resolvedReportIds": Array [ - 2, - 1, - ], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectBlobCids": Array [], -} -`; diff --git a/packages/pds/tests/proxied/admin.test.ts b/packages/pds/tests/proxied/admin.test.ts index 8b4fffae9e1..a51ec048c2d 100644 --- a/packages/pds/tests/proxied/admin.test.ts +++ b/packages/pds/tests/proxied/admin.test.ts @@ -6,11 +6,6 @@ import { REASONSPAM, } from '@atproto/api/src/client/types/com/atproto/moderation/defs' import { forSnapshot } from '../_util' -import { - ACKNOWLEDGE, - FLAG, - TAKEDOWN, -} from '@atproto/api/src/client/types/com/atproto/admin/defs' import { NotFoundError } from '@atproto/api/src/client/types/app/bsky/feed/getPostThread' describe('proxies admin requests', () => { @@ -106,9 +101,9 @@ describe('proxies admin requests', () => { it('takes actions and resolves reports', async () => { const post = sc.posts[sc.dids.bob][1] const { data: actionA } = - await agent.api.com.atproto.admin.takeModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - action: FLAG, + event: { $type: 'com.atproto.admin.defs#modEventAcknowledge' }, subject: { $type: 'com.atproto.repo.strongRef', uri: post.ref.uriStr, @@ -124,9 +119,9 @@ describe('proxies admin requests', () => { ) expect(forSnapshot(actionA)).toMatchSnapshot() const { data: actionB } = - await agent.api.com.atproto.admin.takeModerationAction( + await agent.api.com.atproto.admin.emitModerationEvent( { - action: ACKNOWLEDGE, + event: { $type: 'com.atproto.admin.defs#modEventAcknowledge' }, subject: { $type: 'com.atproto.admin.defs#repoRef', did: sc.dids.bob, @@ -140,39 +135,18 @@ describe('proxies admin requests', () => { }, ) expect(forSnapshot(actionB)).toMatchSnapshot() - const { data: resolved } = - await agent.api.com.atproto.admin.resolveModerationReports( - { - actionId: actionA.id, - reportIds: [1, 2], - createdBy: 'did:example:admin', - }, - { - headers: network.pds.adminAuthHeaders(), - encoding: 'application/json', - }, - ) - expect(forSnapshot(resolved)).toMatchSnapshot() }) - it('fetches report details.', async () => { + it('fetches moderation events.', async () => { const { data: result } = - await agent.api.com.atproto.admin.getModerationReport( - { id: 1 }, - { headers: network.pds.adminAuthHeaders() }, - ) - expect(forSnapshot(result)).toMatchSnapshot() - }) - - it('fetches a list of reports.', async () => { - const { data: result } = - await agent.api.com.atproto.admin.getModerationReports( - { reverse: true }, + await agent.api.com.atproto.admin.queryModerationEvents( + { + subject: sc.posts[sc.dids.bob][1].ref.uriStr, + }, { headers: network.pds.adminAuthHeaders() }, ) - expect(forSnapshot(result)).toMatchSnapshot() + expect(forSnapshot(result.events)).toMatchSnapshot() }) - it('fetches repo details.', async () => { const { data: result } = await agent.api.com.atproto.admin.getRepo( { did: sc.dids.eve }, @@ -190,34 +164,22 @@ describe('proxies admin requests', () => { expect(forSnapshot(result)).toMatchSnapshot() }) - it('reverses action.', async () => { + it('fetches event details.', async () => { const { data: result } = - await agent.api.com.atproto.admin.reverseModerationAction( - { id: 3, createdBy: 'did:example:admin', reason: 'X' }, - { - headers: network.pds.adminAuthHeaders(), - encoding: 'application/json', - }, - ) - expect(forSnapshot(result)).toMatchSnapshot() - }) - - it('fetches action details.', async () => { - const { data: result } = - await agent.api.com.atproto.admin.getModerationAction( - { id: 3 }, + await agent.api.com.atproto.admin.getModerationEvent( + { id: 2 }, { headers: network.pds.adminAuthHeaders() }, ) expect(forSnapshot(result)).toMatchSnapshot() }) - it('fetches a list of actions.', async () => { + it('fetches a list of events.', async () => { const { data: result } = - await agent.api.com.atproto.admin.getModerationActions( + await agent.api.com.atproto.admin.queryModerationEvents( { subject: sc.dids.bob }, { headers: network.pds.adminAuthHeaders() }, ) - expect(forSnapshot(result)).toMatchSnapshot() + expect(forSnapshot(result.events)).toMatchSnapshot() }) it('searches repos.', async () => { @@ -229,11 +191,6 @@ describe('proxies admin requests', () => { }) it('passes through errors.', async () => { - const tryGetReport = agent.api.com.atproto.admin.getModerationReport( - { id: 1000 }, - { headers: network.pds.adminAuthHeaders() }, - ) - await expect(tryGetReport).rejects.toThrow('Report not found') const tryGetRepo = agent.api.com.atproto.admin.getRepo( { did: 'did:does:not:exist' }, { headers: network.pds.adminAuthHeaders() }, @@ -248,24 +205,23 @@ describe('proxies admin requests', () => { it('takesdown and labels repos, and reverts.', async () => { // takedown repo - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.alice, - }, - createdBy: 'did:example:admin', - reason: 'Y', - createLabelVals: ['dogs'], - negateLabelVals: ['cats'], - }, - { - headers: network.pds.adminAuthHeaders(), - encoding: 'application/json', + await agent.api.com.atproto.admin.emitModerationEvent( + { + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.alice, }, - ) + createdBy: 'did:example:admin', + reason: 'Y', + createLabelVals: ['dogs'], + negateLabelVals: ['cats'], + }, + { + headers: network.pds.adminAuthHeaders(), + encoding: 'application/json', + }, + ) // check profile and labels const tryGetProfileAppview = agent.api.app.bsky.actor.getProfile( { actor: sc.dids.alice }, @@ -277,8 +233,18 @@ describe('proxies admin requests', () => { 'Account has been taken down', ) // reverse action - await agent.api.com.atproto.admin.reverseModerationAction( - { id: action.id, createdBy: 'did:example:admin', reason: 'X' }, + await agent.api.com.atproto.admin.emitModerationEvent( + { + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.alice, + }, + event: { + $type: 'com.atproto.admin.defs#modEventReverseTakedown', + }, + createdBy: 'did:example:admin', + reason: 'X', + }, { headers: network.pds.adminAuthHeaders(), encoding: 'application/json', @@ -299,25 +265,24 @@ describe('proxies admin requests', () => { it('takesdown and labels records, and reverts.', async () => { const post = sc.posts[sc.dids.alice][0] // takedown post - const { data: action } = - await agent.api.com.atproto.admin.takeModerationAction( - { - action: TAKEDOWN, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: post.ref.uriStr, - cid: post.ref.cidStr, - }, - createdBy: 'did:example:admin', - reason: 'Y', - createLabelVals: ['dogs'], - negateLabelVals: ['cats'], - }, - { - headers: network.pds.adminAuthHeaders(), - encoding: 'application/json', + await agent.api.com.atproto.admin.emitModerationEvent( + { + event: { $type: 'com.atproto.admin.defs#modEventTakedown' }, + subject: { + $type: 'com.atproto.repo.strongRef', + uri: post.ref.uriStr, + cid: post.ref.cidStr, }, - ) + createdBy: 'did:example:admin', + reason: 'Y', + createLabelVals: ['dogs'], + negateLabelVals: ['cats'], + }, + { + headers: network.pds.adminAuthHeaders(), + encoding: 'application/json', + }, + ) // check thread and labels const tryGetPost = agent.api.app.bsky.feed.getPostThread( { uri: post.ref.uriStr, depth: 0 }, @@ -325,8 +290,17 @@ describe('proxies admin requests', () => { ) await expect(tryGetPost).rejects.toThrow(NotFoundError) // reverse action - await agent.api.com.atproto.admin.reverseModerationAction( - { id: action.id, createdBy: 'did:example:admin', reason: 'X' }, + await agent.api.com.atproto.admin.emitModerationEvent( + { + subject: { + $type: 'com.atproto.repo.strongRef', + uri: post.ref.uriStr, + cid: post.ref.cidStr, + }, + event: { $type: 'com.atproto.admin.defs#modEventReverseTakedown' }, + createdBy: 'did:example:admin', + reason: 'X', + }, { headers: network.pds.adminAuthHeaders(), encoding: 'application/json', diff --git a/packages/pds/tests/seeds/basic.ts b/packages/pds/tests/seeds/basic.ts index 1f71b58ff63..bce8c1b3b92 100644 --- a/packages/pds/tests/seeds/basic.ts +++ b/packages/pds/tests/seeds/basic.ts @@ -1,6 +1,5 @@ import { SeedClient } from '@atproto/dev-env' import { ids } from '../../src/lexicon/lexicons' -import { FLAG } from '../../src/lexicon/types/com/atproto/admin/defs' import usersSeed from './users' export default async ( @@ -132,16 +131,18 @@ export default async ( await sc.repost(dan, alicesReplyToBob.ref) if (opts?.addModLabels) { - await sc.agent.com.atproto.admin.takeModerationAction( + await sc.agent.com.atproto.admin.emitModerationEvent( { - action: FLAG, + event: { + createLabelVals: ['repo-action-label'], + negateLabelVals: [], + $type: 'com.atproto.admin.defs#modEventLabel', + }, subject: { $type: 'com.atproto.admin.defs#repoRef', did: dan, }, createdBy: 'did:example:admin', - reason: 'test', - createLabelVals: ['repo-action-label'], }, { encoding: 'application/json', diff --git a/services/bsky/api.js b/services/bsky/api.js index fac5b0a7c8b..cf63c951043 100644 --- a/services/bsky/api.js +++ b/services/bsky/api.js @@ -26,7 +26,7 @@ const { BskyAppView, ViewMaintainer, makeAlgos, - PeriodicModerationActionReversal, + PeriodicModerationEventReversal, } = require('@atproto/bsky') const main = async () => { @@ -110,18 +110,18 @@ const main = async () => { const viewMaintainer = new ViewMaintainer(migrateDb, 1800) const viewMaintainerRunning = viewMaintainer.run() - const periodicModerationActionReversal = new PeriodicModerationActionReversal( + const periodicModerationEventReversal = new PeriodicModerationEventReversal( bsky.ctx, ) - const periodicModerationActionReversalRunning = - periodicModerationActionReversal.run() + const periodicModerationEventReversalRunning = + periodicModerationEventReversal.run() await bsky.start() // Graceful shutdown (see also https://aws.amazon.com/blogs/containers/graceful-shutdowns-with-ecs/) process.on('SIGTERM', async () => { - // Gracefully shutdown periodic-moderation-action-reversal before destroying bsky instance - periodicModerationActionReversal.destroy() - await periodicModerationActionReversalRunning + // Gracefully shutdown periodic-moderation-event-reversal before destroying bsky instance + periodicModerationEventReversal.destroy() + await periodicModerationEventReversalRunning await bsky.destroy() viewMaintainer.destroy() await viewMaintainerRunning From 4a5b65893d80cfceecfc8704bf8d5affe7d32c12 Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Thu, 30 Nov 2023 20:00:39 +0100 Subject: [PATCH 42/59] :sparkles: Expose labels attached with legacy actions when events are queried and fix email event builder (#1905) * :sparkles: Expose labels attached with legacy actions when events are queried * :bug: Fix property name for modEventEmail * :white_check_mark: Only attach label vals for legacy actions if they exist * :bug: Set empty string as default subjectLine for email event * :broom: Cleaning up --- .../bsky/src/services/moderation/views.ts | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/packages/bsky/src/services/moderation/views.ts b/packages/bsky/src/services/moderation/views.ts index 418253ba649..2dc9c5ec7e4 100644 --- a/packages/bsky/src/services/moderation/views.ts +++ b/packages/bsky/src/services/moderation/views.ts @@ -145,6 +145,29 @@ export class ModerationViews { } } + // This is for legacy data only, for new events, these types of events won't have labels attached + if ( + [ + 'com.atproto.admin.defs#modEventAcknowledge', + 'com.atproto.admin.defs#modEventTakedown', + 'com.atproto.admin.defs#modEventEscalate', + ].includes(res.action) + ) { + if (res.createLabelVals?.length) { + eventView.event = { + ...eventView.event, + createLabelVals: res.createLabelVals.split(' '), + } + } + + if (res.negateLabelVals?.length) { + eventView.event = { + ...eventView.event, + negateLabelVals: res.negateLabelVals.split(' '), + } + } + } + if (res.action === 'com.atproto.admin.defs#modEventReport') { eventView.event = { ...eventView.event, @@ -155,7 +178,7 @@ export class ModerationViews { if (res.action === 'com.atproto.admin.defs#modEventEmail') { eventView.event = { ...eventView.event, - subject: res.meta?.subject ?? undefined, + subjectLine: res.meta?.subjectLine ?? '', } } From 401538a93371beb8fc0590df4a9c1b2800048375 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Thu, 30 Nov 2023 15:42:00 -0600 Subject: [PATCH 43/59] Handle missing creator on lists and feed generators (#1906) handle missing creator on lists and feed generators --- .../src/api/app/bsky/feed/getActorFeeds.ts | 3 ++- .../src/api/app/bsky/feed/getFeedGenerator.ts | 3 +++ .../api/app/bsky/feed/getFeedGenerators.ts | 3 ++- .../api/app/bsky/feed/getSuggestedFeeds.ts | 3 ++- .../bsky/src/api/app/bsky/graph/getList.ts | 3 +++ .../src/api/app/bsky/graph/getListBlocks.ts | 3 ++- .../src/api/app/bsky/graph/getListMutes.ts | 3 ++- .../bsky/src/api/app/bsky/graph/getLists.ts | 3 ++- .../unspecced/getPopularFeedGenerators.ts | 4 +++- packages/bsky/src/services/feed/index.ts | 24 ++++++++++++++----- packages/bsky/src/services/feed/views.ts | 5 +++- packages/bsky/src/services/graph/index.ts | 12 ++++++++-- 12 files changed, 53 insertions(+), 16 deletions(-) diff --git a/packages/bsky/src/api/app/bsky/feed/getActorFeeds.ts b/packages/bsky/src/api/app/bsky/feed/getActorFeeds.ts index deb5c3a5a1b..7a28e4efe67 100644 --- a/packages/bsky/src/api/app/bsky/feed/getActorFeeds.ts +++ b/packages/bsky/src/api/app/bsky/feed/getActorFeeds.ts @@ -1,4 +1,5 @@ import { InvalidRequestError } from '@atproto/xrpc-server' +import { mapDefined } from '@atproto/common' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { TimeCidKeyset, paginate } from '../../../../db/pagination' @@ -42,7 +43,7 @@ export default function (server: Server, ctx: AppContext) { throw new InvalidRequestError(`Actor not found: ${actor}`) } - const feeds = feedsRes.map((row) => { + const feeds = mapDefined(feedsRes, (row) => { const feed = { ...row, viewer: viewer ? { like: row.viewerLike } : undefined, diff --git a/packages/bsky/src/api/app/bsky/feed/getFeedGenerator.ts b/packages/bsky/src/api/app/bsky/feed/getFeedGenerator.ts index 6207ba1e1aa..14a5688db0d 100644 --- a/packages/bsky/src/api/app/bsky/feed/getFeedGenerator.ts +++ b/packages/bsky/src/api/app/bsky/feed/getFeedGenerator.ts @@ -55,6 +55,9 @@ export default function (server: Server, ctx: AppContext) { feedInfo, profiles, ) + if (!feedView) { + throw new InvalidRequestError('could not find feed') + } return { encoding: 'application/json', diff --git a/packages/bsky/src/api/app/bsky/feed/getFeedGenerators.ts b/packages/bsky/src/api/app/bsky/feed/getFeedGenerators.ts index a973ee6c2fb..7b571ab09f6 100644 --- a/packages/bsky/src/api/app/bsky/feed/getFeedGenerators.ts +++ b/packages/bsky/src/api/app/bsky/feed/getFeedGenerators.ts @@ -1,3 +1,4 @@ +import { mapDefined } from '@atproto/common' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { FeedGenInfo, FeedService } from '../../../../services/feed' @@ -60,7 +61,7 @@ const hydration = async (state: SkeletonState, ctx: Context) => { const presentation = (state: HydrationState, ctx: Context) => { const { feedService } = ctx - const feeds = state.generators.map((gen) => + const feeds = mapDefined(state.generators, (gen) => feedService.views.formatFeedGeneratorView(gen, state.profiles), ) return { feeds } diff --git a/packages/bsky/src/api/app/bsky/feed/getSuggestedFeeds.ts b/packages/bsky/src/api/app/bsky/feed/getSuggestedFeeds.ts index 1ad65cdf756..35fac829039 100644 --- a/packages/bsky/src/api/app/bsky/feed/getSuggestedFeeds.ts +++ b/packages/bsky/src/api/app/bsky/feed/getSuggestedFeeds.ts @@ -1,3 +1,4 @@ +import { mapDefined } from '@atproto/common' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' @@ -23,7 +24,7 @@ export default function (server: Server, ctx: AppContext) { const creators = genList.map((gen) => gen.creator) const profiles = await actorService.views.profilesBasic(creators, viewer) - const feedViews = genList.map((gen) => + const feedViews = mapDefined(genList, (gen) => feedService.views.formatFeedGeneratorView(gen, profiles), ) diff --git a/packages/bsky/src/api/app/bsky/graph/getList.ts b/packages/bsky/src/api/app/bsky/graph/getList.ts index 82963183a74..3c95357d005 100644 --- a/packages/bsky/src/api/app/bsky/graph/getList.ts +++ b/packages/bsky/src/api/app/bsky/graph/getList.ts @@ -98,6 +98,9 @@ const presentation = (state: HydrationState, ctx: Context) => { throw new InvalidRequestError(`Actor not found: ${list.handle}`) } const listView = graphService.formatListView(list, actors) + if (!listView) { + throw new InvalidRequestError('List not found') + } const items = mapDefined(listItems, (item) => { const subject = actors[item.did] if (!subject) return diff --git a/packages/bsky/src/api/app/bsky/graph/getListBlocks.ts b/packages/bsky/src/api/app/bsky/graph/getListBlocks.ts index a41c952508b..03fd3496f97 100644 --- a/packages/bsky/src/api/app/bsky/graph/getListBlocks.ts +++ b/packages/bsky/src/api/app/bsky/graph/getListBlocks.ts @@ -1,3 +1,4 @@ +import { mapDefined } from '@atproto/common' import { Server } from '../../../../lexicon' import { QueryParams } from '../../../../lexicon/types/app/bsky/graph/getListBlocks' import { paginate, TimeCidKeyset } from '../../../../db/pagination' @@ -89,7 +90,7 @@ const presentation = (state: HydrationState, ctx: Context) => { profileState, params.viewer, ) - const lists = listInfos.map((list) => + const lists = mapDefined(listInfos, (list) => graphService.formatListView(list, actors), ) return { lists, cursor } diff --git a/packages/bsky/src/api/app/bsky/graph/getListMutes.ts b/packages/bsky/src/api/app/bsky/graph/getListMutes.ts index 49d04b3233f..ab0ac77f47c 100644 --- a/packages/bsky/src/api/app/bsky/graph/getListMutes.ts +++ b/packages/bsky/src/api/app/bsky/graph/getListMutes.ts @@ -1,3 +1,4 @@ +import { mapDefined } from '@atproto/common' import { Server } from '../../../../lexicon' import { paginate, TimeCidKeyset } from '../../../../db/pagination' import AppContext from '../../../../context' @@ -34,7 +35,7 @@ export default function (server: Server, ctx: AppContext) { const actorService = ctx.services.actor(db) const profiles = await actorService.views.profiles(listsRes, requester) - const lists = listsRes.map((row) => + const lists = mapDefined(listsRes, (row) => graphService.formatListView(row, profiles), ) diff --git a/packages/bsky/src/api/app/bsky/graph/getLists.ts b/packages/bsky/src/api/app/bsky/graph/getLists.ts index e6ca61fa9c7..73deb51900b 100644 --- a/packages/bsky/src/api/app/bsky/graph/getLists.ts +++ b/packages/bsky/src/api/app/bsky/graph/getLists.ts @@ -1,3 +1,4 @@ +import { mapDefined } from '@atproto/common' import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import { paginate, TimeCidKeyset } from '../../../../db/pagination' @@ -39,7 +40,7 @@ export default function (server: Server, ctx: AppContext) { throw new InvalidRequestError(`Actor not found: ${actor}`) } - const lists = listsRes.map((row) => + const lists = mapDefined(listsRes, (row) => graphService.formatListView(row, profiles), ) diff --git a/packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts b/packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts index 2971beba381..e135d2cb7c1 100644 --- a/packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts +++ b/packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts @@ -57,7 +57,9 @@ export default function (server: Server, ctx: AppContext) { const gen = genInfos[row.uri] if (!gen) continue const view = feedService.views.formatFeedGeneratorView(gen, profiles) - genViews.push(view) + if (view) { + genViews.push(view) + } } return { diff --git a/packages/bsky/src/services/feed/index.ts b/packages/bsky/src/services/feed/index.ts index f7fe7b2d817..2323e6a74be 100644 --- a/packages/bsky/src/services/feed/index.ts +++ b/packages/bsky/src/services/feed/index.ts @@ -411,14 +411,26 @@ export class FeedService { for (const uri of nestedUris) { const collection = new AtUri(uri).collection if (collection === ids.AppBskyFeedGenerator && feedGenInfos[uri]) { - recordEmbedViews[uri] = { - $type: 'app.bsky.feed.defs#generatorView', - ...this.views.formatFeedGeneratorView(feedGenInfos[uri], actorInfos), + const genView = this.views.formatFeedGeneratorView( + feedGenInfos[uri], + actorInfos, + ) + if (genView) { + recordEmbedViews[uri] = { + $type: 'app.bsky.feed.defs#generatorView', + ...genView, + } } } else if (collection === ids.AppBskyGraphList && listViews[uri]) { - recordEmbedViews[uri] = { - $type: 'app.bsky.graph.defs#listView', - ...this.services.graph.formatListView(listViews[uri], actorInfos), + const listView = this.services.graph.formatListView( + listViews[uri], + actorInfos, + ) + if (listView) { + recordEmbedViews[uri] = { + $type: 'app.bsky.graph.defs#listView', + ...listView, + } } } else if (collection === ids.AppBskyFeedPost && feedState.posts[uri]) { const formatted = this.views.formatPostView( diff --git a/packages/bsky/src/services/feed/views.ts b/packages/bsky/src/services/feed/views.ts index 19dada6dfb7..a848c88caa0 100644 --- a/packages/bsky/src/services/feed/views.ts +++ b/packages/bsky/src/services/feed/views.ts @@ -62,8 +62,11 @@ export class FeedViews { formatFeedGeneratorView( info: FeedGenInfo, profiles: ActorInfoMap, - ): GeneratorView { + ): GeneratorView | undefined { const profile = profiles[info.creator] + if (!profile) { + return undefined + } return { uri: info.uri, cid: info.cid, diff --git a/packages/bsky/src/services/graph/index.ts b/packages/bsky/src/services/graph/index.ts index 4d3a117b0f4..190e3ac0661 100644 --- a/packages/bsky/src/services/graph/index.ts +++ b/packages/bsky/src/services/graph/index.ts @@ -4,6 +4,10 @@ import { ImageUriBuilder } from '../../image/uri' import { valuesList } from '../../db/util' import { ListInfo } from './types' import { ActorInfoMap } from '../actor' +import { + ListView, + ListViewBasic, +} from '../../lexicon/types/app/bsky/graph/defs' export * from './types' @@ -235,7 +239,10 @@ export class GraphService { ) } - formatListView(list: ListInfo, profiles: ActorInfoMap) { + formatListView(list: ListInfo, profiles: ActorInfoMap): ListView | undefined { + if (!profiles[list.creator]) { + return undefined + } return { ...this.formatListViewBasic(list), creator: profiles[list.creator], @@ -243,10 +250,11 @@ export class GraphService { descriptionFacets: list.descriptionFacets ? JSON.parse(list.descriptionFacets) : undefined, + indexedAt: list.sortAt, } } - formatListViewBasic(list: ListInfo) { + formatListViewBasic(list: ListInfo): ListViewBasic { return { uri: list.uri, cid: list.cid, From 6da8182530c4b3727476fc22d3e9c249801ff54e Mon Sep 17 00:00:00 2001 From: Mary <148872143+mary-ext@users.noreply.github.com> Date: Fri, 1 Dec 2023 06:51:07 +0700 Subject: [PATCH 44/59] fix(pds): include aspectRatio on read-sticky posts (#1824) * fix: include aspectRatio on read-sticky * chore: add missing alt text test case --- packages/pds/src/services/local/index.ts | 1 + packages/pds/tests/proxied/read-after-write.test.ts | 3 +++ 2 files changed, 4 insertions(+) diff --git a/packages/pds/src/services/local/index.ts b/packages/pds/src/services/local/index.ts index 83ef39798e8..0dfc2ca9355 100644 --- a/packages/pds/src/services/local/index.ts +++ b/packages/pds/src/services/local/index.ts @@ -220,6 +220,7 @@ export class LocalService { did, img.image.ref.toString(), ), + aspectRatio: img.aspectRatio, alt: img.alt, })) return { diff --git a/packages/pds/tests/proxied/read-after-write.test.ts b/packages/pds/tests/proxied/read-after-write.test.ts index 1e9e4125084..52507a2730a 100644 --- a/packages/pds/tests/proxied/read-after-write.test.ts +++ b/packages/pds/tests/proxied/read-after-write.test.ts @@ -141,6 +141,7 @@ describe('proxy read after write', () => { images: [ { image: img.image, + aspectRatio: { height: 2, width: 1 }, alt: 'alt text', }, ], @@ -190,6 +191,8 @@ describe('proxy read after write', () => { img.image.ref.toString(), ), ) + expect(imgs.images[0].aspectRatio).toEqual({ height: 2, width: 1 }) + expect(imgs.images[0].alt).toBe('alt text') expect(replies[0].replies?.length).toBe(1) // @ts-ignore expect(replies[0].replies[0].post.uri).toEqual(replyRes2.uri) From 3c0ef382c12a413cc971ae47ffb341236c545f60 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 30 Nov 2023 23:53:04 +0000 Subject: [PATCH 45/59] Don't create unnecessary error objects (#1908) * Don't create unnecessary error objects * add changeset --------- Co-authored-by: dholms --- .changeset/brave-swans-kiss.md | 5 +++++ packages/syntax/src/aturi_validation.ts | 10 +++++----- 2 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 .changeset/brave-swans-kiss.md diff --git a/.changeset/brave-swans-kiss.md b/.changeset/brave-swans-kiss.md new file mode 100644 index 00000000000..e6e42a1bcb2 --- /dev/null +++ b/.changeset/brave-swans-kiss.md @@ -0,0 +1,5 @@ +--- +'@atproto/syntax': patch +--- + +prevent unnecessary throw/catch on uri syntax diff --git a/packages/syntax/src/aturi_validation.ts b/packages/syntax/src/aturi_validation.ts index a272b15a082..826dfa78b2a 100644 --- a/packages/syntax/src/aturi_validation.ts +++ b/packages/syntax/src/aturi_validation.ts @@ -38,13 +38,13 @@ export const ensureValidAtUri = (uri: string) => { } try { - ensureValidHandle(parts[2]) - } catch { - try { + if (parts[2].startsWith('did:')) { ensureValidDid(parts[2]) - } catch { - throw new Error('ATURI authority must be a valid handle or DID') + } else { + ensureValidHandle(parts[2]) } + } catch { + throw new Error('ATURI authority must be a valid handle or DID') } if (parts.length >= 4) { From 2fc6ca54c10c970ea813534e5eb0352c62a2572f Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Fri, 1 Dec 2023 00:53:16 +0100 Subject: [PATCH 46/59] perf(bsky): avoid re-creating auth functions on every request (#1822) perf(bsky): avoid re-creating auth utilities on every request Co-authored-by: Daniel Holmgren --- packages/bsky/src/auth.ts | 42 +++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/packages/bsky/src/auth.ts b/packages/bsky/src/auth.ts index 220be08fc32..ba58638d4f9 100644 --- a/packages/bsky/src/auth.ts +++ b/packages/bsky/src/auth.ts @@ -7,37 +7,41 @@ import { ServerConfig } from './config' const BASIC = 'Basic ' const BEARER = 'Bearer ' -export const authVerifier = - (idResolver: IdResolver, opts: { aud: string | null }) => - async (reqCtx: { req: express.Request; res: express.Response }) => { +export const authVerifier = ( + idResolver: IdResolver, + opts: { aud: string | null }, +) => { + const getSigningKey = async ( + did: string, + forceRefresh: boolean, + ): Promise => { + const atprotoData = await idResolver.did.resolveAtprotoData( + did, + forceRefresh, + ) + return atprotoData.signingKey + } + + return async (reqCtx: { req: express.Request; res: express.Response }) => { const jwtStr = getJwtStrFromReq(reqCtx.req) if (!jwtStr) { throw new AuthRequiredError('missing jwt', 'MissingJwt') } - const payload = await verifyJwt( - jwtStr, - opts.aud, - async (did, forceRefresh) => { - const atprotoData = await idResolver.did.resolveAtprotoData( - did, - forceRefresh, - ) - return atprotoData.signingKey - }, - ) + const payload = await verifyJwt(jwtStr, opts.aud, getSigningKey) return { credentials: { did: payload.iss }, artifacts: { aud: opts.aud } } } +} export const authOptionalVerifier = ( idResolver: IdResolver, opts: { aud: string | null }, ) => { - const verify = authVerifier(idResolver, opts) + const verifyAccess = authVerifier(idResolver, opts) return async (reqCtx: { req: express.Request; res: express.Response }) => { if (!reqCtx.req.headers.authorization) { return { credentials: { did: null } } } - return verify(reqCtx) + return verifyAccess(reqCtx) } } @@ -131,9 +135,9 @@ export const buildBasicAuth = (username: string, password: string): string => { } export const getJwtStrFromReq = (req: express.Request): string | null => { - const { authorization = '' } = req.headers - if (!authorization.startsWith(BEARER)) { + const { authorization } = req.headers + if (!authorization?.startsWith(BEARER)) { return null } - return authorization.replace(BEARER, '').trim() + return authorization.slice(BEARER.length).trim() } From 2513bd1b26bdff51a0ee208fffd15563f93b8fe6 Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Fri, 1 Dec 2023 00:53:23 +0100 Subject: [PATCH 47/59] style(xrpc-server): avoid un-neccessary "if" statement (#1826) fix(xrpc-server): avoid un-neccessary "if" statement --- packages/xrpc-server/src/server.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/xrpc-server/src/server.ts b/packages/xrpc-server/src/server.ts index 1a258097d66..22281fc9740 100644 --- a/packages/xrpc-server/src/server.ts +++ b/packages/xrpc-server/src/server.ts @@ -248,11 +248,9 @@ export class Server { } // handle rate limits - if (consumeRateLimit) { - const result = await consumeRateLimit(reqCtx) - if (result instanceof RateLimitExceededError) { - return next(result) - } + const result = await consumeRateLimit(reqCtx) + if (result instanceof RateLimitExceededError) { + return next(result) } // run the handler From 691511b9b9ff5e6177b57febf9e3f097eb8d6128 Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Fri, 1 Dec 2023 00:55:20 +0100 Subject: [PATCH 48/59] fix(debug): properly type debugCatch wrapper result (#1817) --- packages/bsky/src/util/debug.ts | 4 ++-- packages/pds/src/util/debug.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/bsky/src/util/debug.ts b/packages/bsky/src/util/debug.ts index 0060fc98fa0..d83a3b6d1c5 100644 --- a/packages/bsky/src/util/debug.ts +++ b/packages/bsky/src/util/debug.ts @@ -1,7 +1,7 @@ export const debugCatch = any>(fn: Func) => { - return async (...args) => { + return async (...args: Parameters) => { try { - return await fn(...args) + return (await fn(...args)) as Awaited> } catch (err) { console.error(err) throw err diff --git a/packages/pds/src/util/debug.ts b/packages/pds/src/util/debug.ts index 0060fc98fa0..d83a3b6d1c5 100644 --- a/packages/pds/src/util/debug.ts +++ b/packages/pds/src/util/debug.ts @@ -1,7 +1,7 @@ export const debugCatch = any>(fn: Func) => { - return async (...args) => { + return async (...args: Parameters) => { try { - return await fn(...args) + return (await fn(...args)) as Awaited> } catch (err) { console.error(err) throw err From c17971a2d8e424cc7f10c071d97c07c08aa319cf Mon Sep 17 00:00:00 2001 From: bnewbold Date: Thu, 30 Nov 2023 16:19:06 -0800 Subject: [PATCH 49/59] harden datetime verification (#1702) * syntax: add datetime validator (and interop tests) * syntax: improve datetime normalization * lexicon: stronger datetime validation (from syntax package) * syntax: make datetime syntax norm test more flexible * make fmt * datetime: docs, normalize and always variant * bsky replace toSimplifiedISOSafe with normalizeDatetimeAlways * more rigorous datetime parsing on record creation * handle negative dates * syntax: disallow datetimes before year 0010 * syntax: datetime normalization functions validate output --------- Co-authored-by: dholms --- .../syntax/datetime_parse_invalid.txt | 7 ++ .../syntax/datetime_syntax_invalid.txt | 68 ++++++++++ .../syntax/datetime_syntax_valid.txt | 40 ++++++ .../src/services/indexing/plugins/block.ts | 5 +- .../indexing/plugins/feed-generator.ts | 5 +- .../src/services/indexing/plugins/follow.ts | 5 +- .../src/services/indexing/plugins/like.ts | 5 +- .../services/indexing/plugins/list-block.ts | 5 +- .../services/indexing/plugins/list-item.ts | 5 +- .../src/services/indexing/plugins/list.ts | 5 +- .../src/services/indexing/plugins/post.ts | 5 +- .../src/services/indexing/plugins/repost.ts | 5 +- .../services/indexing/plugins/thread-gate.ts | 5 +- packages/bsky/src/services/label/index.ts | 5 +- packages/lexicon/src/validators/formats.ts | 2 +- packages/lexicon/tests/general.test.ts | 4 +- packages/pds/src/repo/prepare.ts | 20 ++- packages/pds/tests/crud.test.ts | 20 ++- packages/syntax/src/datetime.ts | 112 +++++++++++++++++ packages/syntax/src/index.ts | 1 + packages/syntax/tests/datetime.test.ts | 118 ++++++++++++++++++ .../interop-files/datetime_parse_invalid.txt | 1 + .../interop-files/datetime_syntax_invalid.txt | 1 + .../interop-files/datetime_syntax_valid.txt | 1 + 24 files changed, 413 insertions(+), 37 deletions(-) create mode 100644 interop-test-files/syntax/datetime_parse_invalid.txt create mode 100644 interop-test-files/syntax/datetime_syntax_invalid.txt create mode 100644 interop-test-files/syntax/datetime_syntax_valid.txt create mode 100644 packages/syntax/src/datetime.ts create mode 100644 packages/syntax/tests/datetime.test.ts create mode 120000 packages/syntax/tests/interop-files/datetime_parse_invalid.txt create mode 120000 packages/syntax/tests/interop-files/datetime_syntax_invalid.txt create mode 120000 packages/syntax/tests/interop-files/datetime_syntax_valid.txt diff --git a/interop-test-files/syntax/datetime_parse_invalid.txt b/interop-test-files/syntax/datetime_parse_invalid.txt new file mode 100644 index 00000000000..3672453a29f --- /dev/null +++ b/interop-test-files/syntax/datetime_parse_invalid.txt @@ -0,0 +1,7 @@ +# superficial syntax parses ok, but are not valid datetimes for semantic reasons (eg, "month zero") +1985-00-12T23:20:50.123Z +1985-04-00T23:20:50.123Z +1985-13-12T23:20:50.123Z +1985-04-12T25:20:50.123Z +1985-04-12T23:99:50.123Z +1985-04-12T23:20:61.123Z diff --git a/interop-test-files/syntax/datetime_syntax_invalid.txt b/interop-test-files/syntax/datetime_syntax_invalid.txt new file mode 100644 index 00000000000..6702e43e8e7 --- /dev/null +++ b/interop-test-files/syntax/datetime_syntax_invalid.txt @@ -0,0 +1,68 @@ + +# subtle changes to: 1985-04-12T23:20:50.123Z +1985-04-12T23:20:50.123z +01985-04-12T23:20:50.123Z +985-04-12T23:20:50.123Z +1985-04-12T23:20:50.Z +1985-04-32T23;20:50.123Z +1985-04-32T23;20:50.123Z + +# en-dash and em-dash +1985—04-32T23;20:50.123Z +1985–04-32T23;20:50.123Z + +# whitespace + 1985-04-12T23:20:50.123Z +1985-04-12T23:20:50.123Z +1985-04-12T 23:20:50.123Z + +# not enough zero padding +1985-4-12T23:20:50.123Z +1985-04-2T23:20:50.123Z +1985-04-12T3:20:50.123Z +1985-04-12T23:0:50.123Z +1985-04-12T23:20:5.123Z + +# too much zero padding +01985-04-12T23:20:50.123Z +1985-004-12T23:20:50.123Z +1985-04-012T23:20:50.123Z +1985-04-12T023:20:50.123Z +1985-04-12T23:020:50.123Z +1985-04-12T23:20:050.123Z + +# strict capitalization (ISO-8601) +1985-04-12t23:20:50.123Z +1985-04-12T23:20:50.123z + +# RFC-3339, but not ISO-8601 +1985-04-12T23:20:50.123-00:00 +1985-04-12_23:20:50.123Z +1985-04-12 23:20:50.123Z + +# ISO-8601, but weird +1985-04-274T23:20:50.123Z + +# timezone is required +1985-04-12T23:20:50.123 +1985-04-12T23:20:50 + +1985-04-12 +1985-04-12T23:20Z +1985-04-12T23:20:5Z +1985-04-12T23:20:50.123 ++001985-04-12T23:20:50.123Z +23:20:50.123Z + +1985-04-12T23:20:50.123+00 +1985-04-12T23:20:50.123+00:0 +1985-04-12T23:20:50.123+0:00 +1985-04-12T23:20:50.123 +1985-04-12T23:20:50.123+0000 +1985-04-12T23:20:50.123+00 +1985-04-12T23:20:50.123+ +1985-04-12T23:20:50.123- + +# ISO-8601, but normalizes to a negative time +0000-01-01T00:00:00+01:00 +-000001-12-31T23:00:00.000Z diff --git a/interop-test-files/syntax/datetime_syntax_valid.txt b/interop-test-files/syntax/datetime_syntax_valid.txt new file mode 100644 index 00000000000..f47d539c2ff --- /dev/null +++ b/interop-test-files/syntax/datetime_syntax_valid.txt @@ -0,0 +1,40 @@ +# "preferred" +1985-04-12T23:20:50.123Z +1985-04-12T23:20:50.000Z +2000-01-01T00:00:00.000Z +1985-04-12T23:20:50.123456Z +1985-04-12T23:20:50.120Z +1985-04-12T23:20:50.120000Z + +# "supported" +1985-04-12T23:20:50.1235678912345Z +1985-04-12T23:20:50.100Z +1985-04-12T23:20:50Z +1985-04-12T23:20:50.0Z +1985-04-12T23:20:50.123+00:00 +1985-04-12T23:20:50.123-07:00 +1985-04-12T23:20:50.123+07:00 +1985-04-12T23:20:50.123+01:45 +0985-04-12T23:20:50.123-07:00 +1985-04-12T23:20:50.123-07:00 +0123-01-01T00:00:00.000Z + +# various precisions, up through at least 12 digits +1985-04-12T23:20:50.1Z +1985-04-12T23:20:50.12Z +1985-04-12T23:20:50.123Z +1985-04-12T23:20:50.1234Z +1985-04-12T23:20:50.12345Z +1985-04-12T23:20:50.123456Z +1985-04-12T23:20:50.1234567Z +1985-04-12T23:20:50.12345678Z +1985-04-12T23:20:50.123456789Z +1985-04-12T23:20:50.1234567890Z +1985-04-12T23:20:50.12345678901Z +1985-04-12T23:20:50.123456789012Z + +# extreme but currently allowed +0010-12-31T23:00:00.000Z +1000-12-31T23:00:00.000Z +1900-12-31T23:00:00.000Z +3001-12-31T23:00:00.000Z diff --git a/packages/bsky/src/services/indexing/plugins/block.ts b/packages/bsky/src/services/indexing/plugins/block.ts index bf8ae9e5029..88e62b6f5ac 100644 --- a/packages/bsky/src/services/indexing/plugins/block.ts +++ b/packages/bsky/src/services/indexing/plugins/block.ts @@ -1,6 +1,5 @@ import { Selectable } from 'kysely' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { CID } from 'multiformats/cid' import * as Block from '../../../lexicon/types/app/bsky/graph/block' import * as lex from '../../../lexicon/lexicons' @@ -27,7 +26,7 @@ const insertFn = async ( cid: cid.toString(), creator: uri.host, subjectDid: obj.subject, - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), indexedAt: timestamp, }) .onConflict((oc) => oc.doNothing()) diff --git a/packages/bsky/src/services/indexing/plugins/feed-generator.ts b/packages/bsky/src/services/indexing/plugins/feed-generator.ts index e4ae5eb4f5a..be5435966f1 100644 --- a/packages/bsky/src/services/indexing/plugins/feed-generator.ts +++ b/packages/bsky/src/services/indexing/plugins/feed-generator.ts @@ -1,6 +1,5 @@ import { Selectable } from 'kysely' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { CID } from 'multiformats/cid' import * as FeedGenerator from '../../../lexicon/types/app/bsky/feed/generator' import * as lex from '../../../lexicon/lexicons' @@ -33,7 +32,7 @@ const insertFn = async ( ? JSON.stringify(obj.descriptionFacets) : undefined, avatarCid: obj.avatar?.ref.toString(), - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), indexedAt: timestamp, }) .onConflict((oc) => oc.doNothing()) diff --git a/packages/bsky/src/services/indexing/plugins/follow.ts b/packages/bsky/src/services/indexing/plugins/follow.ts index e9a344db2fd..8655c7eba71 100644 --- a/packages/bsky/src/services/indexing/plugins/follow.ts +++ b/packages/bsky/src/services/indexing/plugins/follow.ts @@ -1,6 +1,5 @@ import { Selectable } from 'kysely' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { CID } from 'multiformats/cid' import * as Follow from '../../../lexicon/types/app/bsky/graph/follow' import * as lex from '../../../lexicon/lexicons' @@ -28,7 +27,7 @@ const insertFn = async ( cid: cid.toString(), creator: uri.host, subjectDid: obj.subject, - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), indexedAt: timestamp, }) .onConflict((oc) => oc.doNothing()) diff --git a/packages/bsky/src/services/indexing/plugins/like.ts b/packages/bsky/src/services/indexing/plugins/like.ts index 01e0fa5c4fd..703800f67c8 100644 --- a/packages/bsky/src/services/indexing/plugins/like.ts +++ b/packages/bsky/src/services/indexing/plugins/like.ts @@ -1,6 +1,5 @@ import { Selectable } from 'kysely' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { CID } from 'multiformats/cid' import * as Like from '../../../lexicon/types/app/bsky/feed/like' import * as lex from '../../../lexicon/lexicons' @@ -29,7 +28,7 @@ const insertFn = async ( creator: uri.host, subject: obj.subject.uri, subjectCid: obj.subject.cid, - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), indexedAt: timestamp, }) .onConflict((oc) => oc.doNothing()) diff --git a/packages/bsky/src/services/indexing/plugins/list-block.ts b/packages/bsky/src/services/indexing/plugins/list-block.ts index 33dc7cfc51a..3040f1aa3f9 100644 --- a/packages/bsky/src/services/indexing/plugins/list-block.ts +++ b/packages/bsky/src/services/indexing/plugins/list-block.ts @@ -1,6 +1,5 @@ import { Selectable } from 'kysely' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { CID } from 'multiformats/cid' import * as ListBlock from '../../../lexicon/types/app/bsky/graph/listblock' import * as lex from '../../../lexicon/lexicons' @@ -27,7 +26,7 @@ const insertFn = async ( cid: cid.toString(), creator: uri.host, subjectUri: obj.subject, - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), indexedAt: timestamp, }) .onConflict((oc) => oc.doNothing()) diff --git a/packages/bsky/src/services/indexing/plugins/list-item.ts b/packages/bsky/src/services/indexing/plugins/list-item.ts index 2ab125062a7..9e08145b23e 100644 --- a/packages/bsky/src/services/indexing/plugins/list-item.ts +++ b/packages/bsky/src/services/indexing/plugins/list-item.ts @@ -1,6 +1,5 @@ import { Selectable } from 'kysely' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { CID } from 'multiformats/cid' import * as ListItem from '../../../lexicon/types/app/bsky/graph/listitem' import * as lex from '../../../lexicon/lexicons' @@ -35,7 +34,7 @@ const insertFn = async ( creator: uri.host, subjectDid: obj.subject, listUri: obj.list, - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), indexedAt: timestamp, }) .onConflict((oc) => oc.doNothing()) diff --git a/packages/bsky/src/services/indexing/plugins/list.ts b/packages/bsky/src/services/indexing/plugins/list.ts index 293c457c4fb..0d078572501 100644 --- a/packages/bsky/src/services/indexing/plugins/list.ts +++ b/packages/bsky/src/services/indexing/plugins/list.ts @@ -1,6 +1,5 @@ import { Selectable } from 'kysely' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { CID } from 'multiformats/cid' import * as List from '../../../lexicon/types/app/bsky/graph/list' import * as lex from '../../../lexicon/lexicons' @@ -33,7 +32,7 @@ const insertFn = async ( ? JSON.stringify(obj.descriptionFacets) : undefined, avatarCid: obj.avatar?.ref.toString(), - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), indexedAt: timestamp, }) .onConflict((oc) => oc.doNothing()) diff --git a/packages/bsky/src/services/indexing/plugins/post.ts b/packages/bsky/src/services/indexing/plugins/post.ts index 396544b8f26..5f2fca934ce 100644 --- a/packages/bsky/src/services/indexing/plugins/post.ts +++ b/packages/bsky/src/services/indexing/plugins/post.ts @@ -1,7 +1,6 @@ import { Insertable, Selectable, sql } from 'kysely' import { CID } from 'multiformats/cid' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { jsonStringToLex } from '@atproto/lexicon' import { Record as PostRecord, @@ -68,7 +67,7 @@ const insertFn = async ( cid: cid.toString(), creator: uri.host, text: obj.text, - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), replyRoot: obj.reply?.root?.uri || null, replyRootCid: obj.reply?.root?.cid || null, replyParent: obj.reply?.parent?.uri || null, diff --git a/packages/bsky/src/services/indexing/plugins/repost.ts b/packages/bsky/src/services/indexing/plugins/repost.ts index 9c46b9b3376..ea8d517dc52 100644 --- a/packages/bsky/src/services/indexing/plugins/repost.ts +++ b/packages/bsky/src/services/indexing/plugins/repost.ts @@ -1,7 +1,6 @@ import { Selectable } from 'kysely' import { CID } from 'multiformats/cid' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import * as Repost from '../../../lexicon/types/app/bsky/feed/repost' import * as lex from '../../../lexicon/lexicons' import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' @@ -27,7 +26,7 @@ const insertFn = async ( creator: uri.host, subject: obj.subject.uri, subjectCid: obj.subject.cid, - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), indexedAt: timestamp, } const [inserted] = await Promise.all([ diff --git a/packages/bsky/src/services/indexing/plugins/thread-gate.ts b/packages/bsky/src/services/indexing/plugins/thread-gate.ts index 37f3ddb062e..9a58547f2da 100644 --- a/packages/bsky/src/services/indexing/plugins/thread-gate.ts +++ b/packages/bsky/src/services/indexing/plugins/thread-gate.ts @@ -1,6 +1,5 @@ -import { AtUri } from '@atproto/syntax' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { InvalidRequestError } from '@atproto/xrpc-server' -import { toSimplifiedISOSafe } from '@atproto/common' import { CID } from 'multiformats/cid' import * as Threadgate from '../../../lexicon/types/app/bsky/feed/threadgate' import * as lex from '../../../lexicon/lexicons' @@ -33,7 +32,7 @@ const insertFn = async ( cid: cid.toString(), creator: uri.host, postUri: obj.post, - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), indexedAt: timestamp, }) .onConflict((oc) => oc.doNothing()) diff --git a/packages/bsky/src/services/label/index.ts b/packages/bsky/src/services/label/index.ts index 7d351b95011..f44b0439ddf 100644 --- a/packages/bsky/src/services/label/index.ts +++ b/packages/bsky/src/services/label/index.ts @@ -1,6 +1,5 @@ import { sql } from 'kysely' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { Database } from '../../db' import { Label, isSelfLabels } from '../../lexicon/types/com/atproto/label/defs' import { ids } from '../../lexicon/lexicons' @@ -166,7 +165,7 @@ export function getSelfLabels(details: { const src = new AtUri(uri).host // record creator const cts = typeof record.createdAt === 'string' - ? toSimplifiedISOSafe(record.createdAt) + ? normalizeDatetimeAlways(record.createdAt) : new Date(0).toISOString() return record.labels.values.map(({ val }) => { return { src, uri, cid, val, cts, neg: false } diff --git a/packages/lexicon/src/validators/formats.ts b/packages/lexicon/src/validators/formats.ts index 63fc941628e..b786c68281f 100644 --- a/packages/lexicon/src/validators/formats.ts +++ b/packages/lexicon/src/validators/formats.ts @@ -18,7 +18,7 @@ export function datetime(path: string, value: string): ValidationResult { return { success: false, error: new ValidationError( - `${path} must be an iso8601 formatted datetime`, + `${path} must be an valid atproto datetime (both RFC-3339 and ISO-8601)`, ), } } diff --git a/packages/lexicon/tests/general.test.ts b/packages/lexicon/tests/general.test.ts index 685c99f40e0..ca9cb44dc34 100644 --- a/packages/lexicon/tests/general.test.ts +++ b/packages/lexicon/tests/general.test.ts @@ -659,7 +659,9 @@ describe('Record validation', () => { $type: 'com.example.datetime', datetime: 'bad date', }), - ).toThrow('Record/datetime must be an iso8601 formatted datetime') + ).toThrow( + 'Record/datetime must be an valid atproto datetime (both RFC-3339 and ISO-8601)', + ) }) it('Applies uri formatting constraint', () => { diff --git a/packages/pds/src/repo/prepare.ts b/packages/pds/src/repo/prepare.ts index 88201455300..06b1da95999 100644 --- a/packages/pds/src/repo/prepare.ts +++ b/packages/pds/src/repo/prepare.ts @@ -1,9 +1,10 @@ import { CID } from 'multiformats/cid' -import { AtUri } from '@atproto/syntax' +import { AtUri, ensureValidDatetime } from '@atproto/syntax' import { MINUTE, TID, dataToCborBlock } from '@atproto/common' import { LexiconDefNotFoundError, RepoRecord, + ValidationError, lexToIpld, } from '@atproto/lexicon' import { @@ -115,6 +116,7 @@ export const assertValidRecord = (record: Record) => { } try { lex.lexicons.assertValidRecord(record.$type, record) + assertValidCreatedAt(record) } catch (e) { if (e instanceof LexiconDefNotFoundError) { throw new InvalidRecordError(e.message) @@ -127,6 +129,22 @@ export const assertValidRecord = (record: Record) => { } } +// additional more rigorous check on datetimes +// this check will eventually be in the lex sdk, but this will stop the bleed until then +export const assertValidCreatedAt = (record: Record) => { + const createdAt = record['createdAt'] + if (typeof createdAt !== 'string') { + return + } + try { + ensureValidDatetime(createdAt) + } catch { + throw new ValidationError( + 'createdAt must be an valid atproto datetime (both RFC-3339 and ISO-8601)', + ) + } +} + export const setCollectionName = ( collection: string, record: RepoRecord, diff --git a/packages/pds/tests/crud.test.ts b/packages/pds/tests/crud.test.ts index 65544677ff2..f8f855ce049 100644 --- a/packages/pds/tests/crud.test.ts +++ b/packages/pds/tests/crud.test.ts @@ -13,7 +13,7 @@ import { defaultFetchHandler } from '@atproto/xrpc' import * as Post from '../src/lexicon/types/app/bsky/feed/post' import { paginateAll } from './_util' import AppContext from '../src/context' -import { ids } from '../src/lexicon/lexicons' +import { ids, lexicons } from '../src/lexicon/lexicons' const alice = { email: 'alice@test.com', @@ -579,6 +579,24 @@ describe('crud operations', () => { ) }) + it('validates datetimes more rigorously than lex sdk', async () => { + const postRecord = { + $type: 'app.bsky.feed.post', + text: 'test', + createdAt: '1985-04-12T23:20:50.123', + } + lexicons.assertValidRecord('app.bsky.feed.post', postRecord) + await expect( + aliceAgent.api.com.atproto.repo.createRecord({ + repo: alice.did, + collection: 'app.bsky.feed.post', + record: postRecord, + }), + ).rejects.toThrow( + 'Invalid app.bsky.feed.post record: createdAt must be an valid atproto datetime (both RFC-3339 and ISO-8601)', + ) + }) + describe('compare-and-swap', () => { let recordCount = 0 // Ensures unique cids const postRecord = () => ({ diff --git a/packages/syntax/src/datetime.ts b/packages/syntax/src/datetime.ts new file mode 100644 index 00000000000..96643271c8d --- /dev/null +++ b/packages/syntax/src/datetime.ts @@ -0,0 +1,112 @@ +/* Validates datetime string against atproto Lexicon 'datetime' format. + * Syntax is described at: https://atproto.com/specs/lexicon#datetime + */ +export const ensureValidDatetime = (dtStr: string): void => { + const date = new Date(dtStr) + // must parse as ISO 8601; this also verifies semantics like month is not 13 or 00 + if (isNaN(date.getTime())) { + throw new InvalidDatetimeError('datetime did not parse as ISO 8601') + } + if (date.toISOString().startsWith('-')) { + throw new InvalidDatetimeError('datetime normalized to a negative time') + } + // regex and other checks for RFC-3339 + if ( + !/^[0-9]{4}-[01][0-9]-[0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9](.[0-9]{1,20})?(Z|([+-][0-2][0-9]:[0-5][0-9]))$/.test( + dtStr, + ) + ) { + throw new InvalidDatetimeError("datetime didn't validate via regex") + } + if (dtStr.length > 64) { + throw new InvalidDatetimeError('datetime is too long (64 chars max)') + } + if (dtStr.endsWith('-00:00')) { + throw new InvalidDatetimeError( + 'datetime can not use "-00:00" for UTC timezone', + ) + } + if (dtStr.startsWith('000')) { + throw new InvalidDatetimeError('datetime so close to year zero not allowed') + } +} + +/* Same logic as ensureValidDatetime(), but returns a boolean instead of throwing an exception. + */ +export const isValidDatetime = (dtStr: string): boolean => { + try { + ensureValidDatetime(dtStr) + } catch (err) { + if (err instanceof InvalidDatetimeError) { + return false + } + throw err + } + + return true +} + +/* Takes a flexible datetime sting and normalizes representation. + * + * This function will work with any valid atproto datetime (eg, anything which isValidDatetime() is true for). It *additinally* is more flexible about accepting datetimes that don't comply to RFC 3339, or are missing timezone information, and normalizing them to a valid datetime. + * + * One use-case is a consistent, sortable string. Another is to work with older invalid createdAt datetimes. + * + * Successful output will be a valid atproto datetime with millisecond precision (3 sub-second digits) and UTC timezone with trailing 'Z' syntax. Throws `InvalidDatetimeError` if the input string could not be parsed as a datetime, even with permissive parsing. + * + * Expected output format: YYYY-MM-DDTHH:mm:ss.sssZ + */ +export const normalizeDatetime = (dtStr: string): string => { + if (isValidDatetime(dtStr)) { + const outStr = new Date(dtStr).toISOString() + if (isValidDatetime(outStr)) { + return outStr + } + } + + // check if this permissive datetime is missing a timezone + if (!/.*(([+-]\d\d:?\d\d)|[a-zA-Z])$/.test(dtStr)) { + const date = new Date(dtStr + 'Z') + if (!isNaN(date.getTime())) { + const tzStr = date.toISOString() + if (isValidDatetime(tzStr)) { + return tzStr + } + } + } + + // finally try parsing as simple datetime + const date = new Date(dtStr) + if (isNaN(date.getTime())) { + throw new InvalidDatetimeError( + 'datetime did not parse as any timestamp format', + ) + } + const isoStr = date.toISOString() + if (isValidDatetime(isoStr)) { + return isoStr + } else { + throw new InvalidDatetimeError( + 'datetime normalized to invalid timestamp string', + ) + } +} + +/* Variant of normalizeDatetime() which always returns a valid datetime strings. + * + * If a InvalidDatetimeError is encountered, returns the UNIX epoch time as a UTC datetime (1970-01-01T00:00:00.000Z). + */ +export const normalizeDatetimeAlways = (dtStr: string): string => { + try { + return normalizeDatetime(dtStr) + } catch (err) { + if (err instanceof InvalidDatetimeError) { + return new Date(0).toISOString() + } + throw err + } +} + +/* Indicates a datetime string did not pass full atproto Lexicon datetime string format checks. + */ +export class InvalidDatetimeError extends Error {} diff --git a/packages/syntax/src/index.ts b/packages/syntax/src/index.ts index 0b056b995ae..2a108e53795 100644 --- a/packages/syntax/src/index.ts +++ b/packages/syntax/src/index.ts @@ -2,3 +2,4 @@ export * from './handle' export * from './did' export * from './nsid' export * from './aturi' +export * from './datetime' diff --git a/packages/syntax/tests/datetime.test.ts b/packages/syntax/tests/datetime.test.ts new file mode 100644 index 00000000000..15fdc8dc6e2 --- /dev/null +++ b/packages/syntax/tests/datetime.test.ts @@ -0,0 +1,118 @@ +import { + isValidDatetime, + ensureValidDatetime, + normalizeDatetime, + normalizeDatetimeAlways, + InvalidDatetimeError, +} from '../src' +import * as readline from 'readline' +import * as fs from 'fs' + +describe('datetime validation', () => { + const expectValid = (h: string) => { + ensureValidDatetime(h) + normalizeDatetime(h) + normalizeDatetimeAlways(h) + } + const expectInvalid = (h: string) => { + expect(() => ensureValidDatetime(h)).toThrow(InvalidDatetimeError) + } + + it('conforms to interop valid datetimes', () => { + const lineReader = readline.createInterface({ + input: fs.createReadStream( + `${__dirname}/interop-files/datetime_syntax_valid.txt`, + ), + terminal: false, + }) + lineReader.on('line', (line) => { + if (line.startsWith('#') || line.length == 0) { + return + } + if (!isValidDatetime(line)) { + console.log(line) + } + expectValid(line) + }) + }) + + it('conforms to interop invalid datetimes', () => { + const lineReader = readline.createInterface({ + input: fs.createReadStream( + `${__dirname}/interop-files/datetime_syntax_invalid.txt`, + ), + terminal: false, + }) + lineReader.on('line', (line) => { + if (line.startsWith('#') || line.length == 0) { + return + } + expectInvalid(line) + }) + }) + + it('conforms to interop invalid parse (semantics) datetimes', () => { + const lineReader = readline.createInterface({ + input: fs.createReadStream( + `${__dirname}/interop-files/datetime_parse_invalid.txt`, + ), + terminal: false, + }) + lineReader.on('line', (line) => { + if (line.startsWith('#') || line.length == 0) { + return + } + expectInvalid(line) + }) + }) +}) + +describe('normalization', () => { + it('normalizes datetimes', () => { + expect(normalizeDatetime('1234-04-12T23:20:50Z')).toEqual( + '1234-04-12T23:20:50.000Z', + ) + expect(normalizeDatetime('1985-04-12T23:20:50Z')).toEqual( + '1985-04-12T23:20:50.000Z', + ) + expect(normalizeDatetime('1985-04-12T23:20:50.123')).toEqual( + '1985-04-12T23:20:50.123Z', + ) + expect(normalizeDatetime('1985-04-12 23:20:50.123')).toEqual( + '1985-04-12T23:20:50.123Z', + ) + expect(normalizeDatetime('1985-04-12T10:20:50.1+01:00')).toEqual( + '1985-04-12T09:20:50.100Z', + ) + expect(normalizeDatetime('Fri, 02 Jan 1999 12:34:56 GMT')).toEqual( + '1999-01-02T12:34:56.000Z', + ) + }) + + it('throws on invalid normalized datetimes', () => { + expect(() => normalizeDatetime('')).toThrow(InvalidDatetimeError) + expect(() => normalizeDatetime('blah')).toThrow(InvalidDatetimeError) + expect(() => normalizeDatetime('1999-19-39T23:20:50.123Z')).toThrow( + InvalidDatetimeError, + ) + expect(() => normalizeDatetime('-000001-12-31T23:00:00.000Z')).toThrow( + InvalidDatetimeError, + ) + expect(() => normalizeDatetime('0000-01-01T00:00:00+01:00')).toThrow( + InvalidDatetimeError, + ) + expect(() => normalizeDatetime('0001-01-01T00:00:00+01:00')).toThrow( + InvalidDatetimeError, + ) + }) + + it('normalizes datetimes always', () => { + expect(normalizeDatetimeAlways('1985-04-12T23:20:50Z')).toEqual( + '1985-04-12T23:20:50.000Z', + ) + expect(normalizeDatetimeAlways('blah')).toEqual('1970-01-01T00:00:00.000Z') + expect(normalizeDatetimeAlways('0000-01-01T00:00:00+01:00')).toEqual( + '1970-01-01T00:00:00.000Z', + ) + }) +}) diff --git a/packages/syntax/tests/interop-files/datetime_parse_invalid.txt b/packages/syntax/tests/interop-files/datetime_parse_invalid.txt new file mode 120000 index 00000000000..ef8782df266 --- /dev/null +++ b/packages/syntax/tests/interop-files/datetime_parse_invalid.txt @@ -0,0 +1 @@ +../../../../interop-test-files/syntax/datetime_parse_invalid.txt \ No newline at end of file diff --git a/packages/syntax/tests/interop-files/datetime_syntax_invalid.txt b/packages/syntax/tests/interop-files/datetime_syntax_invalid.txt new file mode 120000 index 00000000000..948c647c88c --- /dev/null +++ b/packages/syntax/tests/interop-files/datetime_syntax_invalid.txt @@ -0,0 +1 @@ +../../../../interop-test-files/syntax/datetime_syntax_invalid.txt \ No newline at end of file diff --git a/packages/syntax/tests/interop-files/datetime_syntax_valid.txt b/packages/syntax/tests/interop-files/datetime_syntax_valid.txt new file mode 120000 index 00000000000..9c74ded2ede --- /dev/null +++ b/packages/syntax/tests/interop-files/datetime_syntax_valid.txt @@ -0,0 +1 @@ +../../../../interop-test-files/syntax/datetime_syntax_valid.txt \ No newline at end of file From 3be9c741ceb6a7b54fe2e7947c8336320189c213 Mon Sep 17 00:00:00 2001 From: bnewbold Date: Fri, 1 Dec 2023 12:28:47 -0800 Subject: [PATCH 50/59] helpers for rkey and tid syntax; validate rkey at record creation time (#1738) * syntax: fix jest config displayName * syntax: TID validation * syntax: add recordkey validation * pds: verify rkey syntax at record creation time --------- Co-authored-by: dholms --- .../syntax/tid_syntax_invalid.txt | 15 +++++++ .../syntax/tid_syntax_valid.txt | 6 +++ .../src/api/com/atproto/repo/createRecord.ts | 4 ++ packages/pds/src/repo/prepare.ts | 8 +++- packages/pds/tests/crud.test.ts | 16 +++++++ packages/syntax/jest.config.js | 2 +- packages/syntax/src/index.ts | 2 + packages/syntax/src/recordkey.ts | 26 ++++++++++++ packages/syntax/src/tid.ts | 24 +++++++++++ .../recordkey_syntax_invalid.txt | 1 + .../interop-files/recordkey_syntax_valid.txt | 1 + .../interop-files/tid_syntax_invalid.txt | 1 + .../tests/interop-files/tid_syntax_valid.txt | 1 + packages/syntax/tests/recordkey.test.ts | 42 +++++++++++++++++++ packages/syntax/tests/tid.test.ts | 42 +++++++++++++++++++ 15 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 interop-test-files/syntax/tid_syntax_invalid.txt create mode 100644 interop-test-files/syntax/tid_syntax_valid.txt create mode 100644 packages/syntax/src/recordkey.ts create mode 100644 packages/syntax/src/tid.ts create mode 120000 packages/syntax/tests/interop-files/recordkey_syntax_invalid.txt create mode 120000 packages/syntax/tests/interop-files/recordkey_syntax_valid.txt create mode 120000 packages/syntax/tests/interop-files/tid_syntax_invalid.txt create mode 120000 packages/syntax/tests/interop-files/tid_syntax_valid.txt create mode 100644 packages/syntax/tests/recordkey.test.ts create mode 100644 packages/syntax/tests/tid.test.ts diff --git a/interop-test-files/syntax/tid_syntax_invalid.txt b/interop-test-files/syntax/tid_syntax_invalid.txt new file mode 100644 index 00000000000..eca90b2db86 --- /dev/null +++ b/interop-test-files/syntax/tid_syntax_invalid.txt @@ -0,0 +1,15 @@ + +# not base32 +3jzfcijpj2z21 +0000000000000 + +# too long/short +3jzfcijpj2z2aa +3jzfcijpj2z2 + +# old dashes syntax not actually supported (TTTT-TTT-TTTT-CC) +3jzf-cij-pj2z-2a + +# high bit can't be high +zzzzzzzzzzzzz +kjzfcijpj2z2a diff --git a/interop-test-files/syntax/tid_syntax_valid.txt b/interop-test-files/syntax/tid_syntax_valid.txt new file mode 100644 index 00000000000..b161a3fe14d --- /dev/null +++ b/interop-test-files/syntax/tid_syntax_valid.txt @@ -0,0 +1,6 @@ +# 13 digits +# 234567abcdefghijklmnopqrstuvwxyz + +3jzfcijpj2z2a +7777777777777 +3zzzzzzzzzzzz diff --git a/packages/pds/src/api/com/atproto/repo/createRecord.ts b/packages/pds/src/api/com/atproto/repo/createRecord.ts index 2c61d20448d..0c12ecc2285 100644 --- a/packages/pds/src/api/com/atproto/repo/createRecord.ts +++ b/packages/pds/src/api/com/atproto/repo/createRecord.ts @@ -1,5 +1,6 @@ import { CID } from 'multiformats/cid' import { InvalidRequestError, AuthRequiredError } from '@atproto/xrpc-server' +import { InvalidRecordKeyError } from '@atproto/syntax' import { prepareCreate } from '../../../../repo' import { Server } from '../../../../lexicon' import { @@ -59,6 +60,9 @@ export default function (server: Server, ctx: AppContext) { if (err instanceof InvalidRecordError) { throw new InvalidRequestError(err.message) } + if (err instanceof InvalidRecordKeyError) { + throw new InvalidRequestError(err.message) + } throw err } diff --git a/packages/pds/src/repo/prepare.ts b/packages/pds/src/repo/prepare.ts index 06b1da95999..0c311462d23 100644 --- a/packages/pds/src/repo/prepare.ts +++ b/packages/pds/src/repo/prepare.ts @@ -1,5 +1,9 @@ import { CID } from 'multiformats/cid' -import { AtUri, ensureValidDatetime } from '@atproto/syntax' +import { + AtUri, + ensureValidRecordKey, + ensureValidDatetime, +} from '@atproto/syntax' import { MINUTE, TID, dataToCborBlock } from '@atproto/common' import { LexiconDefNotFoundError, @@ -188,6 +192,8 @@ export const prepareCreate = async (opts: { } const rkey = opts.rkey || nextRkey.toString() + // @TODO: validate against Lexicon record 'key' type, not just overall recordkey syntax + ensureValidRecordKey(rkey) assertNoExplicitSlurs(rkey, record) return { action: WriteOpAction.Create, diff --git a/packages/pds/tests/crud.test.ts b/packages/pds/tests/crud.test.ts index f8f855ce049..5a7189b3707 100644 --- a/packages/pds/tests/crud.test.ts +++ b/packages/pds/tests/crud.test.ts @@ -567,6 +567,22 @@ describe('crud operations', () => { ) }) + it('requires valid rkey', async () => { + await expect( + aliceAgent.api.com.atproto.repo.createRecord({ + repo: alice.did, + collection: 'app.bsky.feed.generator', + record: { + $type: 'app.bsky.feed.generator', + did: 'did:web:dummy.example.com', + displayName: 'dummy', + createdAt: new Date().toISOString(), + }, + rkey: '..', + }), + ).rejects.toThrow('record key can not be "." or ".."') + }) + it('validates the record on write', async () => { await expect( aliceAgent.api.com.atproto.repo.createRecord({ diff --git a/packages/syntax/jest.config.js b/packages/syntax/jest.config.js index 096d01562c4..08a81143eda 100644 --- a/packages/syntax/jest.config.js +++ b/packages/syntax/jest.config.js @@ -2,5 +2,5 @@ const base = require('../../jest.config.base.js') module.exports = { ...base, - displayName: 'Identifier', + displayName: 'Syntax', } diff --git a/packages/syntax/src/index.ts b/packages/syntax/src/index.ts index 2a108e53795..f8345a770ce 100644 --- a/packages/syntax/src/index.ts +++ b/packages/syntax/src/index.ts @@ -2,4 +2,6 @@ export * from './handle' export * from './did' export * from './nsid' export * from './aturi' +export * from './tid' +export * from './recordkey' export * from './datetime' diff --git a/packages/syntax/src/recordkey.ts b/packages/syntax/src/recordkey.ts new file mode 100644 index 00000000000..6d424a1b8b4 --- /dev/null +++ b/packages/syntax/src/recordkey.ts @@ -0,0 +1,26 @@ +export const ensureValidRecordKey = (rkey: string): void => { + if (rkey.length > 512 || rkey.length < 1) { + throw new InvalidRecordKeyError('record key must be 1 to 512 characters') + } + // simple regex to enforce most constraints via just regex and length. + if (!/^[a-zA-Z0-9_~.-]{1,512}$/.test(rkey)) { + throw new InvalidRecordKeyError('record key syntax not valid (regex)') + } + if (rkey == '.' || rkey == '..') + throw new InvalidRecordKeyError('record key can not be "." or ".."') +} + +export const isValidRecordKey = (rkey: string): boolean => { + try { + ensureValidRecordKey(rkey) + } catch (err) { + if (err instanceof InvalidRecordKeyError) { + return false + } + throw err + } + + return true +} + +export class InvalidRecordKeyError extends Error {} diff --git a/packages/syntax/src/tid.ts b/packages/syntax/src/tid.ts new file mode 100644 index 00000000000..32cccb1a8ec --- /dev/null +++ b/packages/syntax/src/tid.ts @@ -0,0 +1,24 @@ +export const ensureValidTid = (tid: string): void => { + if (tid.length != 13) { + throw new InvalidTidError('TID must be 13 characters') + } + // simple regex to enforce most constraints via just regex and length. + if (!/^[234567abcdefghij][234567abcdefghijklmnopqrstuvwxyz]{12}$/.test(tid)) { + throw new InvalidTidError('TID syntax not valid (regex)') + } +} + +export const isValidTid = (tid: string): boolean => { + try { + ensureValidTid(tid) + } catch (err) { + if (err instanceof InvalidTidError) { + return false + } + throw err + } + + return true +} + +export class InvalidTidError extends Error {} diff --git a/packages/syntax/tests/interop-files/recordkey_syntax_invalid.txt b/packages/syntax/tests/interop-files/recordkey_syntax_invalid.txt new file mode 120000 index 00000000000..ebfbb2dd586 --- /dev/null +++ b/packages/syntax/tests/interop-files/recordkey_syntax_invalid.txt @@ -0,0 +1 @@ +../../../../interop-test-files/syntax/recordkey_syntax_invalid.txt \ No newline at end of file diff --git a/packages/syntax/tests/interop-files/recordkey_syntax_valid.txt b/packages/syntax/tests/interop-files/recordkey_syntax_valid.txt new file mode 120000 index 00000000000..eb4ccf456f8 --- /dev/null +++ b/packages/syntax/tests/interop-files/recordkey_syntax_valid.txt @@ -0,0 +1 @@ +../../../../interop-test-files/syntax/recordkey_syntax_valid.txt \ No newline at end of file diff --git a/packages/syntax/tests/interop-files/tid_syntax_invalid.txt b/packages/syntax/tests/interop-files/tid_syntax_invalid.txt new file mode 120000 index 00000000000..cc52db07c08 --- /dev/null +++ b/packages/syntax/tests/interop-files/tid_syntax_invalid.txt @@ -0,0 +1 @@ +../../../../interop-test-files/syntax/tid_syntax_invalid.txt \ No newline at end of file diff --git a/packages/syntax/tests/interop-files/tid_syntax_valid.txt b/packages/syntax/tests/interop-files/tid_syntax_valid.txt new file mode 120000 index 00000000000..3d5bee46ad7 --- /dev/null +++ b/packages/syntax/tests/interop-files/tid_syntax_valid.txt @@ -0,0 +1 @@ +../../../../interop-test-files/syntax/tid_syntax_valid.txt \ No newline at end of file diff --git a/packages/syntax/tests/recordkey.test.ts b/packages/syntax/tests/recordkey.test.ts new file mode 100644 index 00000000000..002e4cf55c6 --- /dev/null +++ b/packages/syntax/tests/recordkey.test.ts @@ -0,0 +1,42 @@ +import { ensureValidRecordKey, InvalidRecordKeyError } from '../src' +import * as readline from 'readline' +import * as fs from 'fs' + +describe('recordkey validation', () => { + const expectValid = (r: string) => { + ensureValidRecordKey(r) + } + const expectInvalid = (r: string) => { + expect(() => ensureValidRecordKey(r)).toThrow(InvalidRecordKeyError) + } + + it('conforms to interop valid recordkey', () => { + const lineReader = readline.createInterface({ + input: fs.createReadStream( + `${__dirname}/interop-files/recordkey_syntax_valid.txt`, + ), + terminal: false, + }) + lineReader.on('line', (line) => { + if (line.startsWith('#') || line.length == 0) { + return + } + expectValid(line) + }) + }) + + it('conforms to interop invalid recordkeys', () => { + const lineReader = readline.createInterface({ + input: fs.createReadStream( + `${__dirname}/interop-files/recordkey_syntax_invalid.txt`, + ), + terminal: false, + }) + lineReader.on('line', (line) => { + if (line.startsWith('#') || line.length == 0) { + return + } + expectInvalid(line) + }) + }) +}) diff --git a/packages/syntax/tests/tid.test.ts b/packages/syntax/tests/tid.test.ts new file mode 100644 index 00000000000..4ac65ae8fef --- /dev/null +++ b/packages/syntax/tests/tid.test.ts @@ -0,0 +1,42 @@ +import { ensureValidTid, InvalidTidError } from '../src' +import * as readline from 'readline' +import * as fs from 'fs' + +describe('tid validation', () => { + const expectValid = (t: string) => { + ensureValidTid(t) + } + const expectInvalid = (t: string) => { + expect(() => ensureValidTid(t)).toThrow(InvalidTidError) + } + + it('conforms to interop valid tid', () => { + const lineReader = readline.createInterface({ + input: fs.createReadStream( + `${__dirname}/interop-files/tid_syntax_valid.txt`, + ), + terminal: false, + }) + lineReader.on('line', (line) => { + if (line.startsWith('#') || line.length == 0) { + return + } + expectValid(line) + }) + }) + + it('conforms to interop invalid tids', () => { + const lineReader = readline.createInterface({ + input: fs.createReadStream( + `${__dirname}/interop-files/tid_syntax_invalid.txt`, + ), + terminal: false, + }) + lineReader.on('line', (line) => { + if (line.startsWith('#') || line.length == 0) { + return + } + expectInvalid(line) + }) + }) +}) From 8d9b1f70cd22ae3562aac8809c15b73646133f78 Mon Sep 17 00:00:00 2001 From: intrnl Date: Sat, 2 Dec 2023 03:28:54 +0700 Subject: [PATCH 51/59] Attach record URI to listItemView (#1758) * feat: attach record uri to listitemview * chore: attempt to update test snapshots --- lexicons/app/bsky/graph/defs.json | 3 ++- packages/api/src/client/lexicons.ts | 6 +++++- packages/api/src/client/types/app/bsky/graph/defs.ts | 1 + packages/bsky/src/api/app/bsky/graph/getList.ts | 4 ++-- packages/bsky/src/lexicon/lexicons.ts | 6 +++++- packages/bsky/src/lexicon/types/app/bsky/graph/defs.ts | 1 + packages/bsky/src/services/graph/index.ts | 6 +++++- .../bsky/tests/views/__snapshots__/block-lists.test.ts.snap | 2 ++ .../bsky/tests/views/__snapshots__/mute-lists.test.ts.snap | 2 ++ packages/pds/src/lexicon/lexicons.ts | 6 +++++- packages/pds/src/lexicon/types/app/bsky/graph/defs.ts | 1 + packages/pds/tests/proxied/__snapshots__/views.test.ts.snap | 6 ++++-- 12 files changed, 35 insertions(+), 9 deletions(-) diff --git a/lexicons/app/bsky/graph/defs.json b/lexicons/app/bsky/graph/defs.json index c957f211670..219c4a6de27 100644 --- a/lexicons/app/bsky/graph/defs.json +++ b/lexicons/app/bsky/graph/defs.json @@ -40,8 +40,9 @@ }, "listItemView": { "type": "object", - "required": ["subject"], + "required": ["uri", "subject"], "properties": { + "uri": { "type": "string", "format": "at-uri" }, "subject": { "type": "ref", "ref": "app.bsky.actor.defs#profileView" } } }, diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index 25a60f90054..a6377d85a9a 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -6423,8 +6423,12 @@ export const schemaDict = { }, listItemView: { type: 'object', - required: ['subject'], + required: ['uri', 'subject'], properties: { + uri: { + type: 'string', + format: 'at-uri', + }, subject: { type: 'ref', ref: 'lex:app.bsky.actor.defs#profileView', diff --git a/packages/api/src/client/types/app/bsky/graph/defs.ts b/packages/api/src/client/types/app/bsky/graph/defs.ts index 19c2cc81337..0b22401bcdf 100644 --- a/packages/api/src/client/types/app/bsky/graph/defs.ts +++ b/packages/api/src/client/types/app/bsky/graph/defs.ts @@ -58,6 +58,7 @@ export function validateListView(v: unknown): ValidationResult { } export interface ListItemView { + uri: string subject: AppBskyActorDefs.ProfileView [k: string]: unknown } diff --git a/packages/bsky/src/api/app/bsky/graph/getList.ts b/packages/bsky/src/api/app/bsky/graph/getList.ts index 3c95357d005..82a70848cd9 100644 --- a/packages/bsky/src/api/app/bsky/graph/getList.ts +++ b/packages/bsky/src/api/app/bsky/graph/getList.ts @@ -104,7 +104,7 @@ const presentation = (state: HydrationState, ctx: Context) => { const items = mapDefined(listItems, (item) => { const subject = actors[item.did] if (!subject) return - return { subject } + return { uri: item.uri, subject } }) return { list: listView, items, cursor } } @@ -122,7 +122,7 @@ type Params = QueryParams & { type SkeletonState = { params: Params list: Actor & ListInfo - listItems: (Actor & { cid: string; sortAt: string })[] + listItems: (Actor & { uri: string; cid: string; sortAt: string })[] cursor?: string } diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index 25a60f90054..a6377d85a9a 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -6423,8 +6423,12 @@ export const schemaDict = { }, listItemView: { type: 'object', - required: ['subject'], + required: ['uri', 'subject'], properties: { + uri: { + type: 'string', + format: 'at-uri', + }, subject: { type: 'ref', ref: 'lex:app.bsky.actor.defs#profileView', diff --git a/packages/bsky/src/lexicon/types/app/bsky/graph/defs.ts b/packages/bsky/src/lexicon/types/app/bsky/graph/defs.ts index f6c7cb7d77d..be2d8c385d9 100644 --- a/packages/bsky/src/lexicon/types/app/bsky/graph/defs.ts +++ b/packages/bsky/src/lexicon/types/app/bsky/graph/defs.ts @@ -58,6 +58,7 @@ export function validateListView(v: unknown): ValidationResult { } export interface ListItemView { + uri: string subject: AppBskyActorDefs.ProfileView [k: string]: unknown } diff --git a/packages/bsky/src/services/graph/index.ts b/packages/bsky/src/services/graph/index.ts index 190e3ac0661..b154a8c47bb 100644 --- a/packages/bsky/src/services/graph/index.ts +++ b/packages/bsky/src/services/graph/index.ts @@ -109,7 +109,11 @@ export class GraphService { .selectFrom('list_item') .innerJoin('actor as subject', 'subject.did', 'list_item.subjectDid') .selectAll('subject') - .select(['list_item.cid as cid', 'list_item.sortAt as sortAt']) + .select([ + 'list_item.uri as uri', + 'list_item.cid as cid', + 'list_item.sortAt as sortAt', + ]) } async getBlockAndMuteState( diff --git a/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap b/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap index 86fe23283c4..1f4f42f2003 100644 --- a/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap +++ b/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap @@ -489,6 +489,7 @@ Object { "muted": false, }, }, + "uri": "record(4)", }, Object { "subject": Object { @@ -541,6 +542,7 @@ Object { "muted": false, }, }, + "uri": "record(5)", }, ], "list": Object { diff --git a/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap b/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap index 438b48b4fdd..8b88231fe3b 100644 --- a/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap +++ b/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap @@ -518,6 +518,7 @@ Object { }, }, }, + "uri": "record(3)", }, Object { "subject": Object { @@ -545,6 +546,7 @@ Object { }, }, }, + "uri": "record(5)", }, ], "list": Object { diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index 25a60f90054..a6377d85a9a 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -6423,8 +6423,12 @@ export const schemaDict = { }, listItemView: { type: 'object', - required: ['subject'], + required: ['uri', 'subject'], properties: { + uri: { + type: 'string', + format: 'at-uri', + }, subject: { type: 'ref', ref: 'lex:app.bsky.actor.defs#profileView', diff --git a/packages/pds/src/lexicon/types/app/bsky/graph/defs.ts b/packages/pds/src/lexicon/types/app/bsky/graph/defs.ts index f6c7cb7d77d..be2d8c385d9 100644 --- a/packages/pds/src/lexicon/types/app/bsky/graph/defs.ts +++ b/packages/pds/src/lexicon/types/app/bsky/graph/defs.ts @@ -58,6 +58,7 @@ export function validateListView(v: unknown): ValidationResult { } export interface ListItemView { + uri: string subject: AppBskyActorDefs.ProfileView [k: string]: unknown } diff --git a/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap b/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap index 0dbe9b5498d..5fcc0be8faf 100644 --- a/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap +++ b/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap @@ -3326,6 +3326,7 @@ Object { "muted": false, }, }, + "uri": "record(3)", }, Object { "subject": Object { @@ -3341,7 +3342,7 @@ Object { "cts": "1970-01-01T00:00:00.000Z", "neg": false, "src": "user(3)", - "uri": "record(5)", + "uri": "record(7)", "val": "self-label-a", }, Object { @@ -3349,7 +3350,7 @@ Object { "cts": "1970-01-01T00:00:00.000Z", "neg": false, "src": "user(3)", - "uri": "record(5)", + "uri": "record(7)", "val": "self-label-b", }, ], @@ -3358,6 +3359,7 @@ Object { "muted": false, }, }, + "uri": "record(6)", }, ], "list": Object { From a4e5abf5f4f7f855041c34d7cf945cf066566bd8 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Fri, 1 Dec 2023 15:07:17 -0600 Subject: [PATCH 52/59] Fix snapshots for list items (#1911) fix snapshots for list items --- .../tests/views/__snapshots__/block-lists.test.ts.snap | 3 ++- .../tests/views/__snapshots__/mute-lists.test.ts.snap | 9 +++++---- .../pds/tests/proxied/__snapshots__/views.test.ts.snap | 4 ++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap b/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap index 1f4f42f2003..7f0989a5975 100644 --- a/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap +++ b/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap @@ -514,6 +514,7 @@ Object { "muted": false, }, }, + "uri": "record(5)", }, Object { "subject": Object { @@ -542,7 +543,7 @@ Object { "muted": false, }, }, - "uri": "record(5)", + "uri": "record(6)", }, ], "list": Object { diff --git a/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap b/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap index 8b88231fe3b..d4b11f0d235 100644 --- a/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap +++ b/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap @@ -495,6 +495,7 @@ Object { "muted": false, }, }, + "uri": "record(3)", }, Object { "subject": Object { @@ -503,7 +504,7 @@ Object { "labels": Array [], "viewer": Object { "blockedBy": false, - "followedBy": "record(3)", + "followedBy": "record(5)", "muted": true, "mutedByList": Object { "avatar": "https://bsky.public.url/img/avatar/plain/user(0)/cids(1)@jpeg", @@ -518,7 +519,7 @@ Object { }, }, }, - "uri": "record(3)", + "uri": "record(4)", }, Object { "subject": Object { @@ -531,7 +532,7 @@ Object { "labels": Array [], "viewer": Object { "blockedBy": false, - "following": "record(4)", + "following": "record(7)", "muted": true, "mutedByList": Object { "avatar": "https://bsky.public.url/img/avatar/plain/user(0)/cids(1)@jpeg", @@ -546,7 +547,7 @@ Object { }, }, }, - "uri": "record(5)", + "uri": "record(6)", }, ], "list": Object { diff --git a/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap b/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap index 5fcc0be8faf..f856407ccbc 100644 --- a/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap +++ b/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap @@ -3321,8 +3321,8 @@ Object { "labels": Array [], "viewer": Object { "blockedBy": false, - "followedBy": "record(4)", - "following": "record(3)", + "followedBy": "record(5)", + "following": "record(4)", "muted": false, }, }, From 378fc6132f621ca517897c9467ed5bba134b3776 Mon Sep 17 00:00:00 2001 From: devin ivy Date: Fri, 1 Dec 2023 18:10:23 -0500 Subject: [PATCH 53/59] Additional @atproto/api 0.6.24 changeset (#1912) api changeset --- .changeset/spotty-guests-taste.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/spotty-guests-taste.md diff --git a/.changeset/spotty-guests-taste.md b/.changeset/spotty-guests-taste.md new file mode 100644 index 00000000000..3d1d2b3e0ca --- /dev/null +++ b/.changeset/spotty-guests-taste.md @@ -0,0 +1,5 @@ +--- +'@atproto/api': patch +--- + +Contains breaking lexicon changes: removing legacy com.atproto admin endpoints, making uri field required on app.bsky list views. From 1f3fad2829df581221b5837998b97ec4c0147f0e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 1 Dec 2023 18:14:01 -0500 Subject: [PATCH 54/59] Version packages (#1909) Co-authored-by: github-actions[bot] --- .changeset/brave-swans-kiss.md | 5 ----- .changeset/spotty-guests-taste.md | 5 ----- packages/api/CHANGELOG.md | 11 +++++++++++ packages/api/package.json | 2 +- packages/aws/CHANGELOG.md | 7 +++++++ packages/aws/package.json | 2 +- packages/bsky/CHANGELOG.md | 11 +++++++++++ packages/bsky/package.json | 2 +- packages/dev-env/CHANGELOG.md | 12 ++++++++++++ packages/dev-env/package.json | 2 +- packages/lex-cli/CHANGELOG.md | 8 ++++++++ packages/lex-cli/package.json | 2 +- packages/lexicon/CHANGELOG.md | 7 +++++++ packages/lexicon/package.json | 2 +- packages/pds/CHANGELOG.md | 13 +++++++++++++ packages/pds/package.json | 2 +- packages/repo/CHANGELOG.md | 8 ++++++++ packages/repo/package.json | 2 +- packages/syntax/CHANGELOG.md | 6 ++++++ packages/syntax/package.json | 2 +- packages/xrpc-server/CHANGELOG.md | 7 +++++++ packages/xrpc-server/package.json | 2 +- packages/xrpc/CHANGELOG.md | 7 +++++++ packages/xrpc/package.json | 2 +- 24 files changed, 108 insertions(+), 21 deletions(-) delete mode 100644 .changeset/brave-swans-kiss.md delete mode 100644 .changeset/spotty-guests-taste.md diff --git a/.changeset/brave-swans-kiss.md b/.changeset/brave-swans-kiss.md deleted file mode 100644 index e6e42a1bcb2..00000000000 --- a/.changeset/brave-swans-kiss.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@atproto/syntax': patch ---- - -prevent unnecessary throw/catch on uri syntax diff --git a/.changeset/spotty-guests-taste.md b/.changeset/spotty-guests-taste.md deleted file mode 100644 index 3d1d2b3e0ca..00000000000 --- a/.changeset/spotty-guests-taste.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@atproto/api': patch ---- - -Contains breaking lexicon changes: removing legacy com.atproto admin endpoints, making uri field required on app.bsky list views. diff --git a/packages/api/CHANGELOG.md b/packages/api/CHANGELOG.md index ca4ed16617b..a2ecd8e4b5a 100644 --- a/packages/api/CHANGELOG.md +++ b/packages/api/CHANGELOG.md @@ -1,5 +1,16 @@ # @atproto/api +## 0.6.24 + +### Patch Changes + +- [#1912](https://github.com/bluesky-social/atproto/pull/1912) [`378fc613`](https://github.com/bluesky-social/atproto/commit/378fc6132f621ca517897c9467ed5bba134b3776) Thanks [@devinivy](https://github.com/devinivy)! - Contains breaking lexicon changes: removing legacy com.atproto admin endpoints, making uri field required on app.bsky list views. + +- Updated dependencies [[`3c0ef382`](https://github.com/bluesky-social/atproto/commit/3c0ef382c12a413cc971ae47ffb341236c545f60)]: + - @atproto/syntax@0.1.5 + - @atproto/lexicon@0.3.1 + - @atproto/xrpc@0.4.1 + ## 0.6.23 ### Patch Changes diff --git a/packages/api/package.json b/packages/api/package.json index 8b7be93f6dc..9bf7c547b19 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/api", - "version": "0.6.24-next.1", + "version": "0.6.24", "license": "MIT", "description": "Client library for atproto and Bluesky", "keywords": [ diff --git a/packages/aws/CHANGELOG.md b/packages/aws/CHANGELOG.md index e49e47b3f5f..b804e0719e4 100644 --- a/packages/aws/CHANGELOG.md +++ b/packages/aws/CHANGELOG.md @@ -1,5 +1,12 @@ # @atproto/aws +## 0.1.6 + +### Patch Changes + +- Updated dependencies []: + - @atproto/repo@0.3.6 + ## 0.1.5 ### Patch Changes diff --git a/packages/aws/package.json b/packages/aws/package.json index e5f0b6c5507..949cfaa845e 100644 --- a/packages/aws/package.json +++ b/packages/aws/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/aws", - "version": "0.1.5", + "version": "0.1.6", "license": "MIT", "description": "Shared AWS cloud API helpers for atproto services", "keywords": [ diff --git a/packages/bsky/CHANGELOG.md b/packages/bsky/CHANGELOG.md index 7cbb585dfeb..0dd20ba567f 100644 --- a/packages/bsky/CHANGELOG.md +++ b/packages/bsky/CHANGELOG.md @@ -1,5 +1,16 @@ # @atproto/bsky +## 0.0.16 + +### Patch Changes + +- Updated dependencies [[`3c0ef382`](https://github.com/bluesky-social/atproto/commit/3c0ef382c12a413cc971ae47ffb341236c545f60), [`378fc613`](https://github.com/bluesky-social/atproto/commit/378fc6132f621ca517897c9467ed5bba134b3776)]: + - @atproto/syntax@0.1.5 + - @atproto/api@0.6.24 + - @atproto/lexicon@0.3.1 + - @atproto/repo@0.3.6 + - @atproto/xrpc-server@0.4.2 + ## 0.0.15 ### Patch Changes diff --git a/packages/bsky/package.json b/packages/bsky/package.json index c713cd72227..ad86a2fff21 100644 --- a/packages/bsky/package.json +++ b/packages/bsky/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/bsky", - "version": "0.0.15", + "version": "0.0.16", "license": "MIT", "description": "Reference implementation of app.bsky App View (Bluesky API)", "keywords": [ diff --git a/packages/dev-env/CHANGELOG.md b/packages/dev-env/CHANGELOG.md index 62f533f6b96..c36d7e71574 100644 --- a/packages/dev-env/CHANGELOG.md +++ b/packages/dev-env/CHANGELOG.md @@ -1,5 +1,17 @@ # @atproto/dev-env +## 0.2.16 + +### Patch Changes + +- Updated dependencies [[`3c0ef382`](https://github.com/bluesky-social/atproto/commit/3c0ef382c12a413cc971ae47ffb341236c545f60), [`378fc613`](https://github.com/bluesky-social/atproto/commit/378fc6132f621ca517897c9467ed5bba134b3776)]: + - @atproto/syntax@0.1.5 + - @atproto/api@0.6.24 + - @atproto/bsky@0.0.16 + - @atproto/lexicon@0.3.1 + - @atproto/pds@0.3.4 + - @atproto/xrpc-server@0.4.2 + ## 0.2.15 ### Patch Changes diff --git a/packages/dev-env/package.json b/packages/dev-env/package.json index eb6f31d6986..37c5b47630d 100644 --- a/packages/dev-env/package.json +++ b/packages/dev-env/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/dev-env", - "version": "0.2.15", + "version": "0.2.16", "license": "MIT", "description": "Local development environment helper for atproto development", "keywords": [ diff --git a/packages/lex-cli/CHANGELOG.md b/packages/lex-cli/CHANGELOG.md index b9c617f9a9d..29491171a36 100644 --- a/packages/lex-cli/CHANGELOG.md +++ b/packages/lex-cli/CHANGELOG.md @@ -1,5 +1,13 @@ # @atproto/lex-cli +## 0.2.5 + +### Patch Changes + +- Updated dependencies [[`3c0ef382`](https://github.com/bluesky-social/atproto/commit/3c0ef382c12a413cc971ae47ffb341236c545f60)]: + - @atproto/syntax@0.1.5 + - @atproto/lexicon@0.3.1 + ## 0.2.4 ### Patch Changes diff --git a/packages/lex-cli/package.json b/packages/lex-cli/package.json index 2f57a55b3cb..19f77cbe0a9 100644 --- a/packages/lex-cli/package.json +++ b/packages/lex-cli/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/lex-cli", - "version": "0.2.4", + "version": "0.2.5", "license": "MIT", "description": "TypeScript codegen tool for atproto Lexicon schemas", "keywords": [ diff --git a/packages/lexicon/CHANGELOG.md b/packages/lexicon/CHANGELOG.md index 7194b0258ba..24e2ea99a7d 100644 --- a/packages/lexicon/CHANGELOG.md +++ b/packages/lexicon/CHANGELOG.md @@ -1,5 +1,12 @@ # @atproto/lexicon +## 0.3.1 + +### Patch Changes + +- Updated dependencies [[`3c0ef382`](https://github.com/bluesky-social/atproto/commit/3c0ef382c12a413cc971ae47ffb341236c545f60)]: + - @atproto/syntax@0.1.5 + ## 0.3.0 ### Minor Changes diff --git a/packages/lexicon/package.json b/packages/lexicon/package.json index fc776e7c273..4f0b05d20d8 100644 --- a/packages/lexicon/package.json +++ b/packages/lexicon/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/lexicon", - "version": "0.3.0", + "version": "0.3.1", "license": "MIT", "description": "atproto Lexicon schema language library", "keywords": [ diff --git a/packages/pds/CHANGELOG.md b/packages/pds/CHANGELOG.md index 34cda77d01e..3a1bf615d79 100644 --- a/packages/pds/CHANGELOG.md +++ b/packages/pds/CHANGELOG.md @@ -1,5 +1,18 @@ # @atproto/pds +## 0.3.4 + +### Patch Changes + +- Updated dependencies [[`3c0ef382`](https://github.com/bluesky-social/atproto/commit/3c0ef382c12a413cc971ae47ffb341236c545f60), [`378fc613`](https://github.com/bluesky-social/atproto/commit/378fc6132f621ca517897c9467ed5bba134b3776)]: + - @atproto/syntax@0.1.5 + - @atproto/api@0.6.24 + - @atproto/lexicon@0.3.1 + - @atproto/repo@0.3.6 + - @atproto/xrpc@0.4.1 + - @atproto/xrpc-server@0.4.2 + - @atproto/aws@0.1.6 + ## 0.3.3 ### Patch Changes diff --git a/packages/pds/package.json b/packages/pds/package.json index 857fd912611..c1f301c8fda 100644 --- a/packages/pds/package.json +++ b/packages/pds/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/pds", - "version": "0.3.3", + "version": "0.3.4", "license": "MIT", "description": "Reference implementation of atproto Personal Data Server (PDS)", "keywords": [ diff --git a/packages/repo/CHANGELOG.md b/packages/repo/CHANGELOG.md index 35c6c3b9c44..448005d16d2 100644 --- a/packages/repo/CHANGELOG.md +++ b/packages/repo/CHANGELOG.md @@ -1,5 +1,13 @@ # @atproto/repo +## 0.3.6 + +### Patch Changes + +- Updated dependencies [[`3c0ef382`](https://github.com/bluesky-social/atproto/commit/3c0ef382c12a413cc971ae47ffb341236c545f60)]: + - @atproto/syntax@0.1.5 + - @atproto/lexicon@0.3.1 + ## 0.3.5 ### Patch Changes diff --git a/packages/repo/package.json b/packages/repo/package.json index c36cbcf668e..b6a3b87607e 100644 --- a/packages/repo/package.json +++ b/packages/repo/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/repo", - "version": "0.3.5", + "version": "0.3.6", "license": "MIT", "description": "atproto repo and MST implementation", "keywords": [ diff --git a/packages/syntax/CHANGELOG.md b/packages/syntax/CHANGELOG.md index bc243559bbf..ad736e658a0 100644 --- a/packages/syntax/CHANGELOG.md +++ b/packages/syntax/CHANGELOG.md @@ -1,5 +1,11 @@ # @atproto/syntax +## 0.1.5 + +### Patch Changes + +- [#1908](https://github.com/bluesky-social/atproto/pull/1908) [`3c0ef382`](https://github.com/bluesky-social/atproto/commit/3c0ef382c12a413cc971ae47ffb341236c545f60) Thanks [@gaearon](https://github.com/gaearon)! - prevent unnecessary throw/catch on uri syntax + ## 0.1.4 ### Patch Changes diff --git a/packages/syntax/package.json b/packages/syntax/package.json index 645926886f5..d6f0ea11fd6 100644 --- a/packages/syntax/package.json +++ b/packages/syntax/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/syntax", - "version": "0.1.4", + "version": "0.1.5", "license": "MIT", "description": "Validation for atproto identifiers and formats: DID, handle, NSID, AT URI, etc", "keywords": [ diff --git a/packages/xrpc-server/CHANGELOG.md b/packages/xrpc-server/CHANGELOG.md index 29db2da0d61..c96e2de140f 100644 --- a/packages/xrpc-server/CHANGELOG.md +++ b/packages/xrpc-server/CHANGELOG.md @@ -1,5 +1,12 @@ # @atproto/xrpc-server +## 0.4.2 + +### Patch Changes + +- Updated dependencies []: + - @atproto/lexicon@0.3.1 + ## 0.4.1 ### Patch Changes diff --git a/packages/xrpc-server/package.json b/packages/xrpc-server/package.json index d3d2fa63c98..b68ecb03ea4 100644 --- a/packages/xrpc-server/package.json +++ b/packages/xrpc-server/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/xrpc-server", - "version": "0.4.1", + "version": "0.4.2", "license": "MIT", "description": "atproto HTTP API (XRPC) server library", "keywords": [ diff --git a/packages/xrpc/CHANGELOG.md b/packages/xrpc/CHANGELOG.md index 76ffea62682..69977ee06d3 100644 --- a/packages/xrpc/CHANGELOG.md +++ b/packages/xrpc/CHANGELOG.md @@ -1,5 +1,12 @@ # @atproto/xrpc +## 0.4.1 + +### Patch Changes + +- Updated dependencies []: + - @atproto/lexicon@0.3.1 + ## 0.4.0 ### Minor Changes diff --git a/packages/xrpc/package.json b/packages/xrpc/package.json index 49cf7a06aa3..e2accc2750d 100644 --- a/packages/xrpc/package.json +++ b/packages/xrpc/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/xrpc", - "version": "0.4.0", + "version": "0.4.1", "license": "MIT", "description": "atproto HTTP API (XRPC) client library", "keywords": [ From 9cec13ee46aabcda71a6245e28e96d968ed9a289 Mon Sep 17 00:00:00 2001 From: devin ivy Date: Fri, 1 Dec 2023 18:15:27 -0500 Subject: [PATCH 55/59] Do not generate notifs when post violates threadgate (#1901) * do not generate notifs when post violates threadgate * don't count threadgate-violating replies, style --- .../src/services/indexing/plugins/post.ts | 13 ++++++++++ .../bsky/tests/views/threadgating.test.ts | 26 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/packages/bsky/src/services/indexing/plugins/post.ts b/packages/bsky/src/services/indexing/plugins/post.ts index 5f2fca934ce..af581b3bdff 100644 --- a/packages/bsky/src/services/indexing/plugins/post.ts +++ b/packages/bsky/src/services/indexing/plugins/post.ts @@ -112,6 +112,7 @@ const insertFn = async ( obj.reply, ) if (invalidReplyRoot || violatesThreadGate) { + Object.assign(insertedPost, { invalidReplyRoot, violatesThreadGate }) await db .updateTable('post') .where('uri', '=', post.uri) @@ -241,6 +242,13 @@ const notifsForInsert = (obj: IndexedPost) => { } } + if (obj.post.violatesThreadGate) { + // don't generate reply notifications when post violates threadgate + return notifs + } + + // reply notifications + for (const ancestor of obj.ancestors ?? []) { if (ancestor.uri === obj.post.uri) continue // no need to notify for own post if (ancestor.height < REPLY_NOTIF_DEPTH) { @@ -353,6 +361,11 @@ const updateAggregates = async (db: DatabaseSchema, postIdx: IndexedPost) => { replyCount: db .selectFrom('post') .where('post.replyParent', '=', postIdx.post.replyParent) + .where((qb) => + qb + .where('post.violatesThreadGate', 'is', null) + .orWhere('post.violatesThreadGate', '=', false), + ) .select(countAll.as('count')), }) .onConflict((oc) => diff --git a/packages/bsky/tests/views/threadgating.test.ts b/packages/bsky/tests/views/threadgating.test.ts index 53e5961b595..5f530b33536 100644 --- a/packages/bsky/tests/views/threadgating.test.ts +++ b/packages/bsky/tests/views/threadgating.test.ts @@ -64,6 +64,32 @@ describe('views with thread gating', () => { await checkReplyDisabled(post.ref.uriStr, sc.dids.alice, true) }) + it('does not generate notifications when post violates threadgate.', async () => { + const post = await sc.post(sc.dids.carol, 'notifications') + await pdsAgent.api.app.bsky.feed.threadgate.create( + { repo: sc.dids.carol, rkey: post.ref.uri.rkey }, + { post: post.ref.uriStr, createdAt: iso(), allow: [] }, + sc.getHeaders(sc.dids.carol), + ) + const reply = await sc.reply( + sc.dids.alice, + post.ref, + post.ref, + 'notifications reply', + ) + await network.processAll() + const { + data: { notifications }, + } = await agent.api.app.bsky.notification.listNotifications( + {}, + { headers: await network.serviceHeaders(sc.dids.carol) }, + ) + const notificationFromReply = notifications.find( + (notif) => notif.uri === reply.ref.uriStr, + ) + expect(notificationFromReply).toBeUndefined() + }) + it('applies gate for mention rule.', async () => { const post = await sc.post( sc.dids.carol, From 6d21cc1b01b00bb44b0f8e756529fa6dfb73d869 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Fri, 1 Dec 2023 17:29:26 -0600 Subject: [PATCH 56/59] Add flag for running db migrations on appview (#1913) * add flag for running db migrations on appview * lint * another fix --- services/bsky/api.js | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/services/bsky/api.js b/services/bsky/api.js index cf63c951043..2e78d3bafec 100644 --- a/services/bsky/api.js +++ b/services/bsky/api.js @@ -32,6 +32,18 @@ const { const main = async () => { const env = getEnv() assert(env.dbPrimaryPostgresUrl, 'missing configuration for db') + + if (env.enableMigrations) { + // separate db needed for more permissions + const migrateDb = new PrimaryDatabase({ + url: env.dbMigratePostgresUrl, + schema: env.dbPostgresSchema, + poolSize: 2, + }) + await migrateDb.migrateToLatestOrThrow() + await migrateDb.close() + } + const db = new DatabaseCoordinator({ schema: env.dbPostgresSchema, primary: { @@ -102,12 +114,12 @@ const main = async () => { algos, }) // separate db needed for more permissions - const migrateDb = new PrimaryDatabase({ + const viewMaintainerDb = new PrimaryDatabase({ url: env.dbMigratePostgresUrl, schema: env.dbPostgresSchema, poolSize: 2, }) - const viewMaintainer = new ViewMaintainer(migrateDb, 1800) + const viewMaintainer = new ViewMaintainer(viewMaintainerDb, 1800) const viewMaintainerRunning = viewMaintainer.run() const periodicModerationEventReversal = new PeriodicModerationEventReversal( @@ -125,11 +137,12 @@ const main = async () => { await bsky.destroy() viewMaintainer.destroy() await viewMaintainerRunning - await migrateDb.close() + await viewMaintainerDb.close() }) } const getEnv = () => ({ + enableMigrations: process.env.ENABLE_MIGRATIONS === 'true', port: parseInt(process.env.PORT), version: process.env.BSKY_VERSION, dbMigratePostgresUrl: From cad30a7cc8a2195cfa66ce50c782c385fae97b30 Mon Sep 17 00:00:00 2001 From: devin ivy Date: Fri, 1 Dec 2023 19:01:31 -0500 Subject: [PATCH 57/59] Cleanup outdated notifications in appview, add daemon for similar tasks (#1893) * initial notification tidy logic * helper for maintenance across all appview users * tiny reorg * add bsky daemon to tidy notifications * tidy, add bsky daemon service entrypoint * test notifs tidy daemon, add stats * tidy * crash failed notification daemon * fix notification tidy constants --- packages/bsky/src/daemon/config.ts | 50 +++++ packages/bsky/src/daemon/context.ts | 27 +++ packages/bsky/src/daemon/index.ts | 79 ++++++++ packages/bsky/src/daemon/logger.ts | 6 + packages/bsky/src/daemon/notifications.ts | 50 +++++ packages/bsky/src/daemon/services.ts | 20 ++ packages/bsky/src/index.ts | 1 + packages/bsky/src/services/actor/index.ts | 29 +++ .../bsky/src/services/util/notification.ts | 70 +++++++ packages/bsky/tests/daemon.test.ts | 191 ++++++++++++++++++ packages/bsky/tests/views/profile.test.ts | 1 - services/bsky/daemon.js | 44 ++++ 12 files changed, 567 insertions(+), 1 deletion(-) create mode 100644 packages/bsky/src/daemon/config.ts create mode 100644 packages/bsky/src/daemon/context.ts create mode 100644 packages/bsky/src/daemon/index.ts create mode 100644 packages/bsky/src/daemon/logger.ts create mode 100644 packages/bsky/src/daemon/notifications.ts create mode 100644 packages/bsky/src/daemon/services.ts create mode 100644 packages/bsky/src/services/util/notification.ts create mode 100644 packages/bsky/tests/daemon.test.ts create mode 100644 services/bsky/daemon.js diff --git a/packages/bsky/src/daemon/config.ts b/packages/bsky/src/daemon/config.ts new file mode 100644 index 00000000000..e0e789203e4 --- /dev/null +++ b/packages/bsky/src/daemon/config.ts @@ -0,0 +1,50 @@ +import assert from 'assert' + +export interface DaemonConfigValues { + version: string + dbPostgresUrl: string + dbPostgresSchema?: string +} + +export class DaemonConfig { + constructor(private cfg: DaemonConfigValues) {} + + static readEnv(overrides?: Partial) { + const version = process.env.BSKY_VERSION || '0.0.0' + const dbPostgresUrl = + overrides?.dbPostgresUrl || process.env.DB_PRIMARY_POSTGRES_URL + const dbPostgresSchema = + overrides?.dbPostgresSchema || process.env.DB_POSTGRES_SCHEMA + assert(dbPostgresUrl) + return new DaemonConfig({ + version, + dbPostgresUrl, + dbPostgresSchema, + ...stripUndefineds(overrides ?? {}), + }) + } + + get version() { + return this.cfg.version + } + + get dbPostgresUrl() { + return this.cfg.dbPostgresUrl + } + + get dbPostgresSchema() { + return this.cfg.dbPostgresSchema + } +} + +function stripUndefineds( + obj: Record, +): Record { + const result = {} + Object.entries(obj).forEach(([key, val]) => { + if (val !== undefined) { + result[key] = val + } + }) + return result +} diff --git a/packages/bsky/src/daemon/context.ts b/packages/bsky/src/daemon/context.ts new file mode 100644 index 00000000000..dd3d5c1114f --- /dev/null +++ b/packages/bsky/src/daemon/context.ts @@ -0,0 +1,27 @@ +import { PrimaryDatabase } from '../db' +import { DaemonConfig } from './config' +import { Services } from './services' + +export class DaemonContext { + constructor( + private opts: { + db: PrimaryDatabase + cfg: DaemonConfig + services: Services + }, + ) {} + + get db(): PrimaryDatabase { + return this.opts.db + } + + get cfg(): DaemonConfig { + return this.opts.cfg + } + + get services(): Services { + return this.opts.services + } +} + +export default DaemonContext diff --git a/packages/bsky/src/daemon/index.ts b/packages/bsky/src/daemon/index.ts new file mode 100644 index 00000000000..61bcd8568f4 --- /dev/null +++ b/packages/bsky/src/daemon/index.ts @@ -0,0 +1,79 @@ +import { PrimaryDatabase } from '../db' +import { dbLogger } from '../logger' +import { DaemonConfig } from './config' +import { DaemonContext } from './context' +import { createServices } from './services' +import { ImageUriBuilder } from '../image/uri' +import { LabelCache } from '../label-cache' +import { NotificationsDaemon } from './notifications' +import logger from './logger' + +export { DaemonConfig } from './config' +export type { DaemonConfigValues } from './config' + +export class BskyDaemon { + public ctx: DaemonContext + public notifications: NotificationsDaemon + private dbStatsInterval: NodeJS.Timer + private notifStatsInterval: NodeJS.Timer + + constructor(opts: { + ctx: DaemonContext + notifications: NotificationsDaemon + }) { + this.ctx = opts.ctx + this.notifications = opts.notifications + } + + static create(opts: { db: PrimaryDatabase; cfg: DaemonConfig }): BskyDaemon { + const { db, cfg } = opts + const imgUriBuilder = new ImageUriBuilder('https://daemon.invalid') // will not be used by daemon + const labelCache = new LabelCache(db) + const services = createServices({ + imgUriBuilder, + labelCache, + }) + const ctx = new DaemonContext({ + db, + cfg, + services, + }) + const notifications = new NotificationsDaemon(ctx) + return new BskyDaemon({ ctx, notifications }) + } + + async start() { + const { db } = this.ctx + const pool = db.pool + this.notifications.run() + this.dbStatsInterval = setInterval(() => { + dbLogger.info( + { + idleCount: pool.idleCount, + totalCount: pool.totalCount, + waitingCount: pool.waitingCount, + }, + 'db pool stats', + ) + }, 10000) + this.notifStatsInterval = setInterval(() => { + logger.info( + { + count: this.notifications.count, + lastDid: this.notifications.lastDid, + }, + 'notifications daemon stats', + ) + }, 10000) + return this + } + + async destroy(): Promise { + await this.notifications.destroy() + await this.ctx.db.close() + clearInterval(this.dbStatsInterval) + clearInterval(this.notifStatsInterval) + } +} + +export default BskyDaemon diff --git a/packages/bsky/src/daemon/logger.ts b/packages/bsky/src/daemon/logger.ts new file mode 100644 index 00000000000..8599acc315e --- /dev/null +++ b/packages/bsky/src/daemon/logger.ts @@ -0,0 +1,6 @@ +import { subsystemLogger } from '@atproto/common' + +const logger: ReturnType = + subsystemLogger('bsky:daemon') + +export default logger diff --git a/packages/bsky/src/daemon/notifications.ts b/packages/bsky/src/daemon/notifications.ts new file mode 100644 index 00000000000..e8e884b37c2 --- /dev/null +++ b/packages/bsky/src/daemon/notifications.ts @@ -0,0 +1,50 @@ +import { tidyNotifications } from '../services/util/notification' +import DaemonContext from './context' +import logger from './logger' + +export class NotificationsDaemon { + ac = new AbortController() + running: Promise | undefined + count = 0 + lastDid: string | null = null + + constructor(private ctx: DaemonContext) {} + + run(opts?: RunOptions) { + if (this.running) return + this.count = 0 + this.lastDid = null + this.ac = new AbortController() + this.running = this.tidyNotifications({ + ...opts, + forever: opts?.forever !== false, // run forever by default + }) + .catch((err) => { + // allow this to cause an unhandled rejection, let deployment handle the crash. + logger.error({ err }, 'notifications daemon crashed') + throw err + }) + .finally(() => (this.running = undefined)) + } + + private async tidyNotifications(opts: RunOptions) { + const actorService = this.ctx.services.actor(this.ctx.db) + for await (const { did } of actorService.all(opts)) { + if (this.ac.signal.aborted) return + try { + await tidyNotifications(this.ctx.db, did) + this.count++ + this.lastDid = did + } catch (err) { + logger.warn({ err, did }, 'failed to tidy notifications for actor') + } + } + } + + async destroy() { + this.ac.abort() + await this.running + } +} + +type RunOptions = { forever?: boolean; batchSize?: number } diff --git a/packages/bsky/src/daemon/services.ts b/packages/bsky/src/daemon/services.ts new file mode 100644 index 00000000000..a4e7935523c --- /dev/null +++ b/packages/bsky/src/daemon/services.ts @@ -0,0 +1,20 @@ +import { PrimaryDatabase } from '../db' +import { ActorService } from '../services/actor' +import { ImageUriBuilder } from '../image/uri' +import { LabelCache } from '../label-cache' + +export function createServices(resources: { + imgUriBuilder: ImageUriBuilder + labelCache: LabelCache +}): Services { + const { imgUriBuilder, labelCache } = resources + return { + actor: ActorService.creator(imgUriBuilder, labelCache), + } +} + +export type Services = { + actor: FromDbPrimary +} + +type FromDbPrimary = (db: PrimaryDatabase) => T diff --git a/packages/bsky/src/index.ts b/packages/bsky/src/index.ts index 9e0075dce37..7ceba61f990 100644 --- a/packages/bsky/src/index.ts +++ b/packages/bsky/src/index.ts @@ -37,6 +37,7 @@ export { Redis } from './redis' export { ViewMaintainer } from './db/views' export { AppContext } from './context' export { makeAlgos } from './feed-gen' +export * from './daemon' export * from './indexer' export * from './ingester' export { MigrateModerationData } from './migrate-moderation-data' diff --git a/packages/bsky/src/services/actor/index.ts b/packages/bsky/src/services/actor/index.ts index a2f980ce71d..51be90892fc 100644 --- a/packages/bsky/src/services/actor/index.ts +++ b/packages/bsky/src/services/actor/index.ts @@ -1,4 +1,5 @@ import { sql } from 'kysely' +import { wait } from '@atproto/common' import { Database } from '../../db' import { notSoftDeletedClause } from '../../db/util' import { ActorViews } from './views' @@ -144,6 +145,34 @@ export class ActorService { .executeTakeFirst() return res?.repoRev ?? null } + + async *all( + opts: { batchSize?: number; forever?: boolean; cooldownMs?: number } = {}, + ) { + const { cooldownMs = 1000, batchSize = 1000, forever = false } = opts + const baseQuery = this.db.db + .selectFrom('actor') + .selectAll() + .orderBy('did') + .limit(batchSize) + while (true) { + let cursor: ActorResult | undefined + do { + const actors = cursor + ? await baseQuery.where('did', '>', cursor.did).execute() + : await baseQuery.execute() + for (const actor of actors) { + yield actor + } + cursor = actors.at(-1) + } while (cursor) + if (forever) { + await wait(cooldownMs) + } else { + return + } + } + } } type ActorResult = Actor diff --git a/packages/bsky/src/services/util/notification.ts b/packages/bsky/src/services/util/notification.ts new file mode 100644 index 00000000000..811e6e41713 --- /dev/null +++ b/packages/bsky/src/services/util/notification.ts @@ -0,0 +1,70 @@ +import { sql } from 'kysely' +import { countAll } from '../../db/util' +import { PrimaryDatabase } from '../../db' + +// i.e. 30 days before the last time the user checked their notifs +export const BEFORE_LAST_SEEN_DAYS = 30 +// i.e. 180 days before the latest unread notification +export const BEFORE_LATEST_UNREAD_DAYS = 180 +// don't consider culling unreads until they hit this threshold, and then enforce beforeLatestUnreadThresholdDays +export const UNREAD_KEPT_COUNT = 500 + +export const tidyNotifications = async (db: PrimaryDatabase, did: string) => { + const stats = await db.db + .selectFrom('notification') + .select([ + sql<0 | 1>`("sortAt" < "lastSeenNotifs")`.as('read'), + countAll.as('count'), + sql`min("sortAt")`.as('earliestAt'), + sql`max("sortAt")`.as('latestAt'), + sql`max("lastSeenNotifs")`.as('lastSeenAt'), + ]) + .leftJoin('actor_state', 'actor_state.did', 'notification.did') + .where('notification.did', '=', did) + .groupBy(sql`1`) // group by read (i.e. 1st column) + .execute() + const readStats = stats.find((stat) => stat.read) + const unreadStats = stats.find((stat) => !stat.read) + let readCutoffAt: Date | undefined + let unreadCutoffAt: Date | undefined + if (readStats) { + readCutoffAt = addDays( + new Date(readStats.lastSeenAt), + -BEFORE_LAST_SEEN_DAYS, + ) + } + if (unreadStats && unreadStats.count > UNREAD_KEPT_COUNT) { + unreadCutoffAt = addDays( + new Date(unreadStats.latestAt), + -BEFORE_LATEST_UNREAD_DAYS, + ) + } + // take most recent of read/unread cutoffs + const cutoffAt = greatest(readCutoffAt, unreadCutoffAt) + if (cutoffAt) { + // skip delete if it wont catch any notifications + const earliestAt = least(readStats?.earliestAt, unreadStats?.earliestAt) + if (earliestAt && earliestAt < cutoffAt.toISOString()) { + await db.db + .deleteFrom('notification') + .where('did', '=', did) + .where('sortAt', '<', cutoffAt.toISOString()) + .execute() + } + } +} + +const addDays = (date: Date, days: number) => { + date.setDate(date.getDate() + days) + return date +} + +const least = (a: T | undefined, b: T | undefined) => { + return a !== undefined && (b === undefined || a < b) ? a : b +} + +const greatest = (a: T | undefined, b: T | undefined) => { + return a !== undefined && (b === undefined || a > b) ? a : b +} + +type Ordered = string | number | Date diff --git a/packages/bsky/tests/daemon.test.ts b/packages/bsky/tests/daemon.test.ts new file mode 100644 index 00000000000..32f0d6617ab --- /dev/null +++ b/packages/bsky/tests/daemon.test.ts @@ -0,0 +1,191 @@ +import assert from 'assert' +import { AtUri } from '@atproto/api' +import { TestNetwork } from '@atproto/dev-env' +import { BskyDaemon, DaemonConfig, PrimaryDatabase } from '../src' +import usersSeed from './seeds/users' +import { countAll, excluded } from '../src/db/util' +import { NotificationsDaemon } from '../src/daemon/notifications' +import { + BEFORE_LAST_SEEN_DAYS, + BEFORE_LATEST_UNREAD_DAYS, + UNREAD_KEPT_COUNT, +} from '../src/services/util/notification' + +describe('daemon', () => { + let network: TestNetwork + let daemon: BskyDaemon + let db: PrimaryDatabase + let actors: { did: string }[] = [] + + beforeAll(async () => { + network = await TestNetwork.create({ + dbPostgresSchema: 'bsky_daemon', + }) + db = network.bsky.ctx.db.getPrimary() + daemon = BskyDaemon.create({ + db, + cfg: new DaemonConfig({ + version: network.bsky.ctx.cfg.version, + dbPostgresUrl: network.bsky.ctx.cfg.dbPrimaryPostgresUrl, + dbPostgresSchema: network.bsky.ctx.cfg.dbPostgresSchema, + }), + }) + const sc = network.getSeedClient() + await usersSeed(sc) + await network.processAll() + actors = await db.db.selectFrom('actor').selectAll().execute() + }) + + afterAll(async () => { + await network.close() + }) + + describe('notifications daemon', () => { + it('processes all dids', async () => { + for (const { did } of actors) { + await Promise.all([ + setLastSeen(daemon.ctx.db, { did }), + createNotifications(daemon.ctx.db, { + did, + daysAgo: 2 * BEFORE_LAST_SEEN_DAYS, + count: 1, + }), + ]) + } + await expect(countNotifications(db)).resolves.toBe(actors.length) + await runNotifsOnce(daemon.notifications) + await expect(countNotifications(db)).resolves.toBe(0) + }) + + it('removes read notifications older than threshold.', async () => { + const { did } = actors[0] + const lastSeenDaysAgo = 10 + await Promise.all([ + setLastSeen(daemon.ctx.db, { did, daysAgo: lastSeenDaysAgo }), + // read, delete + createNotifications(daemon.ctx.db, { + did, + daysAgo: lastSeenDaysAgo + BEFORE_LAST_SEEN_DAYS + 1, + count: 2, + }), + // read, keep + createNotifications(daemon.ctx.db, { + did, + daysAgo: lastSeenDaysAgo + BEFORE_LAST_SEEN_DAYS - 1, + count: 3, + }), + // unread, keep + createNotifications(daemon.ctx.db, { + did, + daysAgo: lastSeenDaysAgo - 1, + count: 4, + }), + ]) + await expect(countNotifications(db)).resolves.toBe(9) + await runNotifsOnce(daemon.notifications) + await expect(countNotifications(db)).resolves.toBe(7) + await clearNotifications(db) + }) + + it('removes unread notifications older than threshold.', async () => { + const { did } = actors[0] + await Promise.all([ + setLastSeen(daemon.ctx.db, { + did, + daysAgo: 2 * BEFORE_LATEST_UNREAD_DAYS, // all are unread + }), + createNotifications(daemon.ctx.db, { + did, + daysAgo: 0, + count: 1, + }), + createNotifications(daemon.ctx.db, { + did, + daysAgo: BEFORE_LATEST_UNREAD_DAYS - 1, + count: 99, + }), + createNotifications(daemon.ctx.db, { + did, + daysAgo: BEFORE_LATEST_UNREAD_DAYS + 1, + count: 400, + }), + ]) + await expect(countNotifications(db)).resolves.toBe(UNREAD_KEPT_COUNT) + await runNotifsOnce(daemon.notifications) + // none removed when within UNREAD_KEPT_COUNT + await expect(countNotifications(db)).resolves.toBe(UNREAD_KEPT_COUNT) + // add one more, tip over UNREAD_KEPT_COUNT + await createNotifications(daemon.ctx.db, { + did, + daysAgo: BEFORE_LATEST_UNREAD_DAYS + 1, + count: 1, + }) + await runNotifsOnce(daemon.notifications) + // removed all older than BEFORE_LATEST_UNREAD_DAYS + await expect(countNotifications(db)).resolves.toBe(100) + await clearNotifications(db) + }) + }) + + const runNotifsOnce = async (notifsDaemon: NotificationsDaemon) => { + assert(!notifsDaemon.running, 'notifications daemon is already running') + notifsDaemon.run({ forever: false, batchSize: 2 }) + await notifsDaemon.running + } + + const setLastSeen = async ( + db: PrimaryDatabase, + opts: { did: string; daysAgo?: number }, + ) => { + const { did, daysAgo = 0 } = opts + const lastSeenAt = new Date() + lastSeenAt.setDate(lastSeenAt.getDate() - daysAgo) + await db.db + .insertInto('actor_state') + .values({ did, lastSeenNotifs: lastSeenAt.toISOString() }) + .onConflict((oc) => + oc.column('did').doUpdateSet({ + lastSeenNotifs: excluded(db.db, 'lastSeenNotifs'), + }), + ) + .execute() + } + + const createNotifications = async ( + db: PrimaryDatabase, + opts: { + did: string + count: number + daysAgo: number + }, + ) => { + const { did, count, daysAgo } = opts + const sortAt = new Date() + sortAt.setDate(sortAt.getDate() - daysAgo) + await db.db + .insertInto('notification') + .values( + [...Array(count)].map(() => ({ + did, + author: did, + reason: 'none', + recordCid: 'bafycid', + recordUri: AtUri.make(did, 'invalid.collection', 'self').toString(), + sortAt: sortAt.toISOString(), + })), + ) + .execute() + } + + const clearNotifications = async (db: PrimaryDatabase) => { + await db.db.deleteFrom('notification').execute() + } + + const countNotifications = async (db: PrimaryDatabase) => { + const { count } = await db.db + .selectFrom('notification') + .select(countAll.as('count')) + .executeTakeFirstOrThrow() + return count + } +}) diff --git a/packages/bsky/tests/views/profile.test.ts b/packages/bsky/tests/views/profile.test.ts index d4e0c718bed..726fb990a0d 100644 --- a/packages/bsky/tests/views/profile.test.ts +++ b/packages/bsky/tests/views/profile.test.ts @@ -25,7 +25,6 @@ describe('pds profile views', () => { sc = network.getSeedClient() await basicSeed(sc) await network.processAll() - await network.bsky.processAll() alice = sc.dids.alice bob = sc.dids.bob dan = sc.dids.dan diff --git a/services/bsky/daemon.js b/services/bsky/daemon.js new file mode 100644 index 00000000000..bd8322ab58f --- /dev/null +++ b/services/bsky/daemon.js @@ -0,0 +1,44 @@ +'use strict' /* eslint-disable */ + +require('dd-trace/init') // Only works with commonjs + +// Tracer code above must come before anything else +const { PrimaryDatabase, DaemonConfig, BskyDaemon } = require('@atproto/bsky') + +const main = async () => { + const env = getEnv() + const db = new PrimaryDatabase({ + url: env.dbPostgresUrl, + schema: env.dbPostgresSchema, + poolSize: env.dbPoolSize, + poolMaxUses: env.dbPoolMaxUses, + poolIdleTimeoutMs: env.dbPoolIdleTimeoutMs, + }) + const cfg = DaemonConfig.readEnv({ + version: env.version, + dbPostgresUrl: env.dbPostgresUrl, + dbPostgresSchema: env.dbPostgresSchema, + }) + const daemon = BskyDaemon.create({ db, cfg }) + await daemon.start() + process.on('SIGTERM', async () => { + await daemon.destroy() + }) +} + +const getEnv = () => ({ + version: process.env.BSKY_VERSION, + dbPostgresUrl: + process.env.DB_PRIMARY_POSTGRES_URL || process.env.DB_POSTGRES_URL, + dbPostgresSchema: process.env.DB_POSTGRES_SCHEMA || undefined, + dbPoolSize: maybeParseInt(process.env.DB_POOL_SIZE), + dbPoolMaxUses: maybeParseInt(process.env.DB_POOL_MAX_USES), + dbPoolIdleTimeoutMs: maybeParseInt(process.env.DB_POOL_IDLE_TIMEOUT_MS), +}) + +const maybeParseInt = (str) => { + const parsed = parseInt(str) + return isNaN(parsed) ? undefined : parsed +} + +main() From f9fd3e68ca7b498988d5a5d2099e2bfa96e7fa62 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Mon, 4 Dec 2023 18:00:09 -0600 Subject: [PATCH 58/59] Feature branch: PDS v2 (#1789) * cleanup repeat process all * wip * skip actor search test * skip actor search test * tweak processAll * decrease wait to 1 sec * repo_blob -> record_blob * simplify backlink linkTo * return repo_root to one row * sequence before updating repo_root * invite code forUser -> forAccount * ipld_block -> repo_block * use lru-cache fetchMethod * move did_cache to own db * better error handling on did cache * drop did_handle * fix sequencer wait time * debug * debug * more debug * check something * fix bday paradox * fix bday paradox * tidy up pds service auth * rm skipped test * retry http * tidy * improve fanout error handling * fix test * return signing key in did-web * more tests * tidy serivce auth checks * user_account -> account * remove inviteNote * keypair per repo * use an lru cache for keypairs as well * clean up repo * wip * wrap up accoutn manager * tidy * tidy * fix tests * fix disabled codes * fix appview tests * add note * set pragmas * tidy account manager getDb * rename pref transactor * user pref -> account pref * handle blob imports * tidy imports * add reserveSigningKey * wip transferAccount * clean up transferAccount * tests * tidy * tidy * configure entryway url on pds * handle entryway in pds admin endpoints * make importRepo temp * fix imports * make email optional on pds when using entryway * handle diffs * handle pds entryway usage for server, identity, admin endpoints * pds support for credentials from entryway * setup pds tests w/ entryway service * tidy * tidy * update entryway version * wip * test handle updates w/ entryway * split account table into two * tidy * tweak scripts * tidy tests * tidy * better config for actorstore & dbs * clean up cfg more * reorg actorstore fs layout * handle erros on actor db create * pr tidy & fix accoutn deletion test * pr feedback * fix bad merge * unskip test * fix subscribe repos tests * tidy repo root tables * tidy * fix tests * tidy delete tokens * tidy account getters * tidy * bulk deletesg * increase chunk size * handle racing refreshes * wip * fix auth test * invert import flow * clean up actor store on create account failure * tweak sequencer * prevent invite code races on createAccount * rm note * add back in race protection on getAccountInviteCodes * start feature branch * deleted app migration table * patch up new auth test * rm note * g * create accoutn delegated from entryway * tidy * fix test * change plcOp type to unknown * small fixes * sync up w entryway branch * Use proper error when authed account is not found (#1799) provide proper error when account not found in access-takedown check * build branch * build on ghcr * tweak service file * tweak service file * change where we save reserved keys * no tmp dir in blobstore either * fix blobstore temp location again * handle repeat record_blobs * create account before submitting plc op & undo if fail * small tweak * limit the number of local records * push out empty commit on transfer * fix issue with record_blob * add push blob endpoint * Set and validate token audiences on pds v2 (#1793) set and validate token audience on pds v2 * merge * include entryway did on tests * build branch * fix cache issue * xrpc server blob limit * put correct bytes * add auth to routes * handle quarantining/unquarantining a blob that does not exist * tidy * fix transfer tests * fix email request routes for entryway * PDS v2 entryway account deletion (#1819) * add admin lexicon for account deletion * implement admin account deletion endpoint * fix entryway proxying on account email checks * proxy to entryway for acct deletion * read-after-write sanity check * tweak * wip * finish refactor * fix test schema * application retry logic for busy * pr feedback * rm lru-cache * fix test pg schema * fix transfer test * Sqlite instrumentation for pds v2 (#1838) * sqlite instrumentation * build * remove build * dont reimport blobs * send ticks during import * close on error * catch handle validation error * add log * fix test * return emailConfirmedAt on getAccountInfo * Upgrade sharp on pds v2 (#1863) upgrade sharp to 0.32.6 * read all bytes before parsing car * Async car reader (#1867) * asynchronously read in car * dont buffer car * tweak * Gracefully handle indexing of invalid records (#1853) * gracefully handle indexing of invalid records * fix repo tests * Fix role auth for access-or-role verifier, getBlob check on actor takedowns (#1869) fix role auth for access-or-role verifier, fix getBlob actor takedown check * better cleanup of actor-stores * add ability to not ensure leaves * tidy * allow did:web transfer * Migration utility for actor-store (#1873) beginnings of helper for migrating all actors Co-authored-by: Devin Ivy * base case for findBlobRefs * App-level retries for sqlite on pds (#1871) * revamp retry helper to be more flexible re: backoff strategies * sqlite timeout helper * ensure sqlite wal on db creation/migration rather than every open * layer retries for sqlite on writes outside transactions on pds * tidy * fix up lockfile * tidy * fix lex codegen * fix timing bug in threadgate test * No-op update handling (#1916) do no produce commits on no-op updates * Retry on all SQLITE_BUSY error codes (#1917) retry on all sqlite_busy error codes * Pds v2 ensure sqlite ready (#1918) ensure sqlite is ready before making queries * try something * tidy * dont build branch --------- Co-authored-by: Devin Ivy --- lexicons/com/atproto/admin/deleteAccount.json | 20 + lexicons/com/atproto/temp/importRepo.json | 27 + lexicons/com/atproto/temp/pushBlob.json | 24 + .../com/atproto/temp/transferAccount.json | 44 ++ packages/api/src/client/index.ts | 52 ++ packages/api/src/client/lexicons.ts | 156 ++++ .../types/com/atproto/admin/deleteAccount.ts | 32 + .../types/com/atproto/temp/importRepo.ts | 33 + .../client/types/com/atproto/temp/pushBlob.ts | 32 + .../types/com/atproto/temp/transferAccount.ts | 92 +++ packages/aws/src/s3.ts | 66 +- packages/bsky/src/lexicon/index.ts | 50 ++ packages/bsky/src/lexicon/lexicons.ts | 156 ++++ .../types/com/atproto/admin/deleteAccount.ts | 38 + .../types/com/atproto/temp/importRepo.ts | 45 ++ .../types/com/atproto/temp/pushBlob.ts | 39 + .../types/com/atproto/temp/transferAccount.ts | 62 ++ packages/bsky/src/services/indexing/index.ts | 17 +- packages/bsky/src/util/retry.ts | 45 +- packages/bsky/tests/admin/get-repo.test.ts | 8 +- packages/bsky/tests/algos/hot-classic.test.ts | 3 +- packages/bsky/tests/auth.test.ts | 2 +- .../bsky/tests/auto-moderator/labeler.test.ts | 39 +- .../tests/auto-moderator/takedowns.test.ts | 26 +- packages/bsky/tests/blob-resolver.test.ts | 6 +- .../bsky/tests/handle-invalidation.test.ts | 6 +- packages/bsky/tests/indexing.test.ts | 33 +- packages/bsky/tests/seeds/basic.ts | 3 + packages/bsky/tests/subscription/repo.test.ts | 11 +- .../bsky/tests/views/threadgating.test.ts | 12 + packages/common-web/src/async.ts | 27 + packages/common-web/src/index.ts | 1 + packages/common-web/src/retry.ts | 52 ++ packages/common-web/tests/retry.test.ts | 93 +++ packages/common/src/fs.ts | 27 + packages/crypto/package.json | 3 + packages/crypto/src/sha.ts | 7 + packages/crypto/src/types.ts | 4 + packages/crypto/tests/signatures.test.ts | 2 +- packages/dev-env/src/bin.ts | 3 +- packages/dev-env/src/bsky.ts | 17 +- packages/dev-env/src/const.ts | 1 + packages/dev-env/src/network-no-appview.ts | 9 - packages/dev-env/src/network.ts | 25 +- packages/dev-env/src/pds.ts | 37 +- packages/lex-cli/src/codegen/server.ts | 6 + packages/pds/bench/sequencer.bench.ts | 139 ---- packages/pds/package.json | 11 +- packages/pds/src/account-manager/db/index.ts | 21 + .../account-manager/db/migrations/001-init.ts | 115 +++ .../account-manager/db/migrations/index.ts | 5 + .../src/account-manager/db/schema/account.ts | 15 + .../src/account-manager/db/schema/actor.ts | 14 + .../db/schema}/app-password.ts | 0 .../db/schema}/email-token.ts | 2 +- .../src/account-manager/db/schema/index.ts | 23 + .../db/schema}/invite-code.ts | 2 +- .../db/schema}/refresh-token.ts | 0 .../db/schema}/repo-root.ts | 6 +- .../src/account-manager/helpers/account.ts | 210 ++++++ .../pds/src/account-manager/helpers/auth.ts | 184 +++++ .../account-manager/helpers/email-token.ts | 80 +++ .../pds/src/account-manager/helpers/invite.ts | 259 +++++++ .../src/account-manager/helpers/password.ts | 109 +++ .../pds/src/account-manager/helpers/repo.ts | 24 + .../{db => account-manager/helpers}/scrypt.ts | 0 packages/pds/src/account-manager/index.ts | 353 +++++++++ packages/pds/src/actor-store/blob/reader.ts | 76 ++ .../blob/transactor.ts} | 173 ++--- packages/pds/src/actor-store/db/index.ts | 20 + .../src/actor-store/db/migrations/001-init.ts | 105 +++ .../src/actor-store/db/migrations/index.ts | 5 + .../src/actor-store/db/schema/account-pref.ts | 11 + .../db/schema}/backlink.ts | 3 +- .../tables => actor-store/db/schema}/blob.ts | 2 +- .../pds/src/actor-store/db/schema/index.ts | 23 + .../src/actor-store/db/schema/record-blob.ts | 8 + .../db/schema}/record.ts | 3 +- .../src/actor-store/db/schema/repo-block.ts | 10 + .../src/actor-store/db/schema/repo-root.ts | 10 + packages/pds/src/actor-store/index.ts | 261 +++++++ packages/pds/src/actor-store/migrate.ts | 39 + .../pds/src/actor-store/preference/reader.ts | 22 + .../src/actor-store/preference/transactor.ts | 44 ++ .../index.ts => actor-store/record/reader.ts} | 179 ++--- .../pds/src/actor-store/record/transactor.ts | 106 +++ packages/pds/src/actor-store/repo/reader.ts | 17 + .../src/actor-store/repo/sql-repo-reader.ts | 149 ++++ .../actor-store/repo/sql-repo-transactor.ts | 107 +++ .../pds/src/actor-store/repo/transactor.ts | 181 +++++ .../src/api/app/bsky/actor/getPreferences.ts | 7 +- .../pds/src/api/app/bsky/actor/getProfile.ts | 15 +- .../pds/src/api/app/bsky/actor/getProfiles.ts | 13 +- .../src/api/app/bsky/actor/putPreferences.ts | 15 +- .../src/api/app/bsky/feed/getActorLikes.ts | 14 +- .../src/api/app/bsky/feed/getAuthorFeed.ts | 16 +- .../src/api/app/bsky/feed/getPostThread.ts | 56 +- .../pds/src/api/app/bsky/feed/getTimeline.ts | 16 +- .../src/api/app/bsky/graph/getFollowers.ts | 2 +- .../pds/src/api/app/bsky/graph/getFollows.ts | 2 +- .../api/com/atproto/admin/deleteAccount.ts | 19 + .../atproto/admin/disableAccountInvites.ts | 15 +- .../com/atproto/admin/disableInviteCodes.ts | 20 +- .../com/atproto/admin/emitModerationEvent.ts | 2 +- .../com/atproto/admin/enableAccountInvites.ts | 15 +- .../api/com/atproto/admin/getAccountInfo.ts | 23 +- .../api/com/atproto/admin/getInviteCodes.ts | 15 +- .../com/atproto/admin/getModerationEvent.ts | 2 +- .../src/api/com/atproto/admin/getRecord.ts | 2 +- .../pds/src/api/com/atproto/admin/getRepo.ts | 2 +- .../api/com/atproto/admin/getSubjectStatus.ts | 47 +- .../pds/src/api/com/atproto/admin/index.ts | 2 + .../atproto/admin/queryModerationEvents.ts | 2 +- .../atproto/admin/queryModerationStatuses.ts | 2 +- .../src/api/com/atproto/admin/searchRepos.ts | 4 +- .../src/api/com/atproto/admin/sendEmail.ts | 26 +- .../com/atproto/admin/updateAccountEmail.ts | 30 +- .../com/atproto/admin/updateAccountHandle.ts | 45 +- .../com/atproto/admin/updateSubjectStatus.ts | 20 +- .../api/com/atproto/identity/resolveHandle.ts | 2 +- .../api/com/atproto/identity/updateHandle.ts | 54 +- packages/pds/src/api/com/atproto/index.ts | 2 + .../src/api/com/atproto/repo/applyWrites.ts | 28 +- .../src/api/com/atproto/repo/createRecord.ts | 105 +-- .../src/api/com/atproto/repo/deleteRecord.ts | 47 +- .../src/api/com/atproto/repo/describeRepo.ts | 11 +- .../pds/src/api/com/atproto/repo/getRecord.ts | 8 +- .../src/api/com/atproto/repo/listRecords.ts | 21 +- .../pds/src/api/com/atproto/repo/putRecord.ts | 90 ++- .../src/api/com/atproto/repo/uploadBlob.ts | 7 +- .../api/com/atproto/server/confirmEmail.ts | 29 +- .../api/com/atproto/server/createAccount.ts | 367 +++++----- .../com/atproto/server/createAppPassword.ts | 19 +- .../com/atproto/server/createInviteCode.ts | 26 +- .../com/atproto/server/createInviteCodes.ts | 34 +- .../api/com/atproto/server/createSession.ts | 35 +- .../api/com/atproto/server/deleteAccount.ts | 65 +- .../api/com/atproto/server/deleteSession.ts | 15 +- .../atproto/server/getAccountInviteCodes.ts | 86 +-- .../src/api/com/atproto/server/getSession.ts | 19 +- .../pds/src/api/com/atproto/server/index.ts | 2 + .../com/atproto/server/listAppPasswords.ts | 18 +- .../api/com/atproto/server/refreshSession.ts | 27 +- .../atproto/server/requestAccountDelete.ts | 30 +- .../server/requestEmailConfirmation.ts | 30 +- .../com/atproto/server/requestEmailUpdate.ts | 33 +- .../atproto/server/requestPasswordReset.ts | 32 +- .../com/atproto/server/reserveSigningKey.ts | 16 + .../api/com/atproto/server/resetPassword.ts | 22 +- .../com/atproto/server/revokeAppPassword.ts | 19 +- .../src/api/com/atproto/server/updateEmail.ts | 61 +- .../atproto/sync/deprecated/getCheckout.ts | 22 +- .../com/atproto/sync/deprecated/getHead.ts | 10 +- .../pds/src/api/com/atproto/sync/getBlob.ts | 47 +- .../pds/src/api/com/atproto/sync/getBlocks.ts | 10 +- .../api/com/atproto/sync/getLatestCommit.ts | 10 +- .../pds/src/api/com/atproto/sync/getRecord.ts | 38 +- .../pds/src/api/com/atproto/sync/getRepo.ts | 49 +- .../pds/src/api/com/atproto/sync/listBlobs.ts | 27 +- .../pds/src/api/com/atproto/sync/listRepos.ts | 22 +- .../api/com/atproto/sync/subscribeRepos.ts | 2 +- .../src/api/com/atproto/temp/importRepo.ts | 243 +++++++ .../pds/src/api/com/atproto/temp/index.ts | 11 + .../pds/src/api/com/atproto/temp/pushBlob.ts | 23 + .../api/com/atproto/temp/transferAccount.ts | 118 +++ packages/pds/src/api/proxy.ts | 33 + packages/pds/src/auth-verifier.ts | 149 ++-- packages/pds/src/background.ts | 7 +- packages/pds/src/basic-routes.ts | 5 +- packages/pds/src/config/config.ts | 102 +-- packages/pds/src/config/env.ts | 65 +- packages/pds/src/config/secrets.ts | 19 - packages/pds/src/context.ts | 164 +++-- packages/pds/src/crawlers.ts | 39 +- packages/pds/src/db/database-schema.ts | 42 -- packages/pds/src/db/db.ts | 137 ++++ packages/pds/src/db/index.ts | 401 +---------- packages/pds/src/db/leader.ts | 176 ----- .../db/migrations/20230613T164932261Z-init.ts | 375 ---------- .../migrations/20230914T014727199Z-repo-v3.ts | 165 ----- .../20230926T195532354Z-email-tokens.ts | 69 -- .../20230929T213219699Z-takedown-id-as-int.ts | 47 -- .../20231011T155513453Z-takedown-ref.ts | 41 -- packages/pds/src/db/migrations/index.ts | 9 - packages/pds/src/db/migrations/provider.ts | 25 - packages/pds/src/db/migrator.ts | 36 + packages/pds/src/db/tables/app-migration.ts | 9 - packages/pds/src/db/tables/did-cache.ts | 11 - packages/pds/src/db/tables/did-handle.ts | 9 - packages/pds/src/db/tables/ipld-block.ts | 11 - packages/pds/src/db/tables/repo-blob.ts | 11 - packages/pds/src/db/tables/repo-seq.ts | 24 - packages/pds/src/db/tables/runtime-flag.ts | 8 - packages/pds/src/db/tables/user-account.ts | 17 - packages/pds/src/db/tables/user-pref.ts | 12 - packages/pds/src/db/types.ts | 3 - packages/pds/src/db/util.ts | 57 +- packages/pds/src/did-cache.ts | 107 --- packages/pds/src/did-cache/db/index.ts | 21 + packages/pds/src/did-cache/db/migrations.ts | 17 + packages/pds/src/did-cache/db/schema.ts | 9 + packages/pds/src/did-cache/index.ts | 131 ++++ .../pds/src/{storage => }/disk-blobstore.ts | 106 +-- packages/pds/src/index.ts | 43 +- packages/pds/src/lexicon/index.ts | 50 ++ packages/pds/src/lexicon/lexicons.ts | 156 ++++ .../types/com/atproto/admin/deleteAccount.ts | 38 + .../types/com/atproto/temp/importRepo.ts | 45 ++ .../types/com/atproto/temp/pushBlob.ts | 39 + .../types/com/atproto/temp/transferAccount.ts | 62 ++ packages/pds/src/logger.ts | 7 +- packages/pds/src/mailer/index.ts | 2 +- .../src/mailer/templates/reset-password.hbs | 2 +- packages/pds/src/read-after-write/index.ts | 3 + packages/pds/src/read-after-write/types.ts | 36 + .../util.ts} | 38 +- .../index.ts => read-after-write/viewer.ts} | 217 +++--- packages/pds/src/runtime-flags.ts | 51 -- packages/pds/src/sequencer/db/index.ts | 21 + .../src/sequencer/db/migrations/001-init.ts | 35 + .../pds/src/sequencer/db/migrations/index.ts | 5 + packages/pds/src/sequencer/db/schema.ts | 24 + packages/pds/src/sequencer/events.ts | 52 +- packages/pds/src/sequencer/index.ts | 1 - .../pds/src/sequencer/sequencer-leader.ts | 150 ---- packages/pds/src/sequencer/sequencer.ts | 133 +++- packages/pds/src/services/account/index.ts | 673 ------------------ packages/pds/src/services/auth.ts | 175 ----- packages/pds/src/services/index.ts | 66 -- packages/pds/src/services/moderation/index.ts | 125 ---- packages/pds/src/services/repo/index.ts | 305 -------- packages/pds/src/sql-repo-storage.ts | 286 -------- packages/pds/src/storage/index.ts | 2 - packages/pds/src/storage/memory-blobstore.ts | 96 --- packages/pds/src/well-known.ts | 2 +- packages/pds/tests/account-deletion.test.ts | 204 ++++-- packages/pds/tests/account.test.ts | 204 +++--- packages/pds/tests/app-passwords.test.ts | 6 +- packages/pds/tests/auth.test.ts | 58 +- packages/pds/tests/blob-deletes.test.ts | 36 +- packages/pds/tests/crud.test.ts | 31 +- packages/pds/tests/db-notify.test.ts | 138 ---- packages/pds/tests/db.test.ts | 461 +++--------- packages/pds/tests/entryway.test.ts | 172 +++++ packages/pds/tests/file-uploads.test.ts | 156 ++-- packages/pds/tests/handles.test.ts | 8 +- packages/pds/tests/invite-codes.test.ts | 33 +- packages/pds/tests/invites-admin.test.ts | 16 +- packages/pds/tests/moderation.test.ts | 69 +- packages/pds/tests/preferences.test.ts | 24 +- .../proxied/__snapshots__/admin.test.ts.snap | 4 +- packages/pds/tests/proxied/admin.test.ts | 14 - packages/pds/tests/proxied/notif.test.ts | 5 +- packages/pds/tests/races.test.ts | 68 +- packages/pds/tests/seeds/basic.ts | 3 + packages/pds/tests/seeds/users.ts | 2 +- packages/pds/tests/sequencer.test.ts | 13 +- packages/pds/tests/server.test.ts | 17 +- packages/pds/tests/sql-repo-storage.test.ts | 124 ---- .../pds/tests/sync/subscribe-repos.test.ts | 106 +-- packages/pds/tests/sync/sync.test.ts | 19 +- packages/pds/tests/transfer-repo.test.ts | 217 ++++++ packages/repo/src/index.ts | 1 + packages/repo/src/readable-repo.ts | 14 + packages/repo/src/repo.ts | 29 + .../repo/src/storage/memory-blockstore.ts | 4 +- packages/repo/src/storage/types.ts | 4 +- packages/repo/src/sync/consumer.ts | 12 +- packages/repo/src/sync/provider.ts | 4 +- packages/repo/src/types.ts | 18 +- packages/repo/src/util.ts | 32 +- packages/repo/tests/sync.test.ts | 4 +- packages/xrpc-server/src/server.ts | 11 +- packages/xrpc-server/src/types.ts | 5 + packages/xrpc-server/src/util.ts | 6 +- pnpm-lock.yaml | 463 ++++++++---- services/pds/index.js | 29 +- services/pds/package.json | 4 +- 278 files changed, 8597 insertions(+), 6920 deletions(-) create mode 100644 lexicons/com/atproto/admin/deleteAccount.json create mode 100644 lexicons/com/atproto/temp/importRepo.json create mode 100644 lexicons/com/atproto/temp/pushBlob.json create mode 100644 lexicons/com/atproto/temp/transferAccount.json create mode 100644 packages/api/src/client/types/com/atproto/admin/deleteAccount.ts create mode 100644 packages/api/src/client/types/com/atproto/temp/importRepo.ts create mode 100644 packages/api/src/client/types/com/atproto/temp/pushBlob.ts create mode 100644 packages/api/src/client/types/com/atproto/temp/transferAccount.ts create mode 100644 packages/bsky/src/lexicon/types/com/atproto/admin/deleteAccount.ts create mode 100644 packages/bsky/src/lexicon/types/com/atproto/temp/importRepo.ts create mode 100644 packages/bsky/src/lexicon/types/com/atproto/temp/pushBlob.ts create mode 100644 packages/bsky/src/lexicon/types/com/atproto/temp/transferAccount.ts create mode 100644 packages/common-web/src/retry.ts create mode 100644 packages/common-web/tests/retry.test.ts delete mode 100644 packages/pds/bench/sequencer.bench.ts create mode 100644 packages/pds/src/account-manager/db/index.ts create mode 100644 packages/pds/src/account-manager/db/migrations/001-init.ts create mode 100644 packages/pds/src/account-manager/db/migrations/index.ts create mode 100644 packages/pds/src/account-manager/db/schema/account.ts create mode 100644 packages/pds/src/account-manager/db/schema/actor.ts rename packages/pds/src/{db/tables => account-manager/db/schema}/app-password.ts (100%) rename packages/pds/src/{db/tables => account-manager/db/schema}/email-token.ts (93%) create mode 100644 packages/pds/src/account-manager/db/schema/index.ts rename packages/pds/src/{db/tables => account-manager/db/schema}/invite-code.ts (95%) rename packages/pds/src/{db/tables => account-manager/db/schema}/refresh-token.ts (100%) rename packages/pds/src/{db/tables => account-manager/db/schema}/repo-root.ts (58%) create mode 100644 packages/pds/src/account-manager/helpers/account.ts create mode 100644 packages/pds/src/account-manager/helpers/auth.ts create mode 100644 packages/pds/src/account-manager/helpers/email-token.ts create mode 100644 packages/pds/src/account-manager/helpers/invite.ts create mode 100644 packages/pds/src/account-manager/helpers/password.ts create mode 100644 packages/pds/src/account-manager/helpers/repo.ts rename packages/pds/src/{db => account-manager/helpers}/scrypt.ts (100%) create mode 100644 packages/pds/src/account-manager/index.ts create mode 100644 packages/pds/src/actor-store/blob/reader.ts rename packages/pds/src/{services/repo/blobs.ts => actor-store/blob/transactor.ts} (59%) create mode 100644 packages/pds/src/actor-store/db/index.ts create mode 100644 packages/pds/src/actor-store/db/migrations/001-init.ts create mode 100644 packages/pds/src/actor-store/db/migrations/index.ts create mode 100644 packages/pds/src/actor-store/db/schema/account-pref.ts rename packages/pds/src/{db/tables => actor-store/db/schema}/backlink.ts (73%) rename packages/pds/src/{db/tables => actor-store/db/schema}/blob.ts (89%) create mode 100644 packages/pds/src/actor-store/db/schema/index.ts create mode 100644 packages/pds/src/actor-store/db/schema/record-blob.ts rename packages/pds/src/{db/tables => actor-store/db/schema}/record.ts (87%) create mode 100644 packages/pds/src/actor-store/db/schema/repo-block.ts create mode 100644 packages/pds/src/actor-store/db/schema/repo-root.ts create mode 100644 packages/pds/src/actor-store/index.ts create mode 100644 packages/pds/src/actor-store/migrate.ts create mode 100644 packages/pds/src/actor-store/preference/reader.ts create mode 100644 packages/pds/src/actor-store/preference/transactor.ts rename packages/pds/src/{services/record/index.ts => actor-store/record/reader.ts} (53%) create mode 100644 packages/pds/src/actor-store/record/transactor.ts create mode 100644 packages/pds/src/actor-store/repo/reader.ts create mode 100644 packages/pds/src/actor-store/repo/sql-repo-reader.ts create mode 100644 packages/pds/src/actor-store/repo/sql-repo-transactor.ts create mode 100644 packages/pds/src/actor-store/repo/transactor.ts create mode 100644 packages/pds/src/api/com/atproto/admin/deleteAccount.ts create mode 100644 packages/pds/src/api/com/atproto/server/reserveSigningKey.ts create mode 100644 packages/pds/src/api/com/atproto/temp/importRepo.ts create mode 100644 packages/pds/src/api/com/atproto/temp/index.ts create mode 100644 packages/pds/src/api/com/atproto/temp/pushBlob.ts create mode 100644 packages/pds/src/api/com/atproto/temp/transferAccount.ts create mode 100644 packages/pds/src/api/proxy.ts delete mode 100644 packages/pds/src/db/database-schema.ts create mode 100644 packages/pds/src/db/db.ts delete mode 100644 packages/pds/src/db/leader.ts delete mode 100644 packages/pds/src/db/migrations/20230613T164932261Z-init.ts delete mode 100644 packages/pds/src/db/migrations/20230914T014727199Z-repo-v3.ts delete mode 100644 packages/pds/src/db/migrations/20230926T195532354Z-email-tokens.ts delete mode 100644 packages/pds/src/db/migrations/20230929T213219699Z-takedown-id-as-int.ts delete mode 100644 packages/pds/src/db/migrations/20231011T155513453Z-takedown-ref.ts delete mode 100644 packages/pds/src/db/migrations/index.ts delete mode 100644 packages/pds/src/db/migrations/provider.ts create mode 100644 packages/pds/src/db/migrator.ts delete mode 100644 packages/pds/src/db/tables/app-migration.ts delete mode 100644 packages/pds/src/db/tables/did-cache.ts delete mode 100644 packages/pds/src/db/tables/did-handle.ts delete mode 100644 packages/pds/src/db/tables/ipld-block.ts delete mode 100644 packages/pds/src/db/tables/repo-blob.ts delete mode 100644 packages/pds/src/db/tables/repo-seq.ts delete mode 100644 packages/pds/src/db/tables/runtime-flag.ts delete mode 100644 packages/pds/src/db/tables/user-account.ts delete mode 100644 packages/pds/src/db/tables/user-pref.ts delete mode 100644 packages/pds/src/db/types.ts delete mode 100644 packages/pds/src/did-cache.ts create mode 100644 packages/pds/src/did-cache/db/index.ts create mode 100644 packages/pds/src/did-cache/db/migrations.ts create mode 100644 packages/pds/src/did-cache/db/schema.ts create mode 100644 packages/pds/src/did-cache/index.ts rename packages/pds/src/{storage => }/disk-blobstore.ts (53%) create mode 100644 packages/pds/src/lexicon/types/com/atproto/admin/deleteAccount.ts create mode 100644 packages/pds/src/lexicon/types/com/atproto/temp/importRepo.ts create mode 100644 packages/pds/src/lexicon/types/com/atproto/temp/pushBlob.ts create mode 100644 packages/pds/src/lexicon/types/com/atproto/temp/transferAccount.ts create mode 100644 packages/pds/src/read-after-write/index.ts create mode 100644 packages/pds/src/read-after-write/types.ts rename packages/pds/src/{api/app/bsky/util/read-after-write.ts => read-after-write/util.ts} (67%) rename packages/pds/src/{services/local/index.ts => read-after-write/viewer.ts} (60%) delete mode 100644 packages/pds/src/runtime-flags.ts create mode 100644 packages/pds/src/sequencer/db/index.ts create mode 100644 packages/pds/src/sequencer/db/migrations/001-init.ts create mode 100644 packages/pds/src/sequencer/db/migrations/index.ts create mode 100644 packages/pds/src/sequencer/db/schema.ts delete mode 100644 packages/pds/src/sequencer/sequencer-leader.ts delete mode 100644 packages/pds/src/services/account/index.ts delete mode 100644 packages/pds/src/services/auth.ts delete mode 100644 packages/pds/src/services/index.ts delete mode 100644 packages/pds/src/services/moderation/index.ts delete mode 100644 packages/pds/src/services/repo/index.ts delete mode 100644 packages/pds/src/sql-repo-storage.ts delete mode 100644 packages/pds/src/storage/index.ts delete mode 100644 packages/pds/src/storage/memory-blobstore.ts delete mode 100644 packages/pds/tests/db-notify.test.ts create mode 100644 packages/pds/tests/entryway.test.ts delete mode 100644 packages/pds/tests/sql-repo-storage.test.ts create mode 100644 packages/pds/tests/transfer-repo.test.ts diff --git a/lexicons/com/atproto/admin/deleteAccount.json b/lexicons/com/atproto/admin/deleteAccount.json new file mode 100644 index 00000000000..bd7532cef61 --- /dev/null +++ b/lexicons/com/atproto/admin/deleteAccount.json @@ -0,0 +1,20 @@ +{ + "lexicon": 1, + "id": "com.atproto.admin.deleteAccount", + "defs": { + "main": { + "type": "procedure", + "description": "Delete a user account as an administrator.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["did"], + "properties": { + "did": { "type": "string", "format": "did" } + } + } + } + } + } +} diff --git a/lexicons/com/atproto/temp/importRepo.json b/lexicons/com/atproto/temp/importRepo.json new file mode 100644 index 00000000000..f06daa09d73 --- /dev/null +++ b/lexicons/com/atproto/temp/importRepo.json @@ -0,0 +1,27 @@ +{ + "lexicon": 1, + "id": "com.atproto.temp.importRepo", + "defs": { + "main": { + "type": "procedure", + "description": "Gets the did's repo, optionally catching up from a specific revision.", + "parameters": { + "type": "params", + "required": ["did"], + "properties": { + "did": { + "type": "string", + "format": "did", + "description": "The DID of the repo." + } + } + }, + "input": { + "encoding": "application/vnd.ipld.car" + }, + "output": { + "encoding": "text/plain" + } + } + } +} diff --git a/lexicons/com/atproto/temp/pushBlob.json b/lexicons/com/atproto/temp/pushBlob.json new file mode 100644 index 00000000000..9babc8f8e43 --- /dev/null +++ b/lexicons/com/atproto/temp/pushBlob.json @@ -0,0 +1,24 @@ +{ + "lexicon": 1, + "id": "com.atproto.temp.pushBlob", + "defs": { + "main": { + "type": "procedure", + "description": "Gets the did's repo, optionally catching up from a specific revision.", + "parameters": { + "type": "params", + "required": ["did"], + "properties": { + "did": { + "type": "string", + "format": "did", + "description": "The DID of the repo." + } + } + }, + "input": { + "encoding": "*/*" + } + } + } +} diff --git a/lexicons/com/atproto/temp/transferAccount.json b/lexicons/com/atproto/temp/transferAccount.json new file mode 100644 index 00000000000..3cb2035ac0e --- /dev/null +++ b/lexicons/com/atproto/temp/transferAccount.json @@ -0,0 +1,44 @@ +{ + "lexicon": 1, + "id": "com.atproto.temp.transferAccount", + "defs": { + "main": { + "type": "procedure", + "description": "Transfer an account.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["handle", "did", "plcOp"], + "properties": { + "handle": { "type": "string", "format": "handle" }, + "did": { "type": "string", "format": "did" }, + "plcOp": { "type": "unknown" } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["accessJwt", "refreshJwt", "handle", "did"], + "properties": { + "accessJwt": { "type": "string" }, + "refreshJwt": { "type": "string" }, + "handle": { "type": "string", "format": "handle" }, + "did": { "type": "string", "format": "did" } + } + } + }, + "errors": [ + { "name": "InvalidHandle" }, + { "name": "InvalidPassword" }, + { "name": "InvalidInviteCode" }, + { "name": "HandleNotAvailable" }, + { "name": "UnsupportedDomain" }, + { "name": "UnresolvableDid" }, + { "name": "IncompatibleDidDoc" } + ] + } + } +} diff --git a/packages/api/src/client/index.ts b/packages/api/src/client/index.ts index b295fb88b71..a42dbd9320a 100644 --- a/packages/api/src/client/index.ts +++ b/packages/api/src/client/index.ts @@ -8,6 +8,7 @@ import { import { schemas } from './lexicons' import { CID } from 'multiformats/cid' import * as ComAtprotoAdminDefs from './types/com/atproto/admin/defs' +import * as ComAtprotoAdminDeleteAccount from './types/com/atproto/admin/deleteAccount' import * as ComAtprotoAdminDisableAccountInvites from './types/com/atproto/admin/disableAccountInvites' import * as ComAtprotoAdminDisableInviteCodes from './types/com/atproto/admin/disableInviteCodes' import * as ComAtprotoAdminEmitModerationEvent from './types/com/atproto/admin/emitModerationEvent' @@ -76,6 +77,9 @@ import * as ComAtprotoSyncNotifyOfUpdate from './types/com/atproto/sync/notifyOf import * as ComAtprotoSyncRequestCrawl from './types/com/atproto/sync/requestCrawl' import * as ComAtprotoSyncSubscribeRepos from './types/com/atproto/sync/subscribeRepos' import * as ComAtprotoTempFetchLabels from './types/com/atproto/temp/fetchLabels' +import * as ComAtprotoTempImportRepo from './types/com/atproto/temp/importRepo' +import * as ComAtprotoTempPushBlob from './types/com/atproto/temp/pushBlob' +import * as ComAtprotoTempTransferAccount from './types/com/atproto/temp/transferAccount' import * as AppBskyActorDefs from './types/app/bsky/actor/defs' import * as AppBskyActorGetPreferences from './types/app/bsky/actor/getPreferences' import * as AppBskyActorGetProfile from './types/app/bsky/actor/getProfile' @@ -143,6 +147,7 @@ import * as AppBskyUnspeccedSearchActorsSkeleton from './types/app/bsky/unspecce import * as AppBskyUnspeccedSearchPostsSkeleton from './types/app/bsky/unspecced/searchPostsSkeleton' export * as ComAtprotoAdminDefs from './types/com/atproto/admin/defs' +export * as ComAtprotoAdminDeleteAccount from './types/com/atproto/admin/deleteAccount' export * as ComAtprotoAdminDisableAccountInvites from './types/com/atproto/admin/disableAccountInvites' export * as ComAtprotoAdminDisableInviteCodes from './types/com/atproto/admin/disableInviteCodes' export * as ComAtprotoAdminEmitModerationEvent from './types/com/atproto/admin/emitModerationEvent' @@ -211,6 +216,9 @@ export * as ComAtprotoSyncNotifyOfUpdate from './types/com/atproto/sync/notifyOf export * as ComAtprotoSyncRequestCrawl from './types/com/atproto/sync/requestCrawl' export * as ComAtprotoSyncSubscribeRepos from './types/com/atproto/sync/subscribeRepos' export * as ComAtprotoTempFetchLabels from './types/com/atproto/temp/fetchLabels' +export * as ComAtprotoTempImportRepo from './types/com/atproto/temp/importRepo' +export * as ComAtprotoTempPushBlob from './types/com/atproto/temp/pushBlob' +export * as ComAtprotoTempTransferAccount from './types/com/atproto/temp/transferAccount' export * as AppBskyActorDefs from './types/app/bsky/actor/defs' export * as AppBskyActorGetPreferences from './types/app/bsky/actor/getPreferences' export * as AppBskyActorGetProfile from './types/app/bsky/actor/getProfile' @@ -366,6 +374,17 @@ export class AdminNS { this._service = service } + deleteAccount( + data?: ComAtprotoAdminDeleteAccount.InputSchema, + opts?: ComAtprotoAdminDeleteAccount.CallOptions, + ): Promise { + return this._service.xrpc + .call('com.atproto.admin.deleteAccount', opts?.qp, data, opts) + .catch((e) => { + throw ComAtprotoAdminDeleteAccount.toKnownErr(e) + }) + } + disableAccountInvites( data?: ComAtprotoAdminDisableAccountInvites.InputSchema, opts?: ComAtprotoAdminDisableAccountInvites.CallOptions, @@ -1108,6 +1127,39 @@ export class TempNS { throw ComAtprotoTempFetchLabels.toKnownErr(e) }) } + + importRepo( + data?: ComAtprotoTempImportRepo.InputSchema, + opts?: ComAtprotoTempImportRepo.CallOptions, + ): Promise { + return this._service.xrpc + .call('com.atproto.temp.importRepo', opts?.qp, data, opts) + .catch((e) => { + throw ComAtprotoTempImportRepo.toKnownErr(e) + }) + } + + pushBlob( + data?: ComAtprotoTempPushBlob.InputSchema, + opts?: ComAtprotoTempPushBlob.CallOptions, + ): Promise { + return this._service.xrpc + .call('com.atproto.temp.pushBlob', opts?.qp, data, opts) + .catch((e) => { + throw ComAtprotoTempPushBlob.toKnownErr(e) + }) + } + + transferAccount( + data?: ComAtprotoTempTransferAccount.InputSchema, + opts?: ComAtprotoTempTransferAccount.CallOptions, + ): Promise { + return this._service.xrpc + .call('com.atproto.temp.transferAccount', opts?.qp, data, opts) + .catch((e) => { + throw ComAtprotoTempTransferAccount.toKnownErr(e) + }) + } } export class AppNS { diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index a6377d85a9a..90176ef6486 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -820,6 +820,29 @@ export const schemaDict = { }, }, }, + ComAtprotoAdminDeleteAccount: { + lexicon: 1, + id: 'com.atproto.admin.deleteAccount', + defs: { + main: { + type: 'procedure', + description: 'Delete a user account as an administrator.', + input: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['did'], + properties: { + did: { + type: 'string', + format: 'did', + }, + }, + }, + }, + }, + }, + }, ComAtprotoAdminDisableAccountInvites: { lexicon: 1, id: 'com.atproto.admin.disableAccountInvites', @@ -3979,6 +4002,135 @@ export const schemaDict = { }, }, }, + ComAtprotoTempImportRepo: { + lexicon: 1, + id: 'com.atproto.temp.importRepo', + defs: { + main: { + type: 'procedure', + description: + "Gets the did's repo, optionally catching up from a specific revision.", + parameters: { + type: 'params', + required: ['did'], + properties: { + did: { + type: 'string', + format: 'did', + description: 'The DID of the repo.', + }, + }, + }, + input: { + encoding: 'application/vnd.ipld.car', + }, + output: { + encoding: 'text/plain', + }, + }, + }, + }, + ComAtprotoTempPushBlob: { + lexicon: 1, + id: 'com.atproto.temp.pushBlob', + defs: { + main: { + type: 'procedure', + description: + "Gets the did's repo, optionally catching up from a specific revision.", + parameters: { + type: 'params', + required: ['did'], + properties: { + did: { + type: 'string', + format: 'did', + description: 'The DID of the repo.', + }, + }, + }, + input: { + encoding: '*/*', + }, + }, + }, + }, + ComAtprotoTempTransferAccount: { + lexicon: 1, + id: 'com.atproto.temp.transferAccount', + defs: { + main: { + type: 'procedure', + description: 'Transfer an account.', + input: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['handle', 'did', 'plcOp'], + properties: { + handle: { + type: 'string', + format: 'handle', + }, + did: { + type: 'string', + format: 'did', + }, + plcOp: { + type: 'unknown', + }, + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['accessJwt', 'refreshJwt', 'handle', 'did'], + properties: { + accessJwt: { + type: 'string', + }, + refreshJwt: { + type: 'string', + }, + handle: { + type: 'string', + format: 'handle', + }, + did: { + type: 'string', + format: 'did', + }, + }, + }, + }, + errors: [ + { + name: 'InvalidHandle', + }, + { + name: 'InvalidPassword', + }, + { + name: 'InvalidInviteCode', + }, + { + name: 'HandleNotAvailable', + }, + { + name: 'UnsupportedDomain', + }, + { + name: 'UnresolvableDid', + }, + { + name: 'IncompatibleDidDoc', + }, + ], + }, + }, + }, AppBskyActorDefs: { lexicon: 1, id: 'app.bsky.actor.defs', @@ -7671,6 +7823,7 @@ export const schemas: LexiconDoc[] = Object.values(schemaDict) as LexiconDoc[] export const lexicons: Lexicons = new Lexicons(schemas) export const ids = { ComAtprotoAdminDefs: 'com.atproto.admin.defs', + ComAtprotoAdminDeleteAccount: 'com.atproto.admin.deleteAccount', ComAtprotoAdminDisableAccountInvites: 'com.atproto.admin.disableAccountInvites', ComAtprotoAdminDisableInviteCodes: 'com.atproto.admin.disableInviteCodes', @@ -7746,6 +7899,9 @@ export const ids = { ComAtprotoSyncRequestCrawl: 'com.atproto.sync.requestCrawl', ComAtprotoSyncSubscribeRepos: 'com.atproto.sync.subscribeRepos', ComAtprotoTempFetchLabels: 'com.atproto.temp.fetchLabels', + ComAtprotoTempImportRepo: 'com.atproto.temp.importRepo', + ComAtprotoTempPushBlob: 'com.atproto.temp.pushBlob', + ComAtprotoTempTransferAccount: 'com.atproto.temp.transferAccount', AppBskyActorDefs: 'app.bsky.actor.defs', AppBskyActorGetPreferences: 'app.bsky.actor.getPreferences', AppBskyActorGetProfile: 'app.bsky.actor.getProfile', diff --git a/packages/api/src/client/types/com/atproto/admin/deleteAccount.ts b/packages/api/src/client/types/com/atproto/admin/deleteAccount.ts new file mode 100644 index 00000000000..b8b5aa511b8 --- /dev/null +++ b/packages/api/src/client/types/com/atproto/admin/deleteAccount.ts @@ -0,0 +1,32 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import { Headers, XRPCError } from '@atproto/xrpc' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { isObj, hasProp } from '../../../../util' +import { lexicons } from '../../../../lexicons' +import { CID } from 'multiformats/cid' + +export interface QueryParams {} + +export interface InputSchema { + did: string + [k: string]: unknown +} + +export interface CallOptions { + headers?: Headers + qp?: QueryParams + encoding: 'application/json' +} + +export interface Response { + success: boolean + headers: Headers +} + +export function toKnownErr(e: any) { + if (e instanceof XRPCError) { + } + return e +} diff --git a/packages/api/src/client/types/com/atproto/temp/importRepo.ts b/packages/api/src/client/types/com/atproto/temp/importRepo.ts new file mode 100644 index 00000000000..6f9f99f2b9d --- /dev/null +++ b/packages/api/src/client/types/com/atproto/temp/importRepo.ts @@ -0,0 +1,33 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import { Headers, XRPCError } from '@atproto/xrpc' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { isObj, hasProp } from '../../../../util' +import { lexicons } from '../../../../lexicons' +import { CID } from 'multiformats/cid' + +export interface QueryParams { + /** The DID of the repo. */ + did: string +} + +export type InputSchema = string | Uint8Array + +export interface CallOptions { + headers?: Headers + qp?: QueryParams + encoding: 'application/vnd.ipld.car' +} + +export interface Response { + success: boolean + headers: Headers + data: Uint8Array +} + +export function toKnownErr(e: any) { + if (e instanceof XRPCError) { + } + return e +} diff --git a/packages/api/src/client/types/com/atproto/temp/pushBlob.ts b/packages/api/src/client/types/com/atproto/temp/pushBlob.ts new file mode 100644 index 00000000000..32165bc8014 --- /dev/null +++ b/packages/api/src/client/types/com/atproto/temp/pushBlob.ts @@ -0,0 +1,32 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import { Headers, XRPCError } from '@atproto/xrpc' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { isObj, hasProp } from '../../../../util' +import { lexicons } from '../../../../lexicons' +import { CID } from 'multiformats/cid' + +export interface QueryParams { + /** The DID of the repo. */ + did: string +} + +export type InputSchema = string | Uint8Array + +export interface CallOptions { + headers?: Headers + qp?: QueryParams + encoding: string +} + +export interface Response { + success: boolean + headers: Headers +} + +export function toKnownErr(e: any) { + if (e instanceof XRPCError) { + } + return e +} diff --git a/packages/api/src/client/types/com/atproto/temp/transferAccount.ts b/packages/api/src/client/types/com/atproto/temp/transferAccount.ts new file mode 100644 index 00000000000..7ae16c01290 --- /dev/null +++ b/packages/api/src/client/types/com/atproto/temp/transferAccount.ts @@ -0,0 +1,92 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import { Headers, XRPCError } from '@atproto/xrpc' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { isObj, hasProp } from '../../../../util' +import { lexicons } from '../../../../lexicons' +import { CID } from 'multiformats/cid' + +export interface QueryParams {} + +export interface InputSchema { + handle: string + did: string + plcOp: {} + [k: string]: unknown +} + +export interface OutputSchema { + accessJwt: string + refreshJwt: string + handle: string + did: string + [k: string]: unknown +} + +export interface CallOptions { + headers?: Headers + qp?: QueryParams + encoding: 'application/json' +} + +export interface Response { + success: boolean + headers: Headers + data: OutputSchema +} + +export class InvalidHandleError extends XRPCError { + constructor(src: XRPCError) { + super(src.status, src.error, src.message, src.headers) + } +} + +export class InvalidPasswordError extends XRPCError { + constructor(src: XRPCError) { + super(src.status, src.error, src.message, src.headers) + } +} + +export class InvalidInviteCodeError extends XRPCError { + constructor(src: XRPCError) { + super(src.status, src.error, src.message, src.headers) + } +} + +export class HandleNotAvailableError extends XRPCError { + constructor(src: XRPCError) { + super(src.status, src.error, src.message, src.headers) + } +} + +export class UnsupportedDomainError extends XRPCError { + constructor(src: XRPCError) { + super(src.status, src.error, src.message, src.headers) + } +} + +export class UnresolvableDidError extends XRPCError { + constructor(src: XRPCError) { + super(src.status, src.error, src.message, src.headers) + } +} + +export class IncompatibleDidDocError extends XRPCError { + constructor(src: XRPCError) { + super(src.status, src.error, src.message, src.headers) + } +} + +export function toKnownErr(e: any) { + if (e instanceof XRPCError) { + if (e.error === 'InvalidHandle') return new InvalidHandleError(e) + if (e.error === 'InvalidPassword') return new InvalidPasswordError(e) + if (e.error === 'InvalidInviteCode') return new InvalidInviteCodeError(e) + if (e.error === 'HandleNotAvailable') return new HandleNotAvailableError(e) + if (e.error === 'UnsupportedDomain') return new UnsupportedDomainError(e) + if (e.error === 'UnresolvableDid') return new UnresolvableDidError(e) + if (e.error === 'IncompatibleDidDoc') return new IncompatibleDidDocError(e) + } + return e +} diff --git a/packages/aws/src/s3.ts b/packages/aws/src/s3.ts index 66c0cff45bf..fc27d7a71f5 100644 --- a/packages/aws/src/s3.ts +++ b/packages/aws/src/s3.ts @@ -17,7 +17,7 @@ export class S3BlobStore implements BlobStore { private client: aws.S3 private bucket: string - constructor(cfg: S3Config) { + constructor(public did: string, cfg: S3Config) { const { bucket, ...rest } = cfg this.bucket = bucket this.client = new aws.S3({ @@ -26,20 +26,26 @@ export class S3BlobStore implements BlobStore { }) } + static creator(cfg: S3Config) { + return (did: string) => { + return new S3BlobStore(did, cfg) + } + } + private genKey() { return randomStr(32, 'base32') } private getTmpPath(key: string): string { - return `tmp/${key}` + return `tmp/${this.did}/${key}` } private getStoredPath(cid: CID): string { - return `blocks/${cid.toString()}` + return `blocks/${this.did}/${cid.toString()}` } private getQuarantinedPath(cid: CID): string { - return `quarantine/${cid.toString()}` + return `quarantine/${this.did}/${cid.toString()}` } async putTemp(bytes: Uint8Array | stream.Readable): Promise { @@ -122,11 +128,24 @@ export class S3BlobStore implements BlobStore { await this.deleteKey(this.getStoredPath(cid)) } + async deleteMany(cids: CID[]): Promise { + const keys = cids.map((cid) => this.getStoredPath(cid)) + await this.deleteManyKeys(keys) + } + async hasStored(cid: CID): Promise { + return this.hasKey(this.getStoredPath(cid)) + } + + async hasTemp(key: string): Promise { + return this.hasKey(this.getTmpPath(key)) + } + + private async hasKey(key: string) { try { const res = await this.client.headObject({ Bucket: this.bucket, - Key: this.getStoredPath(cid), + Key: key, }) return res.$metadata.httpStatusCode === 200 } catch (err) { @@ -141,17 +160,38 @@ export class S3BlobStore implements BlobStore { }) } - private async move(keys: { from: string; to: string }) { - await this.client.copyObject({ - Bucket: this.bucket, - CopySource: `${this.bucket}/${keys.from}`, - Key: keys.to, - }) - await this.client.deleteObject({ + private async deleteManyKeys(keys: string[]) { + await this.client.deleteObjects({ Bucket: this.bucket, - Key: keys.from, + Delete: { + Objects: keys.map((k) => ({ Key: k })), + }, }) } + + private async move(keys: { from: string; to: string }) { + try { + await this.client.copyObject({ + Bucket: this.bucket, + CopySource: `${this.bucket}/${keys.from}`, + Key: keys.to, + }) + await this.client.deleteObject({ + Bucket: this.bucket, + Key: keys.from, + }) + } catch (err) { + handleErr(err) + } + } +} + +const handleErr = (err: unknown) => { + if (err?.['Code'] === 'NoSuchKey') { + throw new BlobNotFoundError() + } else { + throw err + } } export default S3BlobStore diff --git a/packages/bsky/src/lexicon/index.ts b/packages/bsky/src/lexicon/index.ts index e4a075bee9f..c51998a66e6 100644 --- a/packages/bsky/src/lexicon/index.ts +++ b/packages/bsky/src/lexicon/index.ts @@ -9,6 +9,7 @@ import { StreamAuthVerifier, } from '@atproto/xrpc-server' import { schemas } from './lexicons' +import * as ComAtprotoAdminDeleteAccount from './types/com/atproto/admin/deleteAccount' import * as ComAtprotoAdminDisableAccountInvites from './types/com/atproto/admin/disableAccountInvites' import * as ComAtprotoAdminDisableInviteCodes from './types/com/atproto/admin/disableInviteCodes' import * as ComAtprotoAdminEmitModerationEvent from './types/com/atproto/admin/emitModerationEvent' @@ -73,6 +74,9 @@ import * as ComAtprotoSyncNotifyOfUpdate from './types/com/atproto/sync/notifyOf import * as ComAtprotoSyncRequestCrawl from './types/com/atproto/sync/requestCrawl' import * as ComAtprotoSyncSubscribeRepos from './types/com/atproto/sync/subscribeRepos' import * as ComAtprotoTempFetchLabels from './types/com/atproto/temp/fetchLabels' +import * as ComAtprotoTempImportRepo from './types/com/atproto/temp/importRepo' +import * as ComAtprotoTempPushBlob from './types/com/atproto/temp/pushBlob' +import * as ComAtprotoTempTransferAccount from './types/com/atproto/temp/transferAccount' import * as AppBskyActorGetPreferences from './types/app/bsky/actor/getPreferences' import * as AppBskyActorGetProfile from './types/app/bsky/actor/getProfile' import * as AppBskyActorGetProfiles from './types/app/bsky/actor/getProfiles' @@ -194,6 +198,17 @@ export class AdminNS { this._server = server } + deleteAccount( + cfg: ConfigOf< + AV, + ComAtprotoAdminDeleteAccount.Handler>, + ComAtprotoAdminDeleteAccount.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.admin.deleteAccount' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } + disableAccountInvites( cfg: ConfigOf< AV, @@ -953,6 +968,39 @@ export class TempNS { const nsid = 'com.atproto.temp.fetchLabels' // @ts-ignore return this._server.xrpc.method(nsid, cfg) } + + importRepo( + cfg: ConfigOf< + AV, + ComAtprotoTempImportRepo.Handler>, + ComAtprotoTempImportRepo.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.temp.importRepo' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } + + pushBlob( + cfg: ConfigOf< + AV, + ComAtprotoTempPushBlob.Handler>, + ComAtprotoTempPushBlob.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.temp.pushBlob' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } + + transferAccount( + cfg: ConfigOf< + AV, + ComAtprotoTempTransferAccount.Handler>, + ComAtprotoTempTransferAccount.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.temp.transferAccount' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } } export class AppNS { @@ -1549,11 +1597,13 @@ type RouteRateLimitOpts = { calcKey?: (ctx: T) => string calcPoints?: (ctx: T) => number } +type HandlerOpts = { blobLimit?: number } type HandlerRateLimitOpts = SharedRateLimitOpts | RouteRateLimitOpts type ConfigOf = | Handler | { auth?: Auth + opts?: HandlerOpts rateLimit?: HandlerRateLimitOpts | HandlerRateLimitOpts[] handler: Handler } diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index a6377d85a9a..90176ef6486 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -820,6 +820,29 @@ export const schemaDict = { }, }, }, + ComAtprotoAdminDeleteAccount: { + lexicon: 1, + id: 'com.atproto.admin.deleteAccount', + defs: { + main: { + type: 'procedure', + description: 'Delete a user account as an administrator.', + input: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['did'], + properties: { + did: { + type: 'string', + format: 'did', + }, + }, + }, + }, + }, + }, + }, ComAtprotoAdminDisableAccountInvites: { lexicon: 1, id: 'com.atproto.admin.disableAccountInvites', @@ -3979,6 +4002,135 @@ export const schemaDict = { }, }, }, + ComAtprotoTempImportRepo: { + lexicon: 1, + id: 'com.atproto.temp.importRepo', + defs: { + main: { + type: 'procedure', + description: + "Gets the did's repo, optionally catching up from a specific revision.", + parameters: { + type: 'params', + required: ['did'], + properties: { + did: { + type: 'string', + format: 'did', + description: 'The DID of the repo.', + }, + }, + }, + input: { + encoding: 'application/vnd.ipld.car', + }, + output: { + encoding: 'text/plain', + }, + }, + }, + }, + ComAtprotoTempPushBlob: { + lexicon: 1, + id: 'com.atproto.temp.pushBlob', + defs: { + main: { + type: 'procedure', + description: + "Gets the did's repo, optionally catching up from a specific revision.", + parameters: { + type: 'params', + required: ['did'], + properties: { + did: { + type: 'string', + format: 'did', + description: 'The DID of the repo.', + }, + }, + }, + input: { + encoding: '*/*', + }, + }, + }, + }, + ComAtprotoTempTransferAccount: { + lexicon: 1, + id: 'com.atproto.temp.transferAccount', + defs: { + main: { + type: 'procedure', + description: 'Transfer an account.', + input: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['handle', 'did', 'plcOp'], + properties: { + handle: { + type: 'string', + format: 'handle', + }, + did: { + type: 'string', + format: 'did', + }, + plcOp: { + type: 'unknown', + }, + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['accessJwt', 'refreshJwt', 'handle', 'did'], + properties: { + accessJwt: { + type: 'string', + }, + refreshJwt: { + type: 'string', + }, + handle: { + type: 'string', + format: 'handle', + }, + did: { + type: 'string', + format: 'did', + }, + }, + }, + }, + errors: [ + { + name: 'InvalidHandle', + }, + { + name: 'InvalidPassword', + }, + { + name: 'InvalidInviteCode', + }, + { + name: 'HandleNotAvailable', + }, + { + name: 'UnsupportedDomain', + }, + { + name: 'UnresolvableDid', + }, + { + name: 'IncompatibleDidDoc', + }, + ], + }, + }, + }, AppBskyActorDefs: { lexicon: 1, id: 'app.bsky.actor.defs', @@ -7671,6 +7823,7 @@ export const schemas: LexiconDoc[] = Object.values(schemaDict) as LexiconDoc[] export const lexicons: Lexicons = new Lexicons(schemas) export const ids = { ComAtprotoAdminDefs: 'com.atproto.admin.defs', + ComAtprotoAdminDeleteAccount: 'com.atproto.admin.deleteAccount', ComAtprotoAdminDisableAccountInvites: 'com.atproto.admin.disableAccountInvites', ComAtprotoAdminDisableInviteCodes: 'com.atproto.admin.disableInviteCodes', @@ -7746,6 +7899,9 @@ export const ids = { ComAtprotoSyncRequestCrawl: 'com.atproto.sync.requestCrawl', ComAtprotoSyncSubscribeRepos: 'com.atproto.sync.subscribeRepos', ComAtprotoTempFetchLabels: 'com.atproto.temp.fetchLabels', + ComAtprotoTempImportRepo: 'com.atproto.temp.importRepo', + ComAtprotoTempPushBlob: 'com.atproto.temp.pushBlob', + ComAtprotoTempTransferAccount: 'com.atproto.temp.transferAccount', AppBskyActorDefs: 'app.bsky.actor.defs', AppBskyActorGetPreferences: 'app.bsky.actor.getPreferences', AppBskyActorGetProfile: 'app.bsky.actor.getProfile', diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/deleteAccount.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/deleteAccount.ts new file mode 100644 index 00000000000..13e68eb5c7d --- /dev/null +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/deleteAccount.ts @@ -0,0 +1,38 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' + +export interface QueryParams {} + +export interface InputSchema { + did: string + [k: string]: unknown +} + +export interface HandlerInput { + encoding: 'application/json' + body: InputSchema +} + +export interface HandlerError { + status: number + message?: string +} + +export type HandlerOutput = HandlerError | void +export type HandlerReqCtx = { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +} +export type Handler = ( + ctx: HandlerReqCtx, +) => Promise | HandlerOutput diff --git a/packages/bsky/src/lexicon/types/com/atproto/temp/importRepo.ts b/packages/bsky/src/lexicon/types/com/atproto/temp/importRepo.ts new file mode 100644 index 00000000000..d88361d9856 --- /dev/null +++ b/packages/bsky/src/lexicon/types/com/atproto/temp/importRepo.ts @@ -0,0 +1,45 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import stream from 'stream' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' + +export interface QueryParams { + /** The DID of the repo. */ + did: string +} + +export type InputSchema = string | Uint8Array + +export interface HandlerInput { + encoding: 'application/vnd.ipld.car' + body: stream.Readable +} + +export interface HandlerSuccess { + encoding: 'text/plain' + body: Uint8Array | stream.Readable + headers?: { [key: string]: string } +} + +export interface HandlerError { + status: number + message?: string +} + +export type HandlerOutput = HandlerError | HandlerSuccess +export type HandlerReqCtx = { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +} +export type Handler = ( + ctx: HandlerReqCtx, +) => Promise | HandlerOutput diff --git a/packages/bsky/src/lexicon/types/com/atproto/temp/pushBlob.ts b/packages/bsky/src/lexicon/types/com/atproto/temp/pushBlob.ts new file mode 100644 index 00000000000..97e890dbb14 --- /dev/null +++ b/packages/bsky/src/lexicon/types/com/atproto/temp/pushBlob.ts @@ -0,0 +1,39 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import stream from 'stream' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' + +export interface QueryParams { + /** The DID of the repo. */ + did: string +} + +export type InputSchema = string | Uint8Array + +export interface HandlerInput { + encoding: '*/*' + body: stream.Readable +} + +export interface HandlerError { + status: number + message?: string +} + +export type HandlerOutput = HandlerError | void +export type HandlerReqCtx = { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +} +export type Handler = ( + ctx: HandlerReqCtx, +) => Promise | HandlerOutput diff --git a/packages/bsky/src/lexicon/types/com/atproto/temp/transferAccount.ts b/packages/bsky/src/lexicon/types/com/atproto/temp/transferAccount.ts new file mode 100644 index 00000000000..86c1d750e07 --- /dev/null +++ b/packages/bsky/src/lexicon/types/com/atproto/temp/transferAccount.ts @@ -0,0 +1,62 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' + +export interface QueryParams {} + +export interface InputSchema { + handle: string + did: string + plcOp: {} + [k: string]: unknown +} + +export interface OutputSchema { + accessJwt: string + refreshJwt: string + handle: string + did: string + [k: string]: unknown +} + +export interface HandlerInput { + encoding: 'application/json' + body: InputSchema +} + +export interface HandlerSuccess { + encoding: 'application/json' + body: OutputSchema + headers?: { [key: string]: string } +} + +export interface HandlerError { + status: number + message?: string + error?: + | 'InvalidHandle' + | 'InvalidPassword' + | 'InvalidInviteCode' + | 'HandleNotAvailable' + | 'UnsupportedDomain' + | 'UnresolvableDid' + | 'IncompatibleDidDoc' +} + +export type HandlerOutput = HandlerError | HandlerSuccess +export type HandlerReqCtx = { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +} +export type Handler = ( + ctx: HandlerReqCtx, +) => Promise | HandlerOutput diff --git a/packages/bsky/src/services/indexing/index.ts b/packages/bsky/src/services/indexing/index.ts index 60c465dc0fc..44dd9c3c986 100644 --- a/packages/bsky/src/services/indexing/index.ts +++ b/packages/bsky/src/services/indexing/index.ts @@ -7,6 +7,7 @@ import { verifyRepo, Commit, VerifiedRepo, + getAndParseRecord, } from '@atproto/repo' import { AtUri } from '@atproto/syntax' import { IdResolver, getPds } from '@atproto/identity' @@ -201,10 +202,11 @@ export class IndexingService { if (op.op === 'delete') { await this.deleteRecord(uri) } else { + const parsed = await getAndParseRecord(blocks, cid) await this.indexRecord( uri, cid, - op.value, + parsed.record, op.op === 'create' ? WriteOpAction.Create : WriteOpAction.Update, now, ) @@ -389,19 +391,15 @@ type UriAndCid = { cid: CID } -type RecordDescript = UriAndCid & { - value: unknown -} - type IndexOp = | ({ op: 'create' | 'update' - } & RecordDescript) + } & UriAndCid) | ({ op: 'delete' } & UriAndCid) const findDiffFromCheckout = ( curr: Record, - checkout: Record, + checkout: Record, ): IndexOp[] => { const ops: IndexOp[] = [] for (const uri of Object.keys(checkout)) { @@ -428,14 +426,13 @@ const findDiffFromCheckout = ( const formatCheckout = ( did: string, verifiedRepo: VerifiedRepo, -): Record => { - const records: Record = {} +): Record => { + const records: Record = {} for (const create of verifiedRepo.creates) { const uri = AtUri.make(did, create.collection, create.rkey) records[uri.toString()] = { uri, cid: create.cid, - value: create.record, } } return records diff --git a/packages/bsky/src/util/retry.ts b/packages/bsky/src/util/retry.ts index 75b54aeff07..ab96998642a 100644 --- a/packages/bsky/src/util/retry.ts +++ b/packages/bsky/src/util/retry.ts @@ -1,26 +1,6 @@ import { AxiosError } from 'axios' -import { wait } from '@atproto/common' import { XRPCError, ResponseType } from '@atproto/xrpc' - -export async function retry( - fn: () => Promise, - opts: RetryOptions = {}, -): Promise { - const { max = 3, retryable = () => true } = opts - let retries = 0 - let doneError: unknown - while (!doneError) { - try { - if (retries) await backoff(retries) - return await fn() - } catch (err) { - const willRetry = retries < max && retryable(err) - if (!willRetry) doneError = err - retries += 1 - } - } - throw doneError -} +import { RetryOptions, retry } from '@atproto/common' export async function retryHttp( fn: () => Promise, @@ -44,26 +24,3 @@ export function retryableHttp(err: unknown) { const retryableHttpStatusCodes = new Set([ 408, 425, 429, 500, 502, 503, 504, 522, 524, ]) - -type RetryOptions = { - max?: number - retryable?: (err: unknown) => boolean -} - -// Waits exponential backoff with max and jitter: ~50, ~100, ~200, ~400, ~800, ~1000, ~1000, ... -async function backoff(n: number, multiplier = 50, max = 1000) { - const exponentialMs = Math.pow(2, n) * multiplier - const ms = Math.min(exponentialMs, max) - await wait(jitter(ms)) -} - -// Adds randomness +/-15% of value -function jitter(value: number) { - const delta = value * 0.15 - return value + randomRange(-delta, delta) -} - -function randomRange(from: number, to: number) { - const rand = Math.random() * (to - from) - return rand + from -} diff --git a/packages/bsky/tests/admin/get-repo.test.ts b/packages/bsky/tests/admin/get-repo.test.ts index dbd143bb2ac..1e95f8cc0fc 100644 --- a/packages/bsky/tests/admin/get-repo.test.ts +++ b/packages/bsky/tests/admin/get-repo.test.ts @@ -97,9 +97,11 @@ describe('admin get repo view', () => { expect(beforeEmailVerification.emailConfirmedAt).toBeUndefined() const timestampBeforeVerification = Date.now() const bobsAccount = sc.accounts[sc.dids.bob] - const verificationToken = await network.pds.ctx.services - .account(network.pds.ctx.db) - .createEmailToken(sc.dids.bob, 'confirm_email') + const verificationToken = + await network.pds.ctx.accountManager.createEmailToken( + sc.dids.bob, + 'confirm_email', + ) await agent.api.com.atproto.server.confirmEmail( { email: bobsAccount.email, token: verificationToken }, { diff --git a/packages/bsky/tests/algos/hot-classic.test.ts b/packages/bsky/tests/algos/hot-classic.test.ts index aa96967c2c5..bb44ca5c0e8 100644 --- a/packages/bsky/tests/algos/hot-classic.test.ts +++ b/packages/bsky/tests/algos/hot-classic.test.ts @@ -31,7 +31,6 @@ describe('algo hot-classic', () => { alice = sc.dids.alice bob = sc.dids.bob await network.processAll() - await network.bsky.processAll() }) afterAll(async () => { @@ -59,7 +58,7 @@ describe('algo hot-classic', () => { await sc.like(sc.dids[name], two.ref) await sc.like(sc.dids[name], three.ref) } - await network.bsky.processAll() + await network.processAll() const res = await agent.api.app.bsky.feed.getFeed( { feed: feedUri }, diff --git a/packages/bsky/tests/auth.test.ts b/packages/bsky/tests/auth.test.ts index a3ac3d1d9f9..6b3fbd6b73d 100644 --- a/packages/bsky/tests/auth.test.ts +++ b/packages/bsky/tests/auth.test.ts @@ -36,7 +36,7 @@ describe('auth', () => { { headers: { authorization: `Bearer ${jwt}` } }, ) } - const origSigningKey = network.pds.ctx.repoSigningKey + const origSigningKey = await network.pds.ctx.actorStore.keypair(issuer) const newSigningKey = await Secp256k1Keypair.create({ exportable: true }) // confirm original signing key works await expect(attemptWithKey(origSigningKey)).resolves.toBeDefined() diff --git a/packages/bsky/tests/auto-moderator/labeler.test.ts b/packages/bsky/tests/auto-moderator/labeler.test.ts index dbd486c6061..3687a360980 100644 --- a/packages/bsky/tests/auto-moderator/labeler.test.ts +++ b/packages/bsky/tests/auto-moderator/labeler.test.ts @@ -40,26 +40,25 @@ describe('labeler', () => { await usersSeed(sc) await network.processAll() alice = sc.dids.alice - const repoSvc = pdsCtx.services.repo(pdsCtx.db) - const storeBlob = async (bytes: Uint8Array) => { - const blobRef = await repoSvc.blobs.addUntetheredBlob( - alice, - 'image/jpeg', - Readable.from([bytes], { objectMode: false }), - ) - const preparedBlobRef = { - cid: blobRef.ref, - mimeType: 'image/jpeg', - constraints: {}, - } - await repoSvc.blobs.verifyBlobAndMakePermanent(alice, preparedBlobRef) - await repoSvc.blobs.associateBlob( - preparedBlobRef, - postUri(), - TID.nextStr(), - alice, - ) - return blobRef + const storeBlob = (bytes: Uint8Array) => { + return pdsCtx.actorStore.transact(alice, async (store) => { + const blobRef = await store.repo.blob.addUntetheredBlob( + 'image/jpeg', + Readable.from([bytes], { objectMode: false }), + ) + const preparedBlobRef = { + cid: blobRef.ref, + mimeType: 'image/jpeg', + constraints: {}, + } + await store.repo.blob.verifyBlobAndMakePermanent(preparedBlobRef) + await store.repo.blob.associateBlob( + preparedBlobRef, + postUri(), + TID.nextStr(), + ) + return blobRef + }) } const bytes1 = new Uint8Array([1, 2, 3, 4]) const bytes2 = new Uint8Array([5, 6, 7, 8]) diff --git a/packages/bsky/tests/auto-moderator/takedowns.test.ts b/packages/bsky/tests/auto-moderator/takedowns.test.ts index 43a0fe5c00a..8c1f1a21cdd 100644 --- a/packages/bsky/tests/auto-moderator/takedowns.test.ts +++ b/packages/bsky/tests/auto-moderator/takedowns.test.ts @@ -106,11 +106,15 @@ describe('takedowner', () => { .executeTakeFirst() expect(record?.takedownId).toBeGreaterThan(0) - const recordPds = await network.pds.ctx.db.db - .selectFrom('record') - .where('uri', '=', post.ref.uriStr) - .select('takedownRef') - .executeTakeFirst() + const recordPds = await network.pds.ctx.actorStore.read( + post.ref.uri.hostname, + (store) => + store.db.db + .selectFrom('record') + .where('uri', '=', post.ref.uriStr) + .select('takedownRef') + .executeTakeFirst(), + ) expect(recordPds?.takedownRef).toEqual(takedownEvent.id.toString()) expect(testInvalidator.invalidated.length).toBe(1) @@ -162,11 +166,13 @@ describe('takedowner', () => { .executeTakeFirst() expect(record?.takedownId).toBeGreaterThan(0) - const recordPds = await network.pds.ctx.db.db - .selectFrom('record') - .where('uri', '=', res.data.uri) - .select('takedownRef') - .executeTakeFirst() + const recordPds = await network.pds.ctx.actorStore.read(alice, (store) => + store.db.db + .selectFrom('record') + .where('uri', '=', res.data.uri) + .select('takedownRef') + .executeTakeFirst(), + ) expect(recordPds?.takedownRef).toEqual(takedownEvent.id.toString()) expect(testInvalidator.invalidated.length).toBe(2) diff --git a/packages/bsky/tests/blob-resolver.test.ts b/packages/bsky/tests/blob-resolver.test.ts index 9a4d7f55c72..79491c5601e 100644 --- a/packages/bsky/tests/blob-resolver.test.ts +++ b/packages/bsky/tests/blob-resolver.test.ts @@ -77,8 +77,10 @@ describe('blob resolver', () => { }) it('fails on blob with bad signature check.', async () => { - await network.pds.ctx.blobstore.delete(fileCid) - await network.pds.ctx.blobstore.putPermanent(fileCid, randomBytes(100)) + await network.pds.ctx.blobstore(fileDid).delete(fileCid) + await network.pds.ctx + .blobstore(fileDid) + .putPermanent(fileCid, randomBytes(100)) const tryGetBlob = client.get(`/blob/${fileDid}/${fileCid.toString()}`) await expect(tryGetBlob).rejects.toThrow( 'maxContentLength size of -1 exceeded', diff --git a/packages/bsky/tests/handle-invalidation.test.ts b/packages/bsky/tests/handle-invalidation.test.ts index 972f1b6cc58..cee9cfb61df 100644 --- a/packages/bsky/tests/handle-invalidation.test.ts +++ b/packages/bsky/tests/handle-invalidation.test.ts @@ -102,11 +102,7 @@ describe('handle invalidation', () => { it('deals with handle contention', async () => { await backdateIndexedAt(bob) // update alices handle so that the pds will let bob take her old handle - await network.pds.ctx.db.db - .updateTable('did_handle') - .where('did', '=', alice) - .set({ handle: 'not-alice.test' }) - .execute() + await network.pds.ctx.accountManager.updateHandle(alice, 'not-alice.test') await pdsAgent.api.com.atproto.identity.updateHandle( { diff --git a/packages/bsky/tests/indexing.test.ts b/packages/bsky/tests/indexing.test.ts index 9457544b3e5..f874a084567 100644 --- a/packages/bsky/tests/indexing.test.ts +++ b/packages/bsky/tests/indexing.test.ts @@ -498,7 +498,8 @@ describe('indexing', () => { it('skips invalid records.', async () => { const { db, services } = network.bsky.indexer.ctx - const { db: pdsDb, services: pdsServices } = network.pds.ctx + const { accountManager } = network.pds.ctx + // const { db: pdsDb, services: pdsServices } = network.pds.ctx // Create a good and a bad post record const writes = await Promise.all([ pdsRepo.prepareCreate({ @@ -513,9 +514,20 @@ describe('indexing', () => { validate: false, }), ]) - await pdsServices - .repo(pdsDb) - .processWrites({ did: sc.dids.alice, writes }, 1) + const writeCommit = await network.pds.ctx.actorStore.transact( + sc.dids.alice, + (store) => store.repo.processWrites(writes), + ) + await accountManager.updateRepoRoot( + sc.dids.alice, + writeCommit.cid, + writeCommit.rev, + ) + await network.pds.ctx.sequencer.sequenceCommit( + sc.dids.alice, + writeCommit, + writes, + ) // Index const { data: commit } = await pdsAgent.api.com.atproto.sync.getLatestCommit({ @@ -643,15 +655,10 @@ describe('indexing', () => { ) await expect(getProfileBefore).resolves.toBeDefined() // Delete account on pds - await pdsAgent.api.com.atproto.server.requestAccountDelete(undefined, { - headers: sc.getHeaders(alice), - }) - const { token } = await network.pds.ctx.db.db - .selectFrom('email_token') - .selectAll() - .where('purpose', '=', 'delete_account') - .where('did', '=', alice) - .executeTakeFirstOrThrow() + const token = await network.pds.ctx.accountManager.createEmailToken( + alice, + 'delete_account', + ) await pdsAgent.api.com.atproto.server.deleteAccount({ token, did: alice, diff --git a/packages/bsky/tests/seeds/basic.ts b/packages/bsky/tests/seeds/basic.ts index 22c6fba01c5..b935afd3d6f 100644 --- a/packages/bsky/tests/seeds/basic.ts +++ b/packages/bsky/tests/seeds/basic.ts @@ -103,6 +103,8 @@ export default async (sc: SeedClient, users = true) => { 'tests/sample-img/key-landscape-small.jpg', 'image/jpeg', ) + // must ensure ordering of replies in indexing + await sc.network.processAll() await sc.reply( bob, sc.posts[alice][1].ref, @@ -117,6 +119,7 @@ export default async (sc: SeedClient, users = true) => { sc.posts[alice][1].ref, replies.carol[0], ) + await sc.network.processAll() const alicesReplyToBob = await sc.reply( alice, sc.posts[alice][1].ref, diff --git a/packages/bsky/tests/subscription/repo.test.ts b/packages/bsky/tests/subscription/repo.test.ts index dcdc77cd7a8..1c83e4c0cca 100644 --- a/packages/bsky/tests/subscription/repo.test.ts +++ b/packages/bsky/tests/subscription/repo.test.ts @@ -1,7 +1,6 @@ import AtpAgent from '@atproto/api' import { TestNetwork, SeedClient } from '@atproto/dev-env' import { CommitData } from '@atproto/repo' -import { RepoService } from '@atproto/pds/src/services/repo' import { PreparedWrite } from '@atproto/pds/src/repo' import * as sequencer from '@atproto/pds/src/sequencer' import { cborDecode, cborEncode } from '@atproto/common' @@ -84,9 +83,8 @@ describe('sync', () => { it('indexes actor when commit is unprocessable.', async () => { // mock sequencing to create an unprocessable commit event - const afterWriteProcessingOriginal = - RepoService.prototype.afterWriteProcessing - RepoService.prototype.afterWriteProcessing = async function ( + const sequenceCommitOrig = network.pds.ctx.sequencer.sequenceCommit + network.pds.ctx.sequencer.sequenceCommit = async function ( did: string, commitData: CommitData, writes: PreparedWrite[], @@ -95,7 +93,7 @@ describe('sync', () => { const evt = cborDecode(seqEvt.event) as sequencer.CommitEvt evt.blocks = new Uint8Array() // bad blocks seqEvt.event = cborEncode(evt) - await sequencer.sequenceEvt(this.db, seqEvt) + await network.pds.ctx.sequencer.sequenceEvt(seqEvt) } // create account and index the initial commit event await sc.createAccount('jack', { @@ -103,12 +101,11 @@ describe('sync', () => { email: 'jack@test.com', password: 'password', }) - await network.pds.ctx.sequencerLeader?.isCaughtUp() await network.processAll() // confirm jack was indexed as an actor despite the bad event const actors = await dumpTable(ctx.db.getPrimary(), 'actor', ['did']) expect(actors.map((a) => a.handle)).toContain('jack.test') - RepoService.prototype.afterWriteProcessing = afterWriteProcessingOriginal + network.pds.ctx.sequencer.sequenceCommit = sequenceCommitOrig }) async function updateProfile( diff --git a/packages/bsky/tests/views/threadgating.test.ts b/packages/bsky/tests/views/threadgating.test.ts index 5f530b33536..c0667bcf874 100644 --- a/packages/bsky/tests/views/threadgating.test.ts +++ b/packages/bsky/tests/views/threadgating.test.ts @@ -49,6 +49,7 @@ describe('views with thread gating', () => { { post: post.ref.uriStr, createdAt: iso(), allow: [] }, sc.getHeaders(sc.dids.carol), ) + await network.processAll() await sc.reply(sc.dids.alice, post.ref, post.ref, 'empty rules reply') await network.processAll() const { @@ -71,6 +72,7 @@ describe('views with thread gating', () => { { post: post.ref.uriStr, createdAt: iso(), allow: [] }, sc.getHeaders(sc.dids.carol), ) + await network.processAll() const reply = await sc.reply( sc.dids.alice, post.ref, @@ -118,6 +120,7 @@ describe('views with thread gating', () => { }, sc.getHeaders(sc.dids.carol), ) + await network.processAll() await sc.reply( sc.dids.alice, post.ref, @@ -167,6 +170,7 @@ describe('views with thread gating', () => { }, sc.getHeaders(sc.dids.carol), ) + await network.processAll() // carol only follows alice await sc.reply( sc.dids.dan, @@ -257,6 +261,7 @@ describe('views with thread gating', () => { }, sc.getHeaders(sc.dids.carol), ) + await network.processAll() // await sc.reply(sc.dids.bob, post.ref, post.ref, 'list rule reply disallow') const aliceReply = await sc.reply( @@ -324,6 +329,7 @@ describe('views with thread gating', () => { }, sc.getHeaders(sc.dids.carol), ) + await network.processAll() await sc.reply( sc.dids.alice, post.ref, @@ -365,6 +371,7 @@ describe('views with thread gating', () => { }, sc.getHeaders(sc.dids.carol), ) + await network.processAll() // carol only follows alice, and the post mentions dan. await sc.reply(sc.dids.bob, post.ref, post.ref, 'multi rule reply disallow') const aliceReply = await sc.reply( @@ -423,6 +430,7 @@ describe('views with thread gating', () => { { post: post.ref.uriStr, createdAt: iso() }, sc.getHeaders(sc.dids.carol), ) + await network.processAll() const aliceReply = await sc.reply( sc.dids.alice, post.ref, @@ -458,6 +466,7 @@ describe('views with thread gating', () => { }, sc.getHeaders(sc.dids.carol), ) + await network.processAll() // carol only follows alice const orphanedReply = await sc.reply( sc.dids.alice, @@ -519,6 +528,7 @@ describe('views with thread gating', () => { { post: post.ref.uriStr, createdAt: iso(), allow: [] }, sc.getHeaders(sc.dids.carol), ) + await network.processAll() const selfReply = await sc.reply( sc.dids.carol, post.ref, @@ -553,6 +563,7 @@ describe('views with thread gating', () => { }, sc.getHeaders(sc.dids.carol), ) + await network.processAll() // carol only follows alice const badReply = await sc.reply( sc.dids.dan, @@ -597,6 +608,7 @@ describe('views with thread gating', () => { { post: postB.ref.uriStr, createdAt: iso(), allow: [] }, sc.getHeaders(sc.dids.carol), ) + await network.processAll() await sc.reply(sc.dids.alice, postA.ref, postA.ref, 'ungated reply') await sc.reply(sc.dids.alice, postB.ref, postB.ref, 'ungated reply') await network.processAll() diff --git a/packages/common-web/src/async.ts b/packages/common-web/src/async.ts index 01e381ed337..e6da0fa6200 100644 --- a/packages/common-web/src/async.ts +++ b/packages/common-web/src/async.ts @@ -72,6 +72,8 @@ export class AsyncBuffer { private buffer: T[] = [] private promise: Promise private resolve: () => void + private closed = false + private toThrow: unknown | undefined constructor(public maxSize?: number) { // Initializing to satisfy types/build, immediately reset by resetPromise() @@ -88,6 +90,10 @@ export class AsyncBuffer { return this.buffer.length } + get isClosed(): boolean { + return this.closed + } + resetPromise() { this.promise = new Promise((r) => (this.resolve = r)) } @@ -104,7 +110,17 @@ export class AsyncBuffer { async *events(): AsyncGenerator { while (true) { + if (this.closed && this.buffer.length === 0) { + if (this.toThrow) { + throw this.toThrow + } else { + return + } + } await this.promise + if (this.toThrow) { + throw this.toThrow + } if (this.maxSize && this.size > this.maxSize) { throw new AsyncBufferFullError(this.maxSize) } @@ -117,6 +133,17 @@ export class AsyncBuffer { } } } + + throw(err: unknown) { + this.toThrow = err + this.closed = true + this.resolve() + } + + close() { + this.closed = true + this.resolve() + } } export class AsyncBufferFullError extends Error { diff --git a/packages/common-web/src/index.ts b/packages/common-web/src/index.ts index 8352123536a..ffb31dace2b 100644 --- a/packages/common-web/src/index.ts +++ b/packages/common-web/src/index.ts @@ -6,6 +6,7 @@ export * from './async' export * from './util' export * from './tid' export * from './ipld' +export * from './retry' export * from './types' export * from './times' export * from './strings' diff --git a/packages/common-web/src/retry.ts b/packages/common-web/src/retry.ts new file mode 100644 index 00000000000..357e765e873 --- /dev/null +++ b/packages/common-web/src/retry.ts @@ -0,0 +1,52 @@ +import { wait } from './util' + +export type RetryOptions = { + maxRetries?: number + getWaitMs?: (n: number) => number | null + retryable?: (err: unknown) => boolean +} + +export async function retry( + fn: () => Promise, + opts: RetryOptions = {}, +): Promise { + const { maxRetries = 3, retryable = () => true, getWaitMs = backoffMs } = opts + let retries = 0 + let doneError: unknown + while (!doneError) { + try { + return await fn() + } catch (err) { + const waitMs = getWaitMs(retries) + const willRetry = + retries < maxRetries && waitMs !== null && retryable(err) + if (willRetry) { + retries += 1 + if (waitMs !== 0) { + await wait(waitMs) + } + } else { + doneError = err + } + } + } + throw doneError +} + +// Waits exponential backoff with max and jitter: ~100, ~200, ~400, ~800, ~1000, ~1000, ... +export function backoffMs(n: number, multiplier = 100, max = 1000) { + const exponentialMs = Math.pow(2, n) * multiplier + const ms = Math.min(exponentialMs, max) + return jitter(ms) +} + +// Adds randomness +/-15% of value +function jitter(value: number) { + const delta = value * 0.15 + return value + randomRange(-delta, delta) +} + +function randomRange(from: number, to: number) { + const rand = Math.random() * (to - from) + return rand + from +} diff --git a/packages/common-web/tests/retry.test.ts b/packages/common-web/tests/retry.test.ts new file mode 100644 index 00000000000..641bb133b5f --- /dev/null +++ b/packages/common-web/tests/retry.test.ts @@ -0,0 +1,93 @@ +import { retry } from '../src/index' + +describe('retry', () => { + describe('retry()', () => { + it('retries until max retries', async () => { + let fnCalls = 0 + let waitMsCalls = 0 + const fn = async () => { + fnCalls++ + throw new Error(`Oops ${fnCalls}!`) + } + const getWaitMs = (retries) => { + waitMsCalls++ + expect(retries).toEqual(waitMsCalls - 1) + return 0 + } + await expect(retry(fn, { maxRetries: 13, getWaitMs })).rejects.toThrow( + 'Oops 14!', + ) + expect(fnCalls).toEqual(14) + expect(waitMsCalls).toEqual(14) + }) + + it('retries until max wait', async () => { + let fnCalls = 0 + let waitMsCalls = 0 + const fn = async () => { + fnCalls++ + throw new Error(`Oops ${fnCalls}!`) + } + const getWaitMs = (retries) => { + waitMsCalls++ + expect(retries).toEqual(waitMsCalls - 1) + if (retries === 13) { + return null + } + return 0 + } + await expect( + retry(fn, { maxRetries: Infinity, getWaitMs }), + ).rejects.toThrow('Oops 14!') + expect(fnCalls).toEqual(14) + expect(waitMsCalls).toEqual(14) + }) + + it('retries until non-retryable error', async () => { + let fnCalls = 0 + let waitMsCalls = 0 + const fn = async () => { + fnCalls++ + throw new Error(`Oops ${fnCalls}!`) + } + const getWaitMs = (retries) => { + waitMsCalls++ + expect(retries).toEqual(waitMsCalls - 1) + return 0 + } + const retryable = (err: unknown) => err?.['message'] !== 'Oops 14!' + await expect( + retry(fn, { maxRetries: Infinity, getWaitMs, retryable }), + ).rejects.toThrow('Oops 14!') + expect(fnCalls).toEqual(14) + expect(waitMsCalls).toEqual(14) + }) + + it('returns latest result after retries', async () => { + let fnCalls = 0 + const fn = async () => { + fnCalls++ + if (fnCalls < 14) { + throw new Error(`Oops ${fnCalls}!`) + } + return 'ok' + } + const getWaitMs = () => 0 + const result = await retry(fn, { maxRetries: Infinity, getWaitMs }) + expect(result).toBe('ok') + expect(fnCalls).toBe(14) + }) + + it('returns result immediately on success', async () => { + let fnCalls = 0 + const fn = async () => { + fnCalls++ + return 'ok' + } + const getWaitMs = () => 0 + const result = await retry(fn, { maxRetries: Infinity, getWaitMs }) + expect(result).toBe('ok') + expect(fnCalls).toBe(1) + }) + }) +}) diff --git a/packages/common/src/fs.ts b/packages/common/src/fs.ts index db7c586b621..b8bcf7be310 100644 --- a/packages/common/src/fs.ts +++ b/packages/common/src/fs.ts @@ -13,3 +13,30 @@ export const fileExists = async (location: string): Promise => { throw err } } + +export const readIfExists = async ( + filepath: string, +): Promise => { + try { + return await fs.readFile(filepath) + } catch (err) { + if (isErrnoException(err) && err.code === 'ENOENT') { + return + } + throw err + } +} + +export const rmIfExists = async ( + filepath: string, + recursive = false, +): Promise => { + try { + await fs.rm(filepath, { recursive }) + } catch (err) { + if (isErrnoException(err) && err.code === 'ENOENT') { + return + } + throw err + } +} diff --git a/packages/crypto/package.json b/packages/crypto/package.json index 20e5607e041..bb359c5f36f 100644 --- a/packages/crypto/package.json +++ b/packages/crypto/package.json @@ -28,5 +28,8 @@ "@noble/curves": "^1.1.0", "@noble/hashes": "^1.3.1", "uint8arrays": "3.0.0" + }, + "devDependencies": { + "@atproto/common": "workspace:^" } } diff --git a/packages/crypto/src/sha.ts b/packages/crypto/src/sha.ts index 7c996cdddeb..d25a61cba72 100644 --- a/packages/crypto/src/sha.ts +++ b/packages/crypto/src/sha.ts @@ -9,3 +9,10 @@ export const sha256 = async ( typeof input === 'string' ? uint8arrays.fromString(input, 'utf8') : input return noble.sha256(bytes) } + +export const sha256Hex = async ( + input: Uint8Array | string, +): Promise => { + const hash = await sha256(input) + return uint8arrays.toString(hash, 'hex') +} diff --git a/packages/crypto/src/types.ts b/packages/crypto/src/types.ts index be1dd4ec593..b664b6d4d30 100644 --- a/packages/crypto/src/types.ts +++ b/packages/crypto/src/types.ts @@ -9,6 +9,10 @@ export interface Didable { export interface Keypair extends Signer, Didable {} +export interface ExportableKeypair extends Keypair { + export(): Promise +} + export type DidKeyPlugin = { prefix: Uint8Array jwtAlg: string diff --git a/packages/crypto/tests/signatures.test.ts b/packages/crypto/tests/signatures.test.ts index 8ccaf7d992c..f495e304696 100644 --- a/packages/crypto/tests/signatures.test.ts +++ b/packages/crypto/tests/signatures.test.ts @@ -2,11 +2,11 @@ import fs from 'node:fs' import * as uint8arrays from 'uint8arrays' import { secp256k1 as nobleK256 } from '@noble/curves/secp256k1' import { p256 as nobleP256 } from '@noble/curves/p256' +import { cborEncode } from '@atproto/common' import EcdsaKeypair from '../src/p256/keypair' import Secp256k1Keypair from '../src/secp256k1/keypair' import * as p256 from '../src/p256/operations' import * as secp from '../src/secp256k1/operations' -import { cborEncode } from '@atproto/common' import { bytesToMultibase, multibaseToBytes, diff --git a/packages/dev-env/src/bin.ts b/packages/dev-env/src/bin.ts index 12228579a48..769dcca7c1b 100644 --- a/packages/dev-env/src/bin.ts +++ b/packages/dev-env/src/bin.ts @@ -18,11 +18,12 @@ const run = async () => { pds: { port: 2583, hostname: 'localhost', - dbPostgresSchema: 'pds', enableDidDocWithSession: true, }, bsky: { dbPostgresSchema: 'bsky', + port: 2584, + publicUrl: 'http://localhost:2584', }, plc: { port: 2582 }, }) diff --git a/packages/dev-env/src/bsky.ts b/packages/dev-env/src/bsky.ts index 968bceb9536..8320130eb43 100644 --- a/packages/dev-env/src/bsky.ts +++ b/packages/dev-env/src/bsky.ts @@ -306,7 +306,6 @@ export async function processAll( network: TestNetworkNoAppView, ingester: bsky.BskyIngester, ) { - assert(network.pds.ctx.sequencerLeader, 'sequencer leader does not exist') await network.pds.processAll() await ingestAll(network, ingester) // eslint-disable-next-line no-constant-condition @@ -326,25 +325,17 @@ export async function ingestAll( network: TestNetworkNoAppView, ingester: bsky.BskyIngester, ) { - assert(network.pds.ctx.sequencerLeader, 'sequencer leader does not exist') - const pdsDb = network.pds.ctx.db.db + const sequencer = network.pds.ctx.sequencer await network.pds.processAll() // eslint-disable-next-line no-constant-condition while (true) { await wait(50) - // check sequencer - const sequencerCaughtUp = await network.pds.ctx.sequencerLeader.isCaughtUp() - if (!sequencerCaughtUp) continue // check ingester - const [ingesterCursor, { lastSeq }] = await Promise.all([ + const [ingesterCursor, curr] = await Promise.all([ ingester.sub.getCursor(), - pdsDb - .selectFrom('repo_seq') - .where('seq', 'is not', null) - .select(pdsDb.fn.max('repo_seq.seq').as('lastSeq')) - .executeTakeFirstOrThrow(), + sequencer.curr(), ]) - const ingesterCaughtUp = ingesterCursor === lastSeq + const ingesterCaughtUp = curr !== null && ingesterCursor === curr if (ingesterCaughtUp) return } } diff --git a/packages/dev-env/src/const.ts b/packages/dev-env/src/const.ts index 137b8efd2c5..50a6dae52d9 100644 --- a/packages/dev-env/src/const.ts +++ b/packages/dev-env/src/const.ts @@ -1,3 +1,4 @@ export const ADMIN_PASSWORD = 'admin-pass' export const MOD_PASSWORD = 'mod-pass' export const TRIAGE_PASSWORD = 'triage-pass' +export const JWT_SECRET = 'jwt-secret' diff --git a/packages/dev-env/src/network-no-appview.ts b/packages/dev-env/src/network-no-appview.ts index 25054b2ab4e..30b978b5b79 100644 --- a/packages/dev-env/src/network-no-appview.ts +++ b/packages/dev-env/src/network-no-appview.ts @@ -13,17 +13,8 @@ export class TestNetworkNoAppView { static async create( params: Partial = {}, ): Promise { - const dbPostgresUrl = params.dbPostgresUrl || process.env.DB_POSTGRES_URL - const dbPostgresSchema = - params.dbPostgresSchema || process.env.DB_POSTGRES_SCHEMA - const dbSqliteLocation = - dbPostgresUrl === undefined ? ':memory:' : undefined - const plc = await TestPlc.create(params.plc ?? {}) const pds = await TestPds.create({ - dbPostgresUrl, - dbPostgresSchema, - dbSqliteLocation, didPlcUrl: plc.url, ...params.pds, }) diff --git a/packages/dev-env/src/network.ts b/packages/dev-env/src/network.ts index e1666935de7..6079ec4c968 100644 --- a/packages/dev-env/src/network.ts +++ b/packages/dev-env/src/network.ts @@ -45,9 +45,6 @@ export class TestNetwork extends TestNetworkNoAppView { }) const pds = await TestPds.create({ port: pdsPort, - dbPostgresUrl, - dbPostgresSchema, - dbPostgresPoolSize: 5, didPlcUrl: plc.url, bskyAppViewUrl: bsky.url, bskyAppViewDid: bsky.ctx.cfg.serverDid, @@ -62,26 +59,17 @@ export class TestNetwork extends TestNetworkNoAppView { async processFullSubscription(timeout = 5000) { const sub = this.bsky.indexer.sub - const { db } = this.pds.ctx.db const start = Date.now() + const lastSeq = await this.pds.ctx.sequencer.curr() + if (!lastSeq) return while (Date.now() - start < timeout) { - await wait(50) - if (!this.pds.ctx.sequencerLeader) { - throw new Error('Sequencer leader not configured on the pds') - } - const caughtUp = await this.pds.ctx.sequencerLeader.isCaughtUp() - if (!caughtUp) continue - const { lastSeq } = await db - .selectFrom('repo_seq') - .where('seq', 'is not', null) - .select(db.fn.max('repo_seq.seq').as('lastSeq')) - .executeTakeFirstOrThrow() - const { cursor } = sub.partitions.get(0) - if (cursor === lastSeq) { + const partitionState = sub.partitions.get(0) + if (partitionState?.cursor >= lastSeq) { // has seen last seq, just need to wait for it to finish processing await sub.repoQueue.main.onIdle() return } + await wait(5) } throw new Error(`Sequence was not processed within ${timeout}ms`) } @@ -93,10 +81,11 @@ export class TestNetwork extends TestNetworkNoAppView { } async serviceHeaders(did: string, aud?: string) { + const keypair = await this.pds.ctx.actorStore.keypair(did) const jwt = await createServiceJwt({ iss: did, aud: aud ?? this.bsky.ctx.cfg.serverDid, - keypair: this.pds.ctx.repoSigningKey, + keypair, }) return { authorization: `Bearer ${jwt}` } } diff --git a/packages/dev-env/src/pds.ts b/packages/dev-env/src/pds.ts index ada6dbec0a4..c52ada4a9e9 100644 --- a/packages/dev-env/src/pds.ts +++ b/packages/dev-env/src/pds.ts @@ -1,13 +1,19 @@ import path from 'node:path' import os from 'node:os' +import fs from 'node:fs/promises' import getPort from 'get-port' import * as ui8 from 'uint8arrays' import * as pds from '@atproto/pds' +import { createSecretKeyObject } from '@atproto/pds/src/auth-verifier' import { Secp256k1Keypair, randomStr } from '@atproto/crypto' import { AtpAgent } from '@atproto/api' import { PdsConfig } from './types' -import { uniqueLockId } from './util' -import { ADMIN_PASSWORD, MOD_PASSWORD, TRIAGE_PASSWORD } from './const' +import { + ADMIN_PASSWORD, + JWT_SECRET, + MOD_PASSWORD, + TRIAGE_PASSWORD, +} from './const' export class TestPds { constructor( @@ -17,8 +23,6 @@ export class TestPds { ) {} static async create(config: PdsConfig): Promise { - const repoSigningKey = await Secp256k1Keypair.create({ exportable: true }) - const repoSigningPriv = ui8.toString(await repoSigningKey.export(), 'hex') const plcRotationKey = await Secp256k1Keypair.create({ exportable: true }) const plcRotationPriv = ui8.toString(await plcRotationKey.export(), 'hex') const recoveryKey = (await Secp256k1Keypair.create()).did() @@ -27,21 +31,22 @@ export class TestPds { const url = `http://localhost:${port}` const blobstoreLoc = path.join(os.tmpdir(), randomStr(8, 'base32')) + const dataDirectory = path.join(os.tmpdir(), randomStr(8, 'base32')) + await fs.mkdir(dataDirectory, { recursive: true }) const env: pds.ServerEnvironment = { port, + dataDirectory: dataDirectory, blobstoreDiskLocation: blobstoreLoc, recoveryDidKey: recoveryKey, adminPassword: ADMIN_PASSWORD, moderatorPassword: MOD_PASSWORD, triagePassword: TRIAGE_PASSWORD, - jwtSecret: 'jwt-secret', + jwtSecret: JWT_SECRET, serviceHandleDomains: ['.test'], - sequencerLeaderLockId: uniqueLockId(), bskyAppViewUrl: 'https://appview.invalid', bskyAppViewDid: 'did:example:invalid', bskyAppViewCdnUrlPattern: 'http://cdn.appview.com/%s/%s/%s', - repoSigningKeyK256PrivateKeyHex: repoSigningPriv, plcRotationKeyK256PrivateKeyHex: plcRotationPriv, inviteRequired: false, ...config, @@ -51,20 +56,6 @@ export class TestPds { const server = await pds.PDS.create(cfg, secrets) - // Separate migration db on postgres in case migration changes some - // connection state that we need in the tests, e.g. "alter database ... set ..." - const migrationDb = - cfg.db.dialect === 'pg' - ? pds.Database.postgres({ - url: cfg.db.url, - schema: cfg.db.schema, - }) - : server.ctx.db - await migrationDb.migrateToLatestOrThrow() - if (migrationDb !== server.ctx.db) { - await migrationDb.close() - } - await server.start() return new TestPds(url, port, server) @@ -97,6 +88,10 @@ export class TestPds { } } + jwtSecretKey() { + return createSecretKeyObject(JWT_SECRET) + } + async processAll() { await this.ctx.backgroundQueue.processAll() } diff --git a/packages/lex-cli/src/codegen/server.ts b/packages/lex-cli/src/codegen/server.ts index 8363f1630c6..a86867d293b 100644 --- a/packages/lex-cli/src/codegen/server.ts +++ b/packages/lex-cli/src/codegen/server.ts @@ -203,6 +203,11 @@ const indexTs = ( }`, }) + file.addTypeAlias({ + name: 'HandlerOpts', + type: `{ blobLimit?: number }`, + }) + file.addTypeAlias({ name: 'HandlerRateLimitOpts', typeParameters: [{ name: 'T' }], @@ -220,6 +225,7 @@ const indexTs = ( | Handler | { auth?: Auth + opts?: HandlerOpts rateLimit?: HandlerRateLimitOpts | HandlerRateLimitOpts[] handler: Handler }`, diff --git a/packages/pds/bench/sequencer.bench.ts b/packages/pds/bench/sequencer.bench.ts deleted file mode 100644 index b7b054e9d8a..00000000000 --- a/packages/pds/bench/sequencer.bench.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { randomBytes } from '@atproto/crypto' -import { cborEncode } from '@atproto/common' -import { randomCid } from '@atproto/repo/tests/_util' -import { BlockMap, blocksToCarFile } from '@atproto/repo' -import { byFrame } from '@atproto/xrpc-server' -import { WebSocket } from 'ws' -import { Database } from '../src' -import { TestNetworkNoAppView } from '@atproto/dev-env' - -describe('sequencer bench', () => { - let network: TestNetworkNoAppView - - let db: Database - - beforeAll(async () => { - network = await TestNetworkNoAppView.create({ - dbPostgresSchema: 'sequencer_bench', - pds: { - maxSubscriptionBuffer: 20000, - }, - }) - if (network.pds.ctx.cfg.db.dialect !== 'pg') { - throw new Error('no postgres url') - } - db = Database.postgres({ - url: network.pds.ctx.cfg.db.url, - schema: network.pds.ctx.cfg.db.schema, - poolSize: 50, - }) - - network.pds.ctx.sequencerLeader?.destroy() - }) - - afterAll(async () => { - await network.close() - }) - - const doWrites = async (batches: number, batchSize: number) => { - const cid = await randomCid() - const blocks = new BlockMap() - await blocks.add(randomBytes(500)) - await blocks.add(randomBytes(500)) - await blocks.add(randomBytes(500)) - await blocks.add(randomBytes(500)) - await blocks.add(randomBytes(500)) - await blocks.add(randomBytes(500)) - - const car = await blocksToCarFile(cid, blocks) - const evt = { - rebase: false, - tooBig: false, - repo: 'did:plc:123451234', - commit: cid, - prev: cid, - ops: [{ action: 'create', path: 'app.bsky.feed.post/abcdefg1234', cid }], - blocks: car, - blobs: [], - } - const encodeEvt = cborEncode(evt) - - const promises: Promise[] = [] - for (let i = 0; i < batches; i++) { - const rows: any[] = [] - for (let j = 0; j < batchSize; j++) { - rows.push({ - did: 'did:web:example.com', - eventType: 'append', - event: encodeEvt, - sequencedAt: new Date().toISOString(), - }) - } - const insert = db.db.insertInto('repo_seq').values(rows).execute() - promises.push(insert) - } - await Promise.all(promises) - } - - const readAll = async ( - totalToRead: number, - cursor?: number, - ): Promise => { - const serverHost = network.pds.url.replace('http://', '') - let url = `ws://${serverHost}/xrpc/com.atproto.sync.subscribeRepos` - if (cursor !== undefined) { - url += `?cursor=${cursor}` - } - const ws = new WebSocket(url) - - let start = Date.now() - let count = 0 - const gen = byFrame(ws) - for await (const _frame of gen) { - if (count === 0) { - start = Date.now() - } - count++ - if (count >= totalToRead) { - break - } - } - if (count < totalToRead) { - throw new Error('Did not read full websocket') - } - return Date.now() - start - } - - it('benches', async () => { - const BATCHES = 100 - const BATCH_SIZE = 100 - const TOTAL = BATCHES * BATCH_SIZE - const readAllPromise = readAll(TOTAL, 0) - - const start = Date.now() - - await doWrites(BATCHES, BATCH_SIZE) - const setup = Date.now() - - await network.pds.ctx.sequencerLeader?.sequenceOutgoing() - const sequencingTime = Date.now() - setup - - const liveTailTime = await readAllPromise - const backfillTime = await readAll(TOTAL, 0) - - console.log(` -${TOTAL} events -Setup: ${setup - start} ms -Sequencing: ${sequencingTime} ms -Sequencing Rate: ${formatRate(TOTAL, sequencingTime)} evt/s -Live tail: ${liveTailTime} ms -Live tail Rate: ${formatRate(TOTAL, liveTailTime)} evt/s -Backfilled: ${backfillTime} ms -Backfill Rate: ${formatRate(TOTAL, backfillTime)} evt/s`) - }) -}) - -const formatRate = (evts: number, timeMs: number): string => { - const evtPerSec = (evts * 1000) / timeMs - return evtPerSec.toFixed(3) -} diff --git a/packages/pds/package.json b/packages/pds/package.json index c1f301c8fda..4f455d5d472 100644 --- a/packages/pds/package.json +++ b/packages/pds/package.json @@ -24,10 +24,11 @@ "build": "node ./build.js", "postbuild": "tsc --build tsconfig.build.json", "test": "../dev-infra/with-test-redis-and-db.sh jest", + "test:sqlite": "jest", + "test:sqlite-only": "jest --testPathIgnorePatterns /tests/proxied/*", + "test:log": "tail -50 test.log | pino-pretty", "update-main-to-dist": "node ../../update-main-to-dist.js packages/pds", "bench": "../dev-infra/with-test-redis-and-db.sh jest --config jest.bench.config.js", - "test:sqlite": "jest --testPathIgnorePatterns /tests/proxied/*", - "test:log": "tail -50 test.log | pino-pretty", "test:updateSnapshot": "jest --updateSnapshot", "migration:create": "ts-node ./bin/migration-create.ts" }, @@ -56,7 +57,8 @@ "http-errors": "^2.0.0", "http-terminator": "^3.2.0", "ioredis": "^5.3.2", - "jsonwebtoken": "^8.5.1", + "jose": "^5.0.1", + "key-encoder": "^2.0.3", "kysely": "^0.22.0", "multiformats": "^9.9.0", "nodemailer": "^6.8.0", @@ -75,16 +77,17 @@ "@atproto/bsky": "workspace:^", "@atproto/dev-env": "workspace:^", "@atproto/lex-cli": "workspace:^", + "@atproto/pds-entryway": "npm:@atproto/pds@0.3.0-entryway.2", "@did-plc/server": "^0.0.1", "@types/cors": "^2.8.12", "@types/disposable-email": "^0.2.0", "@types/express": "^4.17.13", "@types/express-serve-static-core": "^4.17.36", - "@types/jsonwebtoken": "^8.5.9", "@types/nodemailer": "^6.4.6", "@types/pg": "^8.6.6", "@types/qs": "^6.9.7", "axios": "^0.27.2", + "get-port": "^6.1.2", "ws": "^8.12.0" } } diff --git a/packages/pds/src/account-manager/db/index.ts b/packages/pds/src/account-manager/db/index.ts new file mode 100644 index 00000000000..30c22b16aa7 --- /dev/null +++ b/packages/pds/src/account-manager/db/index.ts @@ -0,0 +1,21 @@ +import { Database, Migrator } from '../../db' +import { DatabaseSchema } from './schema' +import migrations from './migrations' + +export * from './schema' + +export type AccountDb = Database + +export const getDb = ( + location: string, + disableWalAutoCheckpoint = false, +): AccountDb => { + const pragmas: Record = disableWalAutoCheckpoint + ? { wal_autocheckpoint: '0' } + : {} + return Database.sqlite(location, { pragmas }) +} + +export const getMigrator = (db: AccountDb) => { + return new Migrator(db.db, migrations) +} diff --git a/packages/pds/src/account-manager/db/migrations/001-init.ts b/packages/pds/src/account-manager/db/migrations/001-init.ts new file mode 100644 index 00000000000..14976ee2ed8 --- /dev/null +++ b/packages/pds/src/account-manager/db/migrations/001-init.ts @@ -0,0 +1,115 @@ +import { Kysely, sql } from 'kysely' + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('app_password') + .addColumn('did', 'varchar', (col) => col.notNull()) + .addColumn('name', 'varchar', (col) => col.notNull()) + .addColumn('passwordScrypt', 'varchar', (col) => col.notNull()) + .addColumn('createdAt', 'varchar', (col) => col.notNull()) + .addPrimaryKeyConstraint('app_password_pkey', ['did', 'name']) + .execute() + + await db.schema + .createTable('invite_code') + .addColumn('code', 'varchar', (col) => col.primaryKey()) + .addColumn('availableUses', 'integer', (col) => col.notNull()) + .addColumn('disabled', 'int2', (col) => col.defaultTo(0)) + .addColumn('forAccount', 'varchar', (col) => col.notNull()) + .addColumn('createdBy', 'varchar', (col) => col.notNull()) + .addColumn('createdAt', 'varchar', (col) => col.notNull()) + .execute() + await db.schema + .createIndex('invite_code_for_account_idx') + .on('invite_code') + .column('forAccount') + .execute() + + await db.schema + .createTable('invite_code_use') + .addColumn('code', 'varchar', (col) => col.notNull()) + .addColumn('usedBy', 'varchar', (col) => col.notNull()) + .addColumn('usedAt', 'varchar', (col) => col.notNull()) + .addPrimaryKeyConstraint(`invite_code_use_pkey`, ['code', 'usedBy']) + .execute() + + await db.schema + .createTable('refresh_token') + .addColumn('id', 'varchar', (col) => col.primaryKey()) + .addColumn('did', 'varchar', (col) => col.notNull()) + .addColumn('expiresAt', 'varchar', (col) => col.notNull()) + .addColumn('nextId', 'varchar') + .addColumn('appPasswordName', 'varchar') + .execute() + await db.schema // Aids in refresh token cleanup + .createIndex('refresh_token_did_idx') + .on('refresh_token') + .column('did') + .execute() + + await db.schema + .createTable('repo_root') + .addColumn('did', 'varchar', (col) => col.primaryKey()) + .addColumn('cid', 'varchar', (col) => col.notNull()) + .addColumn('rev', 'varchar', (col) => col.notNull()) + .addColumn('indexedAt', 'varchar', (col) => col.notNull()) + .execute() + + await db.schema + .createTable('actor') + .addColumn('did', 'varchar', (col) => col.primaryKey()) + .addColumn('handle', 'varchar') + .addColumn('createdAt', 'varchar', (col) => col.notNull()) + .addColumn('takedownRef', 'varchar') + .execute() + await db.schema + .createIndex(`actor_handle_lower_idx`) + .unique() + .on('actor') + .expression(sql`lower("handle")`) + .execute() + await db.schema + .createIndex('actor_cursor_idx') + .on('actor') + .columns(['createdAt', 'did']) + .execute() + + await db.schema + .createTable('account') + .addColumn('did', 'varchar', (col) => col.primaryKey()) + .addColumn('email', 'varchar', (col) => col.notNull()) + .addColumn('passwordScrypt', 'varchar', (col) => col.notNull()) + .addColumn('emailConfirmedAt', 'varchar') + .addColumn('invitesDisabled', 'int2', (col) => col.notNull().defaultTo(0)) + .execute() + await db.schema + .createIndex(`account_email_lower_idx`) + .unique() + .on('account') + .expression(sql`lower("email")`) + .execute() + + await db.schema + .createTable('email_token') + .addColumn('purpose', 'varchar', (col) => col.notNull()) + .addColumn('did', 'varchar', (col) => col.notNull()) + .addColumn('token', 'varchar', (col) => col.notNull()) + .addColumn('requestedAt', 'varchar', (col) => col.notNull()) + .addPrimaryKeyConstraint('email_token_pkey', ['purpose', 'did']) + .addUniqueConstraint('email_token_purpose_token_unique', [ + 'purpose', + 'token', + ]) + .execute() +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable('email_token').execute() + await db.schema.dropTable('account').execute() + await db.schema.dropTable('actor').execute() + await db.schema.dropTable('repo_root').execute() + await db.schema.dropTable('refresh_token').execute() + await db.schema.dropTable('invite_code_use').execute() + await db.schema.dropTable('invite_code').execute() + await db.schema.dropTable('app_password').execute() +} diff --git a/packages/pds/src/account-manager/db/migrations/index.ts b/packages/pds/src/account-manager/db/migrations/index.ts new file mode 100644 index 00000000000..4b694f0f0f4 --- /dev/null +++ b/packages/pds/src/account-manager/db/migrations/index.ts @@ -0,0 +1,5 @@ +import * as init from './001-init' + +export default { + '001': init, +} diff --git a/packages/pds/src/account-manager/db/schema/account.ts b/packages/pds/src/account-manager/db/schema/account.ts new file mode 100644 index 00000000000..458f809310d --- /dev/null +++ b/packages/pds/src/account-manager/db/schema/account.ts @@ -0,0 +1,15 @@ +import { Generated, Selectable } from 'kysely' + +export interface Account { + did: string + email: string + passwordScrypt: string + emailConfirmedAt: string | null + invitesDisabled: Generated<0 | 1> +} + +export type AccountEntry = Selectable + +export const tableName = 'account' + +export type PartialDB = { [tableName]: Account } diff --git a/packages/pds/src/account-manager/db/schema/actor.ts b/packages/pds/src/account-manager/db/schema/actor.ts new file mode 100644 index 00000000000..cfb3fcbe66b --- /dev/null +++ b/packages/pds/src/account-manager/db/schema/actor.ts @@ -0,0 +1,14 @@ +import { Selectable } from 'kysely' + +export interface Actor { + did: string + handle: string | null + createdAt: string + takedownRef: string | null +} + +export type ActorEntry = Selectable + +export const tableName = 'actor' + +export type PartialDB = { [tableName]: Actor } diff --git a/packages/pds/src/db/tables/app-password.ts b/packages/pds/src/account-manager/db/schema/app-password.ts similarity index 100% rename from packages/pds/src/db/tables/app-password.ts rename to packages/pds/src/account-manager/db/schema/app-password.ts diff --git a/packages/pds/src/db/tables/email-token.ts b/packages/pds/src/account-manager/db/schema/email-token.ts similarity index 93% rename from packages/pds/src/db/tables/email-token.ts rename to packages/pds/src/account-manager/db/schema/email-token.ts index b8f42bde198..c544a95ce54 100644 --- a/packages/pds/src/db/tables/email-token.ts +++ b/packages/pds/src/account-manager/db/schema/email-token.ts @@ -8,7 +8,7 @@ export interface EmailToken { purpose: EmailTokenPurpose did: string token: string - requestedAt: Date + requestedAt: string } export const tableName = 'email_token' diff --git a/packages/pds/src/account-manager/db/schema/index.ts b/packages/pds/src/account-manager/db/schema/index.ts new file mode 100644 index 00000000000..6bcd95a9138 --- /dev/null +++ b/packages/pds/src/account-manager/db/schema/index.ts @@ -0,0 +1,23 @@ +import * as actor from './actor' +import * as account from './account' +import * as repoRoot from './repo-root' +import * as refreshToken from './refresh-token' +import * as appPassword from './app-password' +import * as inviteCode from './invite-code' +import * as emailToken from './email-token' + +export type DatabaseSchema = actor.PartialDB & + account.PartialDB & + refreshToken.PartialDB & + appPassword.PartialDB & + repoRoot.PartialDB & + inviteCode.PartialDB & + emailToken.PartialDB + +export type { Actor, ActorEntry } from './actor' +export type { Account, AccountEntry } from './account' +export type { RepoRoot } from './repo-root' +export type { RefreshToken } from './refresh-token' +export type { AppPassword } from './app-password' +export type { InviteCode, InviteCodeUse } from './invite-code' +export type { EmailToken, EmailTokenPurpose } from './email-token' diff --git a/packages/pds/src/db/tables/invite-code.ts b/packages/pds/src/account-manager/db/schema/invite-code.ts similarity index 95% rename from packages/pds/src/db/tables/invite-code.ts rename to packages/pds/src/account-manager/db/schema/invite-code.ts index 99690c16af3..57fbcb5901a 100644 --- a/packages/pds/src/db/tables/invite-code.ts +++ b/packages/pds/src/account-manager/db/schema/invite-code.ts @@ -2,7 +2,7 @@ export interface InviteCode { code: string availableUses: number disabled: 0 | 1 - forUser: string + forAccount: string createdBy: string createdAt: string } diff --git a/packages/pds/src/db/tables/refresh-token.ts b/packages/pds/src/account-manager/db/schema/refresh-token.ts similarity index 100% rename from packages/pds/src/db/tables/refresh-token.ts rename to packages/pds/src/account-manager/db/schema/refresh-token.ts diff --git a/packages/pds/src/db/tables/repo-root.ts b/packages/pds/src/account-manager/db/schema/repo-root.ts similarity index 58% rename from packages/pds/src/db/tables/repo-root.ts rename to packages/pds/src/account-manager/db/schema/repo-root.ts index 74ca31d3f80..a8c0a6d0ba8 100644 --- a/packages/pds/src/db/tables/repo-root.ts +++ b/packages/pds/src/account-manager/db/schema/repo-root.ts @@ -1,10 +1,8 @@ -// @NOTE also used by app-view (moderation) export interface RepoRoot { did: string - root: string - rev: string | null + cid: string + rev: string indexedAt: string - takedownRef: string | null } export const tableName = 'repo_root' diff --git a/packages/pds/src/account-manager/helpers/account.ts b/packages/pds/src/account-manager/helpers/account.ts new file mode 100644 index 00000000000..f0e87c6d0ed --- /dev/null +++ b/packages/pds/src/account-manager/helpers/account.ts @@ -0,0 +1,210 @@ +import { isErrUniqueViolation, notSoftDeletedClause } from '../../db' +import { AccountDb, ActorEntry } from '../db' +import { StatusAttr } from '../../lexicon/types/com/atproto/admin/defs' + +export class UserAlreadyExistsError extends Error {} + +export type ActorAccount = ActorEntry & { + email: string | null + emailConfirmedAt: string | null + invitesDisabled: 0 | 1 | null +} + +const selectAccountQB = (db: AccountDb, includeSoftDeleted: boolean) => { + const { ref } = db.db.dynamic + return db.db + .selectFrom('actor') + .leftJoin('account', 'actor.did', 'account.did') + .if(!includeSoftDeleted, (qb) => + qb.where(notSoftDeletedClause(ref('actor'))), + ) + .select([ + 'actor.did', + 'actor.handle', + 'actor.createdAt', + 'actor.takedownRef', + 'account.email', + 'account.emailConfirmedAt', + 'account.invitesDisabled', + ]) +} + +export const getAccount = async ( + db: AccountDb, + handleOrDid: string, + includeSoftDeleted = false, +): Promise => { + const found = await selectAccountQB(db, includeSoftDeleted) + .where((qb) => { + if (handleOrDid.startsWith('did:')) { + return qb.where('actor.did', '=', handleOrDid) + } else { + return qb.where('actor.handle', '=', handleOrDid) + } + }) + .executeTakeFirst() + return found || null +} + +export const getAccountByEmail = async ( + db: AccountDb, + email: string, + includeSoftDeleted = false, +): Promise => { + const found = await selectAccountQB(db, includeSoftDeleted) + .where('email', '=', email.toLowerCase()) + .executeTakeFirst() + return found || null +} + +export const registerActor = async ( + db: AccountDb, + opts: { + did: string + handle: string + }, +) => { + const { did, handle } = opts + const [registered] = await db.executeWithRetry( + db.db + .insertInto('actor') + .values({ + did, + handle, + createdAt: new Date().toISOString(), + }) + .onConflict((oc) => oc.doNothing()) + .returning('did'), + ) + if (!registered) { + throw new UserAlreadyExistsError() + } +} + +export const registerAccount = async ( + db: AccountDb, + opts: { + did: string + email: string + passwordScrypt: string + }, +) => { + const { did, email, passwordScrypt } = opts + const [registered] = await db.executeWithRetry( + db.db + .insertInto('account') + .values({ + did, + email: email.toLowerCase(), + passwordScrypt, + }) + .onConflict((oc) => oc.doNothing()) + .returning('did'), + ) + if (!registered) { + throw new UserAlreadyExistsError() + } +} + +export const deleteAccount = async ( + db: AccountDb, + did: string, +): Promise => { + // Not done in transaction because it would be too long, prone to contention. + // Also, this can safely be run multiple times if it fails. + await db.executeWithRetry( + db.db.deleteFrom('repo_root').where('did', '=', did), + ) + await db.executeWithRetry( + db.db.deleteFrom('email_token').where('did', '=', did), + ) + await db.executeWithRetry( + db.db.deleteFrom('refresh_token').where('did', '=', did), + ) + await db.executeWithRetry( + db.db.deleteFrom('account').where('account.did', '=', did), + ) + await db.executeWithRetry( + db.db.deleteFrom('actor').where('actor.did', '=', did), + ) +} + +export const updateHandle = async ( + db: AccountDb, + did: string, + handle: string, +) => { + const [res] = await db.executeWithRetry( + db.db + .updateTable('actor') + .set({ handle }) + .where('did', '=', did) + .whereNotExists( + db.db.selectFrom('actor').where('handle', '=', handle).selectAll(), + ), + ) + if (res.numUpdatedRows < 1) { + throw new UserAlreadyExistsError() + } +} + +export const updateEmail = async ( + db: AccountDb, + did: string, + email: string, +) => { + try { + await db.executeWithRetry( + db.db + .updateTable('account') + .set({ email: email.toLowerCase(), emailConfirmedAt: null }) + .where('did', '=', did), + ) + } catch (err) { + if (isErrUniqueViolation(err)) { + throw new UserAlreadyExistsError() + } + throw err + } +} + +export const setEmailConfirmedAt = async ( + db: AccountDb, + did: string, + emailConfirmedAt: string, +) => { + await db.executeWithRetry( + db.db + .updateTable('account') + .set({ emailConfirmedAt }) + .where('did', '=', did), + ) +} + +export const getAccountTakedownStatus = async ( + db: AccountDb, + did: string, +): Promise => { + const res = await db.db + .selectFrom('actor') + .select('takedownRef') + .where('did', '=', did) + .executeTakeFirst() + if (!res) return null + return res.takedownRef + ? { applied: true, ref: res.takedownRef } + : { applied: false } +} + +export const updateAccountTakedownStatus = async ( + db: AccountDb, + did: string, + takedown: StatusAttr, +) => { + const takedownRef = takedown.applied + ? takedown.ref ?? new Date().toISOString() + : null + await db.executeWithRetry( + db.db.updateTable('actor').set({ takedownRef }).where('did', '=', did), + ) +} diff --git a/packages/pds/src/account-manager/helpers/auth.ts b/packages/pds/src/account-manager/helpers/auth.ts new file mode 100644 index 00000000000..083d5cfd39b --- /dev/null +++ b/packages/pds/src/account-manager/helpers/auth.ts @@ -0,0 +1,184 @@ +import assert from 'node:assert' +import { KeyObject } from 'node:crypto' +import * as jose from 'jose' +import * as ui8 from 'uint8arrays' +import * as crypto from '@atproto/crypto' +import { AuthScope } from '../../auth-verifier' +import { AccountDb } from '../db' + +export type AuthToken = { + scope: AuthScope + sub: string + exp: number +} + +export type RefreshToken = AuthToken & { scope: AuthScope.Refresh; jti: string } + +export const createTokens = async (opts: { + did: string + jwtKey: KeyObject + serviceDid: string + scope?: AuthScope + jti?: string + expiresIn?: string | number +}) => { + const { did, jwtKey, serviceDid, scope, jti, expiresIn } = opts + const [accessJwt, refreshJwt] = await Promise.all([ + createAccessToken({ did, jwtKey, serviceDid, scope, expiresIn }), + createRefreshToken({ did, jwtKey, serviceDid, jti, expiresIn }), + ]) + return { accessJwt, refreshJwt } +} + +export const createAccessToken = (opts: { + did: string + jwtKey: KeyObject + serviceDid: string + scope?: AuthScope + expiresIn?: string | number +}): Promise => { + const { + did, + jwtKey, + serviceDid, + scope = AuthScope.Access, + expiresIn = '120mins', + } = opts + const signer = new jose.SignJWT({ scope }) + .setProtectedHeader({ alg: 'HS256' }) // only symmetric keys supported + .setAudience(serviceDid) + .setSubject(did) + .setIssuedAt() + .setExpirationTime(expiresIn) + return signer.sign(jwtKey) +} + +export const createRefreshToken = (opts: { + did: string + jwtKey: KeyObject + serviceDid: string + jti?: string + expiresIn?: string | number +}): Promise => { + const { + did, + jwtKey, + serviceDid, + jti = getRefreshTokenId(), + expiresIn = '90days', + } = opts + const signer = new jose.SignJWT({ scope: AuthScope.Refresh }) + .setProtectedHeader({ alg: 'HS256' }) // only symmetric keys supported + .setAudience(serviceDid) + .setSubject(did) + .setJti(jti) + .setIssuedAt() + .setExpirationTime(expiresIn) + return signer.sign(jwtKey) +} + +// @NOTE unsafe for verification, should only be used w/ direct output from createRefreshToken() or createTokens() +export const decodeRefreshToken = (jwt: string) => { + const token = jose.decodeJwt(jwt) + assert.ok(token.scope === AuthScope.Refresh, 'not a refresh token') + return token as RefreshToken +} + +export const storeRefreshToken = async ( + db: AccountDb, + payload: RefreshToken, + appPasswordName: string | null, +) => { + const [result] = await db.executeWithRetry( + db.db + .insertInto('refresh_token') + .values({ + id: payload.jti, + did: payload.sub, + appPasswordName, + expiresAt: new Date(payload.exp * 1000).toISOString(), + }) + .onConflict((oc) => oc.doNothing()), // E.g. when re-granting during a refresh grace period + ) + return result +} + +export const getRefreshToken = async (db: AccountDb, id: string) => { + return db.db + .selectFrom('refresh_token') + .where('id', '=', id) + .selectAll() + .executeTakeFirst() +} + +export const deleteExpiredRefreshTokens = async ( + db: AccountDb, + did: string, + now: string, +) => { + await db.executeWithRetry( + db.db + .deleteFrom('refresh_token') + .where('did', '=', did) + .where('expiresAt', '<=', now), + ) +} + +export const addRefreshGracePeriod = async ( + db: AccountDb, + opts: { + id: string + expiresAt: string + nextId: string + }, +) => { + const { id, expiresAt, nextId } = opts + const [res] = await db.executeWithRetry( + db.db + .updateTable('refresh_token') + .where('id', '=', id) + .where((inner) => + inner.where('nextId', 'is', null).orWhere('nextId', '=', nextId), + ) + .set({ expiresAt, nextId }) + .returningAll(), + ) + if (!res) { + throw new ConcurrentRefreshError() + } +} + +export const revokeRefreshToken = async (db: AccountDb, id: string) => { + const [{ numDeletedRows }] = await db.executeWithRetry( + db.db.deleteFrom('refresh_token').where('id', '=', id), + ) + return numDeletedRows > 0 +} + +export const revokeRefreshTokensByDid = async (db: AccountDb, did: string) => { + const [{ numDeletedRows }] = await db.executeWithRetry( + db.db.deleteFrom('refresh_token').where('did', '=', did), + ) + return numDeletedRows > 0 +} + +export const revokeAppPasswordRefreshToken = async ( + db: AccountDb, + did: string, + appPassName: string, +) => { + const [{ numDeletedRows }] = await db.executeWithRetry( + db.db + .deleteFrom('refresh_token') + .where('did', '=', did) + .where('appPasswordName', '=', appPassName), + ) + + return numDeletedRows > 0 +} + +export const getRefreshTokenId = () => { + return ui8.toString(crypto.randomBytes(32), 'base64') +} + +export class ConcurrentRefreshError extends Error {} diff --git a/packages/pds/src/account-manager/helpers/email-token.ts b/packages/pds/src/account-manager/helpers/email-token.ts new file mode 100644 index 00000000000..85be5d4b6d0 --- /dev/null +++ b/packages/pds/src/account-manager/helpers/email-token.ts @@ -0,0 +1,80 @@ +import { InvalidRequestError } from '@atproto/xrpc-server' +import { MINUTE, lessThanAgoMs } from '@atproto/common' +import { getRandomToken } from '../../api/com/atproto/server/util' +import { AccountDb, EmailTokenPurpose } from '../db' + +export const createEmailToken = async ( + db: AccountDb, + did: string, + purpose: EmailTokenPurpose, +): Promise => { + const token = getRandomToken().toUpperCase() + const now = new Date().toISOString() + await db.executeWithRetry( + db.db + .insertInto('email_token') + .values({ purpose, did, token, requestedAt: now }) + .onConflict((oc) => + oc.columns(['purpose', 'did']).doUpdateSet({ token, requestedAt: now }), + ), + ) + return token +} + +export const deleteEmailToken = async ( + db: AccountDb, + did: string, + purpose: EmailTokenPurpose, +) => { + await db.executeWithRetry( + db.db + .deleteFrom('email_token') + .where('did', '=', did) + .where('purpose', '=', purpose), + ) +} + +export const assertValidToken = async ( + db: AccountDb, + did: string, + purpose: EmailTokenPurpose, + token: string, + expirationLen = 15 * MINUTE, +) => { + const res = await db.db + .selectFrom('email_token') + .selectAll() + .where('purpose', '=', purpose) + .where('did', '=', did) + .where('token', '=', token.toUpperCase()) + .executeTakeFirst() + if (!res) { + throw new InvalidRequestError('Token is invalid', 'InvalidToken') + } + const expired = !lessThanAgoMs(new Date(res.requestedAt), expirationLen) + if (expired) { + throw new InvalidRequestError('Token is expired', 'ExpiredToken') + } +} + +export const assertValidTokenAndFindDid = async ( + db: AccountDb, + purpose: EmailTokenPurpose, + token: string, + expirationLen = 15 * MINUTE, +): Promise => { + const res = await db.db + .selectFrom('email_token') + .selectAll() + .where('purpose', '=', purpose) + .where('token', '=', token.toUpperCase()) + .executeTakeFirst() + if (!res) { + throw new InvalidRequestError('Token is invalid', 'InvalidToken') + } + const expired = !lessThanAgoMs(new Date(res.requestedAt), expirationLen) + if (expired) { + throw new InvalidRequestError('Token is expired', 'ExpiredToken') + } + return res.did +} diff --git a/packages/pds/src/account-manager/helpers/invite.ts b/packages/pds/src/account-manager/helpers/invite.ts new file mode 100644 index 00000000000..81034b7f5d9 --- /dev/null +++ b/packages/pds/src/account-manager/helpers/invite.ts @@ -0,0 +1,259 @@ +import { chunkArray } from '@atproto/common' +import { InvalidRequestError } from '@atproto/xrpc-server' +import { AccountDb, InviteCode } from '../db' +import { countAll } from '../../db' + +export const createInviteCodes = async ( + db: AccountDb, + toCreate: { account: string; codes: string[] }[], + useCount: number, +) => { + const now = new Date().toISOString() + const rows = toCreate.flatMap((account) => + account.codes.map((code) => ({ + code: code, + availableUses: useCount, + disabled: 0 as const, + forAccount: account.account, + createdBy: 'admin', + createdAt: now, + })), + ) + await Promise.all( + chunkArray(rows, 50).map((chunk) => + db.executeWithRetry(db.db.insertInto('invite_code').values(chunk)), + ), + ) +} + +export const createAccountInviteCodes = async ( + db: AccountDb, + forAccount: string, + codes: string[], + expectedTotal: number, + disabled: 0 | 1, +): Promise => { + const now = new Date().toISOString() + const rows = codes.map( + (code) => + ({ + code, + availableUses: 1, + disabled, + forAccount, + createdBy: forAccount, + createdAt: now, + } as InviteCode), + ) + await db.executeWithRetry(db.db.insertInto('invite_code').values(rows)) + + const finalRoutineInviteCodes = await db.db + .selectFrom('invite_code') + .where('forAccount', '=', forAccount) + .where('createdBy', '!=', 'admin') // dont count admin-gifted codes aginast the user + .selectAll() + .execute() + if (finalRoutineInviteCodes.length > expectedTotal) { + throw new InvalidRequestError( + 'attempted to create additional codes in another request', + 'DuplicateCreate', + ) + } + + return rows.map((row) => ({ + ...row, + available: 1, + disabled: row.disabled === 1, + uses: [], + })) +} + +export const recordInviteUse = async ( + db: AccountDb, + opts: { + did: string + inviteCode: string | undefined + now: string + }, +) => { + if (!opts.inviteCode) return + await db.executeWithRetry( + db.db.insertInto('invite_code_use').values({ + code: opts.inviteCode, + usedBy: opts.did, + usedAt: opts.now, + }), + ) +} + +export const ensureInviteIsAvailable = async ( + db: AccountDb, + inviteCode: string, +): Promise => { + const invite = await db.db + .selectFrom('invite_code') + .leftJoin('actor', 'actor.did', 'invite_code.forAccount') + .where('takedownRef', 'is', null) + .selectAll('invite_code') + .where('code', '=', inviteCode) + .executeTakeFirst() + + if (!invite || invite.disabled) { + throw new InvalidRequestError( + 'Provided invite code not available', + 'InvalidInviteCode', + ) + } + + const uses = await db.db + .selectFrom('invite_code_use') + .select(countAll.as('count')) + .where('code', '=', inviteCode) + .executeTakeFirstOrThrow() + + if (invite.availableUses <= uses.count) { + throw new InvalidRequestError( + 'Provided invite code not available', + 'InvalidInviteCode', + ) + } +} + +export const selectInviteCodesQb = (db: AccountDb) => { + const ref = db.db.dynamic.ref + const builder = db.db + .selectFrom('invite_code') + .select([ + 'invite_code.code as code', + 'invite_code.availableUses as available', + 'invite_code.disabled as disabled', + 'invite_code.forAccount as forAccount', + 'invite_code.createdBy as createdBy', + 'invite_code.createdAt as createdAt', + db.db + .selectFrom('invite_code_use') + .select(countAll.as('count')) + .whereRef('invite_code_use.code', '=', ref('invite_code.code')) + .as('uses'), + ]) + return db.db.selectFrom(builder.as('codes')).selectAll() +} + +export const getAccountInviteCodes = async ( + db: AccountDb, + did: string, +): Promise => { + const res = await selectInviteCodesQb(db) + .where('forAccount', '=', did) + .execute() + const codes = res.map((row) => row.code) + const uses = await getInviteCodesUses(db, codes) + return res.map((row) => ({ + ...row, + uses: uses[row.code] ?? [], + disabled: row.disabled === 1, + })) +} + +export const getInviteCodesUses = async ( + db: AccountDb, + codes: string[], +): Promise> => { + const uses: Record = {} + if (codes.length > 0) { + const usesRes = await db.db + .selectFrom('invite_code_use') + .where('code', 'in', codes) + .orderBy('usedAt', 'desc') + .selectAll() + .execute() + for (const use of usesRes) { + const { code, usedBy, usedAt } = use + uses[code] ??= [] + uses[code].push({ usedBy, usedAt }) + } + } + return uses +} + +export const getInvitedByForAccounts = async ( + db: AccountDb, + dids: string[], +): Promise> => { + if (dids.length < 1) return {} + const codeDetailsRes = await selectInviteCodesQb(db) + .where('code', 'in', (qb) => + qb + .selectFrom('invite_code_use') + .where('usedBy', 'in', dids) + .select('code') + .distinct(), + ) + .execute() + const uses = await getInviteCodesUses( + db, + codeDetailsRes.map((row) => row.code), + ) + const codeDetails = codeDetailsRes.map((row) => ({ + ...row, + uses: uses[row.code] ?? [], + disabled: row.disabled === 1, + })) + return codeDetails.reduce((acc, cur) => { + for (const use of cur.uses) { + acc[use.usedBy] = cur + } + return acc + }, {} as Record) +} + +export const disableInviteCodes = async ( + db: AccountDb, + opts: { codes: string[]; accounts: string[] }, +) => { + const { codes, accounts } = opts + if (codes.length > 0) { + await db.executeWithRetry( + db.db + .updateTable('invite_code') + .set({ disabled: 1 }) + .where('code', 'in', codes), + ) + } + if (accounts.length > 0) { + await db.executeWithRetry( + db.db + .updateTable('invite_code') + .set({ disabled: 1 }) + .where('forAccount', 'in', accounts), + ) + } +} + +export const setAccountInvitesDisabled = async ( + db: AccountDb, + did: string, + disabled: boolean, +) => { + await db.executeWithRetry( + db.db + .updateTable('account') + .where('did', '=', did) + .set({ invitesDisabled: disabled ? 1 : 0 }), + ) +} + +export type CodeDetail = { + code: string + available: number + disabled: boolean + forAccount: string + createdBy: string + createdAt: string + uses: CodeUse[] +} + +type CodeUse = { + usedBy: string + usedAt: string +} diff --git a/packages/pds/src/account-manager/helpers/password.ts b/packages/pds/src/account-manager/helpers/password.ts new file mode 100644 index 00000000000..d7dc7ae1e62 --- /dev/null +++ b/packages/pds/src/account-manager/helpers/password.ts @@ -0,0 +1,109 @@ +import { randomStr } from '@atproto/crypto' +import { InvalidRequestError } from '@atproto/xrpc-server' +import * as scrypt from './scrypt' +import { AccountDb } from '../db' +import { AppPassword } from '../../lexicon/types/com/atproto/server/createAppPassword' + +export const verifyAccountPassword = async ( + db: AccountDb, + did: string, + password: string, +): Promise => { + const found = await db.db + .selectFrom('account') + .selectAll() + .where('did', '=', did) + .executeTakeFirst() + return found ? await scrypt.verify(password, found.passwordScrypt) : false +} + +export const verifyAppPassword = async ( + db: AccountDb, + did: string, + password: string, +): Promise => { + const passwordScrypt = await scrypt.hashAppPassword(did, password) + const found = await db.db + .selectFrom('app_password') + .selectAll() + .where('did', '=', did) + .where('passwordScrypt', '=', passwordScrypt) + .executeTakeFirst() + return found?.name ?? null +} + +export const updateUserPassword = async ( + db: AccountDb, + opts: { + did: string + passwordScrypt: string + }, +) => { + await db.executeWithRetry( + db.db + .updateTable('account') + .set({ passwordScrypt: opts.passwordScrypt }) + .where('did', '=', opts.did), + ) +} + +export const createAppPassword = async ( + db: AccountDb, + did: string, + name: string, +): Promise => { + // create an app password with format: + // 1234-abcd-5678-efgh + const str = randomStr(16, 'base32').slice(0, 16) + const chunks = [ + str.slice(0, 4), + str.slice(4, 8), + str.slice(8, 12), + str.slice(12, 16), + ] + const password = chunks.join('-') + const passwordScrypt = await scrypt.hashAppPassword(did, password) + const [got] = await db.executeWithRetry( + db.db + .insertInto('app_password') + .values({ + did, + name, + passwordScrypt, + createdAt: new Date().toISOString(), + }) + .returningAll(), + ) + if (!got) { + throw new InvalidRequestError('could not create app-specific password') + } + return { + name, + password, + createdAt: got.createdAt, + } +} + +export const listAppPasswords = async ( + db: AccountDb, + did: string, +): Promise<{ name: string; createdAt: string }[]> => { + return db.db + .selectFrom('app_password') + .select(['name', 'createdAt']) + .where('did', '=', did) + .execute() +} + +export const deleteAppPassword = async ( + db: AccountDb, + did: string, + name: string, +) => { + await db.executeWithRetry( + db.db + .deleteFrom('app_password') + .where('did', '=', did) + .where('name', '=', name), + ) +} diff --git a/packages/pds/src/account-manager/helpers/repo.ts b/packages/pds/src/account-manager/helpers/repo.ts new file mode 100644 index 00000000000..30988efe040 --- /dev/null +++ b/packages/pds/src/account-manager/helpers/repo.ts @@ -0,0 +1,24 @@ +import { CID } from 'multiformats/cid' +import { AccountDb } from '../db' + +export const updateRoot = async ( + db: AccountDb, + did: string, + cid: CID, + rev: string, +) => { + // @TODO balance risk of a race in the case of a long retry + await db.executeWithRetry( + db.db + .insertInto('repo_root') + .values({ + did, + cid: cid.toString(), + rev, + indexedAt: new Date().toISOString(), + }) + .onConflict((oc) => + oc.column('did').doUpdateSet({ cid: cid.toString(), rev }), + ), + ) +} diff --git a/packages/pds/src/db/scrypt.ts b/packages/pds/src/account-manager/helpers/scrypt.ts similarity index 100% rename from packages/pds/src/db/scrypt.ts rename to packages/pds/src/account-manager/helpers/scrypt.ts diff --git a/packages/pds/src/account-manager/index.ts b/packages/pds/src/account-manager/index.ts new file mode 100644 index 00000000000..1a6a2493fb9 --- /dev/null +++ b/packages/pds/src/account-manager/index.ts @@ -0,0 +1,353 @@ +import { KeyObject } from 'node:crypto' +import { HOUR } from '@atproto/common' +import { CID } from 'multiformats/cid' +import { AccountDb, EmailTokenPurpose, getDb, getMigrator } from './db' +import * as scrypt from './helpers/scrypt' +import * as account from './helpers/account' +import { ActorAccount } from './helpers/account' +import * as repo from './helpers/repo' +import * as auth from './helpers/auth' +import * as invite from './helpers/invite' +import * as password from './helpers/password' +import * as emailToken from './helpers/email-token' +import { AuthScope } from '../auth-verifier' +import { StatusAttr } from '../lexicon/types/com/atproto/admin/defs' + +export class AccountManager { + db: AccountDb + + constructor( + dbLocation: string, + private jwtKey: KeyObject, + private serviceDid: string, + disableWalAutoCheckpoint = false, + ) { + this.db = getDb(dbLocation, disableWalAutoCheckpoint) + } + + async migrateOrThrow() { + await this.db.ensureWal() + await getMigrator(this.db).migrateToLatestOrThrow() + } + + close() { + this.db.close() + } + + // Account + // ---------- + + async getAccount( + handleOrDid: string, + includeSoftDeleted = false, + ): Promise { + return account.getAccount(this.db, handleOrDid, includeSoftDeleted) + } + + async getAccountByEmail( + email: string, + includeSoftDeleted = false, + ): Promise { + return account.getAccountByEmail(this.db, email, includeSoftDeleted) + } + + // Repo exists and is not taken-down + async isRepoAvailable(did: string) { + const got = await this.getAccount(did) + return !!got + } + + async getDidForActor( + handleOrDid: string, + includeSoftDeleted = false, + ): Promise { + const got = await this.getAccount(handleOrDid, includeSoftDeleted) + return got?.did ?? null + } + + async createAccount(opts: { + did: string + handle: string + email?: string + password?: string + repoCid: CID + repoRev: string + inviteCode?: string + }) { + const { did, handle, email, password, repoCid, repoRev, inviteCode } = opts + const passwordScrypt = password + ? await scrypt.genSaltAndHash(password) + : undefined + + const { accessJwt, refreshJwt } = await auth.createTokens({ + did, + jwtKey: this.jwtKey, + serviceDid: this.serviceDid, + scope: AuthScope.Access, + }) + const refreshPayload = auth.decodeRefreshToken(refreshJwt) + const now = new Date().toISOString() + await this.db.transaction(async (dbTxn) => { + if (inviteCode) { + await invite.ensureInviteIsAvailable(dbTxn, inviteCode) + } + await Promise.all([ + account.registerActor(dbTxn, { did, handle }), + email && passwordScrypt + ? account.registerAccount(dbTxn, { did, email, passwordScrypt }) + : Promise.resolve(), + invite.recordInviteUse(dbTxn, { + did, + inviteCode, + now, + }), + auth.storeRefreshToken(dbTxn, refreshPayload, null), + repo.updateRoot(dbTxn, did, repoCid, repoRev), + ]) + }) + return { accessJwt, refreshJwt } + } + + // @NOTE should always be paired with a sequenceHandle(). + // the token output from this method should be passed to sequenceHandle(). + async updateHandle(did: string, handle: string) { + return account.updateHandle(this.db, did, handle) + } + + async deleteAccount(did: string) { + return account.deleteAccount(this.db, did) + } + + async takedownAccount(did: string, takedown: StatusAttr) { + await this.db.transaction((dbTxn) => + Promise.all([ + account.updateAccountTakedownStatus(dbTxn, did, takedown), + auth.revokeRefreshTokensByDid(dbTxn, did), + ]), + ) + } + + async getAccountTakedownStatus(did: string) { + return account.getAccountTakedownStatus(this.db, did) + } + + async updateRepoRoot(did: string, cid: CID, rev: string) { + return repo.updateRoot(this.db, did, cid, rev) + } + + // Auth + // ---------- + + async createSession(did: string, appPasswordName: string | null) { + const { accessJwt, refreshJwt } = await auth.createTokens({ + did, + jwtKey: this.jwtKey, + serviceDid: this.serviceDid, + scope: appPasswordName === null ? AuthScope.Access : AuthScope.AppPass, + }) + const refreshPayload = auth.decodeRefreshToken(refreshJwt) + await auth.storeRefreshToken(this.db, refreshPayload, appPasswordName) + return { accessJwt, refreshJwt } + } + + async rotateRefreshToken(id: string) { + const token = await auth.getRefreshToken(this.db, id) + if (!token) return null + + const now = new Date() + + // take the chance to tidy all of a user's expired tokens + // does not need to be transactional since this is just best-effort + await auth.deleteExpiredRefreshTokens(this.db, token.did, now.toISOString()) + + // Shorten the refresh token lifespan down from its + // original expiration time to its revocation grace period. + const prevExpiresAt = new Date(token.expiresAt) + const REFRESH_GRACE_MS = 2 * HOUR + const graceExpiresAt = new Date(now.getTime() + REFRESH_GRACE_MS) + + const expiresAt = + graceExpiresAt < prevExpiresAt ? graceExpiresAt : prevExpiresAt + + if (expiresAt <= now) { + return null + } + + // Determine the next refresh token id: upon refresh token + // reuse you always receive a refresh token with the same id. + const nextId = token.nextId ?? auth.getRefreshTokenId() + + const { accessJwt, refreshJwt } = await auth.createTokens({ + did: token.did, + jwtKey: this.jwtKey, + serviceDid: this.serviceDid, + scope: + token.appPasswordName === null ? AuthScope.Access : AuthScope.AppPass, + jti: nextId, + }) + + const refreshPayload = auth.decodeRefreshToken(refreshJwt) + try { + await this.db.transaction((dbTxn) => + Promise.all([ + auth.addRefreshGracePeriod(dbTxn, { + id, + expiresAt: expiresAt.toISOString(), + nextId, + }), + auth.storeRefreshToken(dbTxn, refreshPayload, token.appPasswordName), + ]), + ) + } catch (err) { + if (err instanceof auth.ConcurrentRefreshError) { + return this.rotateRefreshToken(id) + } + throw err + } + return { accessJwt, refreshJwt } + } + + async revokeRefreshToken(id: string) { + return auth.revokeRefreshToken(this.db, id) + } + + // Passwords + // ---------- + + async createAppPassword(did: string, name: string) { + return password.createAppPassword(this.db, did, name) + } + + async listAppPasswords(did: string) { + return password.listAppPasswords(this.db, did) + } + + async verifyAccountPassword( + did: string, + passwordStr: string, + ): Promise { + return password.verifyAccountPassword(this.db, did, passwordStr) + } + + async verifyAppPassword( + did: string, + passwordStr: string, + ): Promise { + return password.verifyAppPassword(this.db, did, passwordStr) + } + + async revokeAppPassword(did: string, name: string) { + await this.db.transaction(async (dbTxn) => + Promise.all([ + password.deleteAppPassword(dbTxn, did, name), + auth.revokeAppPasswordRefreshToken(dbTxn, did, name), + ]), + ) + } + + // Invites + // ---------- + + async ensureInviteIsAvailable(code: string) { + return invite.ensureInviteIsAvailable(this.db, code) + } + + async createInviteCodes( + toCreate: { account: string; codes: string[] }[], + useCount: number, + ) { + return invite.createInviteCodes(this.db, toCreate, useCount) + } + + async createAccountInviteCodes( + forAccount: string, + codes: string[], + expectedTotal: number, + disabled: 0 | 1, + ) { + return invite.createAccountInviteCodes( + this.db, + forAccount, + codes, + expectedTotal, + disabled, + ) + } + + async getAccountInvitesCodes(did: string) { + return invite.getAccountInviteCodes(this.db, did) + } + + async getInvitedByForAccounts(dids: string[]) { + return invite.getInvitedByForAccounts(this.db, dids) + } + + async getInviteCodesUses(codes: string[]) { + return invite.getInviteCodesUses(this.db, codes) + } + + async setAccountInvitesDisabled(did: string, disabled: boolean) { + return invite.setAccountInvitesDisabled(this.db, did, disabled) + } + + async disableInviteCodes(opts: { codes: string[]; accounts: string[] }) { + return invite.disableInviteCodes(this.db, opts) + } + + // Email Tokens + // ---------- + + async createEmailToken(did: string, purpose: EmailTokenPurpose) { + return emailToken.createEmailToken(this.db, did, purpose) + } + + async assertValidEmailToken( + did: string, + purpose: EmailTokenPurpose, + token: string, + ) { + return emailToken.assertValidToken(this.db, did, purpose, token) + } + + async confirmEmail(opts: { did: string; token: string }) { + const { did, token } = opts + await emailToken.assertValidToken(this.db, did, 'confirm_email', token) + const now = new Date().toISOString() + await this.db.transaction((dbTxn) => + Promise.all([ + emailToken.deleteEmailToken(dbTxn, did, 'confirm_email'), + account.setEmailConfirmedAt(dbTxn, did, now), + ]), + ) + } + + async updateEmail(opts: { did: string; email: string; token?: string }) { + const { did, email, token } = opts + if (token) { + await this.db.transaction((dbTxn) => + Promise.all([ + account.updateEmail(dbTxn, did, email), + emailToken.deleteEmailToken(dbTxn, did, 'update_email'), + ]), + ) + } else { + return account.updateEmail(this.db, did, email) + } + } + + async resetPassword(opts: { password: string; token: string }) { + const did = await emailToken.assertValidTokenAndFindDid( + this.db, + 'reset_password', + opts.token, + ) + const passwordScrypt = await scrypt.genSaltAndHash(opts.password) + await this.db.transaction(async (dbTxn) => + Promise.all([ + password.updateUserPassword(dbTxn, { did, passwordScrypt }), + emailToken.deleteEmailToken(dbTxn, did, 'reset_password'), + auth.revokeRefreshTokensByDid(dbTxn, did), + ]), + ) + } +} diff --git a/packages/pds/src/actor-store/blob/reader.ts b/packages/pds/src/actor-store/blob/reader.ts new file mode 100644 index 00000000000..bb38ed92e69 --- /dev/null +++ b/packages/pds/src/actor-store/blob/reader.ts @@ -0,0 +1,76 @@ +import stream from 'stream' +import { CID } from 'multiformats/cid' +import { BlobNotFoundError, BlobStore } from '@atproto/repo' +import { InvalidRequestError } from '@atproto/xrpc-server' +import { ActorDb } from '../db' +import { notSoftDeletedClause } from '../../db/util' +import { StatusAttr } from '../../lexicon/types/com/atproto/admin/defs' + +export class BlobReader { + constructor(public db: ActorDb, public blobstore: BlobStore) {} + + async getBlob( + cid: CID, + ): Promise<{ size: number; mimeType?: string; stream: stream.Readable }> { + const { ref } = this.db.db.dynamic + const found = await this.db.db + .selectFrom('blob') + .selectAll() + .where('blob.cid', '=', cid.toString()) + .where(notSoftDeletedClause(ref('blob'))) + .executeTakeFirst() + if (!found) { + throw new InvalidRequestError('Blob not found') + } + let blobStream + try { + blobStream = await this.blobstore.getStream(cid) + } catch (err) { + if (err instanceof BlobNotFoundError) { + throw new InvalidRequestError('Blob not found') + } + throw err + } + return { + size: found.size, + mimeType: found.mimeType, + stream: blobStream, + } + } + + async listBlobs(opts: { + since?: string + cursor?: string + limit: number + }): Promise { + const { since, cursor, limit } = opts + let builder = this.db.db + .selectFrom('record_blob') + .select('blobCid') + .orderBy('blobCid', 'asc') + .groupBy('blobCid') + .limit(limit) + if (since) { + builder = builder + .innerJoin('record', 'record.uri', 'record_blob.recordUri') + .where('record.repoRev', '>', since) + } + if (cursor) { + builder = builder.where('blobCid', '>', cursor) + } + const res = await builder.execute() + return res.map((row) => row.blobCid) + } + + async getBlobTakedownStatus(cid: CID): Promise { + const res = await this.db.db + .selectFrom('blob') + .select('takedownRef') + .where('cid', '=', cid.toString()) + .executeTakeFirst() + if (!res) return null + return res.takedownRef + ? { applied: true, ref: res.takedownRef } + : { applied: false } + } +} diff --git a/packages/pds/src/services/repo/blobs.ts b/packages/pds/src/actor-store/blob/transactor.ts similarity index 59% rename from packages/pds/src/services/repo/blobs.ts rename to packages/pds/src/actor-store/blob/transactor.ts index 318a5a26c4f..013235639a0 100644 --- a/packages/pds/src/services/repo/blobs.ts +++ b/packages/pds/src/actor-store/blob/transactor.ts @@ -3,27 +3,33 @@ import crypto from 'crypto' import { CID } from 'multiformats/cid' import bytes from 'bytes' import { fromStream as fileTypeFromStream } from 'file-type' -import { BlobStore, CidSet, WriteOpAction } from '@atproto/repo' +import { BlobNotFoundError, BlobStore, WriteOpAction } from '@atproto/repo' import { AtUri } from '@atproto/syntax' import { cloneStream, sha256RawToCid, streamSize } from '@atproto/common' import { InvalidRequestError } from '@atproto/xrpc-server' import { BlobRef } from '@atproto/lexicon' -import { PreparedBlobRef, PreparedWrite } from '../../repo/types' -import Database from '../../db' -import { Blob as BlobTable } from '../../db/tables/blob' +import { ActorDb, Blob as BlobTable } from '../db' +import { + PreparedBlobRef, + PreparedWrite, + PreparedDelete, + PreparedUpdate, +} from '../../repo/types' import * as img from '../../image' -import { PreparedDelete, PreparedUpdate } from '../../repo' import { BackgroundQueue } from '../../background' +import { BlobReader } from './reader' +import { StatusAttr } from '../../lexicon/types/com/atproto/admin/defs' -export class RepoBlobs { +export class BlobTransactor extends BlobReader { constructor( - public db: Database, + public db: ActorDb, public blobstore: BlobStore, public backgroundQueue: BackgroundQueue, - ) {} + ) { + super(db, blobstore) + } async addUntetheredBlob( - creator: string, userSuggestedMime: string, blobStream: stream.Readable, ): Promise { @@ -41,7 +47,6 @@ export class RepoBlobs { await this.db.db .insertInto('blob') .values({ - creator, cid: cid.toString(), mimeType, size, @@ -52,7 +57,7 @@ export class RepoBlobs { }) .onConflict((oc) => oc - .columns(['creator', 'cid']) + .column('cid') .doUpdateSet({ tempKey }) .where('blob.tempKey', 'is not', null), ) @@ -60,8 +65,8 @@ export class RepoBlobs { return new BlobRef(cid, mimeType, size) } - async processWriteBlobs(did: string, rev: string, writes: PreparedWrite[]) { - await this.deleteDereferencedBlobs(did, writes) + async processWriteBlobs(rev: string, writes: PreparedWrite[]) { + await this.deleteDereferencedBlobs(writes) const blobPromises: Promise[] = [] for (const write of writes) { @@ -70,15 +75,37 @@ export class RepoBlobs { write.action === WriteOpAction.Update ) { for (const blob of write.blobs) { - blobPromises.push(this.verifyBlobAndMakePermanent(did, blob)) - blobPromises.push(this.associateBlob(blob, write.uri, rev, did)) + blobPromises.push(this.verifyBlobAndMakePermanent(blob)) + blobPromises.push(this.associateBlob(blob, write.uri)) } } } await Promise.all(blobPromises) } - async deleteDereferencedBlobs(did: string, writes: PreparedWrite[]) { + async updateBlobTakedownStatus(blob: CID, takedown: StatusAttr) { + const takedownRef = takedown.applied + ? takedown.ref ?? new Date().toISOString() + : null + await this.db.db + .updateTable('blob') + .set({ takedownRef }) + .where('cid', '=', blob.toString()) + .executeTakeFirst() + try { + if (takedown.applied) { + await this.blobstore.quarantine(blob) + } else { + await this.blobstore.unquarantine(blob) + } + } catch (err) { + if (!(err instanceof BlobNotFoundError)) { + throw err + } + } + } + + async deleteDereferencedBlobs(writes: PreparedWrite[]) { const deletes = writes.filter( (w) => w.action === WriteOpAction.Delete, ) as PreparedDelete[] @@ -89,19 +116,17 @@ export class RepoBlobs { if (uris.length === 0) return const deletedRepoBlobs = await this.db.db - .deleteFrom('repo_blob') - .where('did', '=', did) + .deleteFrom('record_blob') .where('recordUri', 'in', uris) .returningAll() .execute() if (deletedRepoBlobs.length < 1) return - const deletedRepoBlobCids = deletedRepoBlobs.map((row) => row.cid) + const deletedRepoBlobCids = deletedRepoBlobs.map((row) => row.blobCid) const duplicateCids = await this.db.db - .selectFrom('repo_blob') - .where('did', '=', did) - .where('cid', 'in', deletedRepoBlobCids) - .select('cid') + .selectFrom('record_blob') + .where('blobCid', 'in', deletedRepoBlobCids) + .select('blobCid') .execute() const newBlobCids = writes @@ -112,7 +137,10 @@ export class RepoBlobs { ) .flat() .map((b) => b.cid.toString()) - const cidsToKeep = [...newBlobCids, ...duplicateCids.map((row) => row.cid)] + const cidsToKeep = [ + ...newBlobCids, + ...duplicateCids.map((row) => row.blobCid), + ] const cidsToDelete = deletedRepoBlobCids.filter( (cid) => !cidsToKeep.includes(cid), ) @@ -120,51 +148,23 @@ export class RepoBlobs { await this.db.db .deleteFrom('blob') - .where('creator', '=', did) .where('cid', 'in', cidsToDelete) .execute() - - // check if these blobs are used by other users before deleting from blobstore - const stillUsedRes = await this.db.db - .selectFrom('blob') - .where('cid', 'in', cidsToDelete) - .select('cid') - .distinct() - .execute() - const stillUsed = stillUsedRes.map((row) => row.cid) - - const blobsToDelete = cidsToDelete.filter((cid) => !stillUsed.includes(cid)) - - // move actual blob deletion to the background queue - if (blobsToDelete.length > 0) { - this.db.onCommit(() => { - this.backgroundQueue.add(async () => { - await Promise.allSettled( - blobsToDelete.map((cid) => this.blobstore.delete(CID.parse(cid))), - ) - }) + this.db.onCommit(() => { + this.backgroundQueue.add(async () => { + await Promise.allSettled( + cidsToDelete.map((cid) => this.blobstore.delete(CID.parse(cid))), + ) }) - } + }) } - async verifyBlobAndMakePermanent( - creator: string, - blob: PreparedBlobRef, - ): Promise { - const { ref } = this.db.db.dynamic + async verifyBlobAndMakePermanent(blob: PreparedBlobRef): Promise { const found = await this.db.db .selectFrom('blob') .selectAll() - .where('creator', '=', creator) .where('cid', '=', blob.cid.toString()) - .whereNotExists( - // Check if blob has been taken down - this.db.db - .selectFrom('repo_blob') - .selectAll() - .where('takedownRef', 'is not', null) - .whereRef('cid', '=', ref('blob.cid')), - ) + .where('takedownRef', 'is', null) .executeTakeFirst() if (!found) { throw new InvalidRequestError( @@ -183,63 +183,16 @@ export class RepoBlobs { } } - async associateBlob( - blob: PreparedBlobRef, - recordUri: AtUri, - repoRev: string, - did: string, - ): Promise { + async associateBlob(blob: PreparedBlobRef, recordUri: AtUri): Promise { await this.db.db - .insertInto('repo_blob') + .insertInto('record_blob') .values({ - cid: blob.cid.toString(), + blobCid: blob.cid.toString(), recordUri: recordUri.toString(), - repoRev, - did, }) .onConflict((oc) => oc.doNothing()) .execute() } - - async listSinceRev(did: string, rev?: string): Promise { - let builder = this.db.db - .selectFrom('repo_blob') - .where('did', '=', did) - .select('cid') - if (rev) { - builder = builder.where('repoRev', '>', rev) - } - const res = await builder.execute() - const cids = res.map((row) => CID.parse(row.cid)) - return new CidSet(cids).toList() - } - - async deleteForUser(did: string): Promise { - // Not done in transaction because it would be too long, prone to contention. - // Also, this can safely be run multiple times if it fails. - const deleted = await this.db.db - .deleteFrom('blob') - .where('creator', '=', did) - .returningAll() - .execute() - await this.db.db.deleteFrom('repo_blob').where('did', '=', did).execute() - const deletedCids = deleted.map((d) => d.cid) - let duplicateCids: string[] = [] - if (deletedCids.length > 0) { - const res = await this.db.db - .selectFrom('repo_blob') - .where('cid', 'in', deletedCids) - .selectAll() - .execute() - duplicateCids = res.map((d) => d.cid) - } - const toDelete = deletedCids.filter((cid) => !duplicateCids.includes(cid)) - if (toDelete.length > 0) { - await Promise.all( - toDelete.map((cid) => this.blobstore.delete(CID.parse(cid))), - ) - } - } } export class CidNotFound extends Error { diff --git a/packages/pds/src/actor-store/db/index.ts b/packages/pds/src/actor-store/db/index.ts new file mode 100644 index 00000000000..0f12b949d82 --- /dev/null +++ b/packages/pds/src/actor-store/db/index.ts @@ -0,0 +1,20 @@ +import { DatabaseSchema } from './schema' +import { Database, Migrator } from '../../db' +import migrations from './migrations' +export * from './schema' + +export type ActorDb = Database + +export const getDb = ( + location: string, + disableWalAutoCheckpoint = false, +): ActorDb => { + const pragmas: Record = disableWalAutoCheckpoint + ? { wal_autocheckpoint: '0' } + : {} + return Database.sqlite(location, { pragmas }) +} + +export const getMigrator = (db: Database) => { + return new Migrator(db.db, migrations) +} diff --git a/packages/pds/src/actor-store/db/migrations/001-init.ts b/packages/pds/src/actor-store/db/migrations/001-init.ts new file mode 100644 index 00000000000..b414c6ca211 --- /dev/null +++ b/packages/pds/src/actor-store/db/migrations/001-init.ts @@ -0,0 +1,105 @@ +import { Kysely } from 'kysely' + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('repo_root') + .addColumn('did', 'varchar', (col) => col.primaryKey()) + .addColumn('cid', 'varchar', (col) => col.notNull()) + .addColumn('rev', 'varchar', (col) => col.notNull()) + .addColumn('indexedAt', 'varchar', (col) => col.notNull()) + .execute() + + await db.schema + .createTable('repo_block') + .addColumn('cid', 'varchar', (col) => col.primaryKey()) + .addColumn('repoRev', 'varchar', (col) => col.notNull()) + .addColumn('size', 'integer', (col) => col.notNull()) + .addColumn('content', 'blob', (col) => col.notNull()) + .execute() + + await db.schema + .createIndex('repo_block_repo_rev_idx') + .on('repo_block') + .columns(['repoRev', 'cid']) + .execute() + + await db.schema + .createTable('record') + .addColumn('uri', 'varchar', (col) => col.primaryKey()) + .addColumn('cid', 'varchar', (col) => col.notNull()) + .addColumn('collection', 'varchar', (col) => col.notNull()) + .addColumn('rkey', 'varchar', (col) => col.notNull()) + .addColumn('repoRev', 'varchar', (col) => col.notNull()) + .addColumn('indexedAt', 'varchar', (col) => col.notNull()) + .addColumn('takedownRef', 'varchar') + .execute() + await db.schema + .createIndex('record_cid_idx') + .on('record') + .column('cid') + .execute() + await db.schema + .createIndex('record_collection_idx') + .on('record') + .column('collection') + .execute() + await db.schema + .createIndex('record_repo_rev_idx') + .on('record') + .column('repoRev') + .execute() + + await db.schema + .createTable('blob') + .addColumn('cid', 'varchar', (col) => col.primaryKey()) + .addColumn('mimeType', 'varchar', (col) => col.notNull()) + .addColumn('size', 'integer', (col) => col.notNull()) + .addColumn('tempKey', 'varchar') + .addColumn('width', 'integer') + .addColumn('height', 'integer') + .addColumn('createdAt', 'varchar', (col) => col.notNull()) + .addColumn('takedownRef', 'varchar') + .execute() + await db.schema + .createIndex('blob_tempkey_idx') + .on('blob') + .column('tempKey') + .execute() + + await db.schema + .createTable('record_blob') + .addColumn('blobCid', 'varchar', (col) => col.notNull()) + .addColumn('recordUri', 'varchar', (col) => col.notNull()) + .addPrimaryKeyConstraint(`record_blob_pkey`, ['blobCid', 'recordUri']) + .execute() + + await db.schema + .createTable('backlink') + .addColumn('uri', 'varchar', (col) => col.notNull()) + .addColumn('path', 'varchar', (col) => col.notNull()) + .addColumn('linkTo', 'varchar', (col) => col.notNull()) + .addPrimaryKeyConstraint('backlinks_pkey', ['uri', 'path']) + .execute() + await db.schema + .createIndex('backlink_link_to_idx') + .on('backlink') + .columns(['path', 'linkTo']) + .execute() + + await db.schema + .createTable('account_pref') + .addColumn('id', 'integer', (col) => col.autoIncrement().primaryKey()) + .addColumn('name', 'varchar', (col) => col.notNull()) + .addColumn('valueJson', 'text', (col) => col.notNull()) + .execute() +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable('account_pref').execute() + await db.schema.dropTable('backlink').execute() + await db.schema.dropTable('record_blob').execute() + await db.schema.dropTable('blob').execute() + await db.schema.dropTable('record').execute() + await db.schema.dropTable('repo_block').execute() + await db.schema.dropTable('repo_root').execute() +} diff --git a/packages/pds/src/actor-store/db/migrations/index.ts b/packages/pds/src/actor-store/db/migrations/index.ts new file mode 100644 index 00000000000..4b694f0f0f4 --- /dev/null +++ b/packages/pds/src/actor-store/db/migrations/index.ts @@ -0,0 +1,5 @@ +import * as init from './001-init' + +export default { + '001': init, +} diff --git a/packages/pds/src/actor-store/db/schema/account-pref.ts b/packages/pds/src/actor-store/db/schema/account-pref.ts new file mode 100644 index 00000000000..3ae93b8ff27 --- /dev/null +++ b/packages/pds/src/actor-store/db/schema/account-pref.ts @@ -0,0 +1,11 @@ +import { GeneratedAlways } from 'kysely' + +export interface AccountPref { + id: GeneratedAlways + name: string + valueJson: string // json +} + +export const tableName = 'account_pref' + +export type PartialDB = { [tableName]: AccountPref } diff --git a/packages/pds/src/db/tables/backlink.ts b/packages/pds/src/actor-store/db/schema/backlink.ts similarity index 73% rename from packages/pds/src/db/tables/backlink.ts rename to packages/pds/src/actor-store/db/schema/backlink.ts index 3b552a95c57..cf50f09b8ea 100644 --- a/packages/pds/src/db/tables/backlink.ts +++ b/packages/pds/src/actor-store/db/schema/backlink.ts @@ -1,8 +1,7 @@ export interface Backlink { uri: string path: string - linkToUri: string | null - linkToDid: string | null + linkTo: string } export const tableName = 'backlink' diff --git a/packages/pds/src/db/tables/blob.ts b/packages/pds/src/actor-store/db/schema/blob.ts similarity index 89% rename from packages/pds/src/db/tables/blob.ts rename to packages/pds/src/actor-store/db/schema/blob.ts index afea699db16..0bd99a65c53 100644 --- a/packages/pds/src/db/tables/blob.ts +++ b/packages/pds/src/actor-store/db/schema/blob.ts @@ -1,5 +1,4 @@ export interface Blob { - creator: string cid: string mimeType: string size: number @@ -7,6 +6,7 @@ export interface Blob { width: number | null height: number | null createdAt: string + takedownRef: string | null } export const tableName = 'blob' diff --git a/packages/pds/src/actor-store/db/schema/index.ts b/packages/pds/src/actor-store/db/schema/index.ts new file mode 100644 index 00000000000..4d0199b5e30 --- /dev/null +++ b/packages/pds/src/actor-store/db/schema/index.ts @@ -0,0 +1,23 @@ +import * as accountPref from './account-pref' +import * as repoRoot from './repo-root' +import * as record from './record' +import * as backlink from './backlink' +import * as repoBlock from './repo-block' +import * as blob from './blob' +import * as recordBlob from './record-blob' + +export type DatabaseSchema = accountPref.PartialDB & + repoRoot.PartialDB & + record.PartialDB & + backlink.PartialDB & + repoBlock.PartialDB & + blob.PartialDB & + recordBlob.PartialDB + +export type { AccountPref } from './account-pref' +export type { RepoRoot } from './repo-root' +export type { Record } from './record' +export type { Backlink } from './backlink' +export type { RepoBlock } from './repo-block' +export type { Blob } from './blob' +export type { RecordBlob } from './record-blob' diff --git a/packages/pds/src/actor-store/db/schema/record-blob.ts b/packages/pds/src/actor-store/db/schema/record-blob.ts new file mode 100644 index 00000000000..dcf63164d63 --- /dev/null +++ b/packages/pds/src/actor-store/db/schema/record-blob.ts @@ -0,0 +1,8 @@ +export interface RecordBlob { + blobCid: string + recordUri: string +} + +export const tableName = 'record_blob' + +export type PartialDB = { [tableName]: RecordBlob } diff --git a/packages/pds/src/db/tables/record.ts b/packages/pds/src/actor-store/db/schema/record.ts similarity index 87% rename from packages/pds/src/db/tables/record.ts rename to packages/pds/src/actor-store/db/schema/record.ts index 04d4dd8524f..2b94139b348 100644 --- a/packages/pds/src/db/tables/record.ts +++ b/packages/pds/src/actor-store/db/schema/record.ts @@ -2,10 +2,9 @@ export interface Record { uri: string cid: string - did: string collection: string rkey: string - repoRev: string | null + repoRev: string indexedAt: string takedownRef: string | null } diff --git a/packages/pds/src/actor-store/db/schema/repo-block.ts b/packages/pds/src/actor-store/db/schema/repo-block.ts new file mode 100644 index 00000000000..e5fd1b600c6 --- /dev/null +++ b/packages/pds/src/actor-store/db/schema/repo-block.ts @@ -0,0 +1,10 @@ +export interface RepoBlock { + cid: string + repoRev: string + size: number + content: Uint8Array +} + +export const tableName = 'repo_block' + +export type PartialDB = { [tableName]: RepoBlock } diff --git a/packages/pds/src/actor-store/db/schema/repo-root.ts b/packages/pds/src/actor-store/db/schema/repo-root.ts new file mode 100644 index 00000000000..71527d9a1d8 --- /dev/null +++ b/packages/pds/src/actor-store/db/schema/repo-root.ts @@ -0,0 +1,10 @@ +export interface RepoRoot { + did: string + cid: string + rev: string + indexedAt: string +} + +const tableName = 'repo_root' + +export type PartialDB = { [tableName]: RepoRoot } diff --git a/packages/pds/src/actor-store/index.ts b/packages/pds/src/actor-store/index.ts new file mode 100644 index 00000000000..b18716380e6 --- /dev/null +++ b/packages/pds/src/actor-store/index.ts @@ -0,0 +1,261 @@ +import path from 'path' +import fs from 'fs/promises' +import * as crypto from '@atproto/crypto' +import { Keypair, ExportableKeypair } from '@atproto/crypto' +import { BlobStore } from '@atproto/repo' +import { + chunkArray, + fileExists, + readIfExists, + rmIfExists, +} from '@atproto/common' +import { ActorDb, getDb, getMigrator } from './db' +import { BackgroundQueue } from '../background' +import { RecordReader } from './record/reader' +import { PreferenceReader } from './preference/reader' +import { RepoReader } from './repo/reader' +import { RepoTransactor } from './repo/transactor' +import { PreferenceTransactor } from './preference/transactor' +import { InvalidRequestError } from '@atproto/xrpc-server' +import { RecordTransactor } from './record/transactor' +import { CID } from 'multiformats/cid' +import DiskBlobStore from '../disk-blobstore' +import { mkdir } from 'fs/promises' +import { ActorStoreConfig } from '../config' +import { retrySqlite } from '../db' + +type ActorStoreResources = { + blobstore: (did: string) => BlobStore + backgroundQueue: BackgroundQueue + reservedKeyDir?: string +} + +export class ActorStore { + reservedKeyDir: string + + constructor( + public cfg: ActorStoreConfig, + public resources: ActorStoreResources, + ) { + this.reservedKeyDir = path.join(cfg.directory, 'reserved_keys') + } + + async getLocation(did: string) { + const didHash = await crypto.sha256Hex(did) + const directory = path.join(this.cfg.directory, didHash.slice(0, 2), did) + const dbLocation = path.join(directory, `store.sqlite`) + const keyLocation = path.join(directory, `key`) + return { directory, dbLocation, keyLocation } + } + + async exists(did: string): Promise { + const location = await this.getLocation(did) + return await fileExists(location.dbLocation) + } + + async keypair(did: string): Promise { + const { keyLocation } = await this.getLocation(did) + const privKey = await fs.readFile(keyLocation) + return crypto.Secp256k1Keypair.import(privKey) + } + + async openDb(did: string): Promise { + const { dbLocation } = await this.getLocation(did) + const exists = await fileExists(dbLocation) + if (!exists) { + throw new InvalidRequestError('Repo not found', 'NotFound') + } + + const db = getDb(dbLocation, this.cfg.disableWalAutoCheckpoint) + + // run a simple select with retry logic to ensure the db is ready (not in wal recovery mode) + try { + await retrySqlite(() => + db.db.selectFrom('repo_root').selectAll().execute(), + ) + } catch (err) { + db.close() + throw err + } + + return db + } + + async read(did: string, fn: ActorStoreReadFn) { + const db = await this.openDb(did) + try { + const reader = createActorReader(did, db, this.resources, () => + this.keypair(did), + ) + return await fn(reader) + } finally { + db.close() + } + } + + async transact(did: string, fn: ActorStoreTransactFn) { + const keypair = await this.keypair(did) + const db = await this.openDb(did) + try { + return await db.transaction((dbTxn) => { + const store = createActorTransactor(did, dbTxn, keypair, this.resources) + return fn(store) + }) + } finally { + db.close() + } + } + + async create(did: string, keypair: ExportableKeypair) { + const { directory, dbLocation, keyLocation } = await this.getLocation(did) + // ensure subdir exists + await mkdir(directory, { recursive: true }) + const exists = await fileExists(dbLocation) + if (exists) { + throw new InvalidRequestError('Repo already exists', 'AlreadyExists') + } + const privKey = await keypair.export() + await fs.writeFile(keyLocation, privKey) + + const db: ActorDb = getDb(dbLocation, this.cfg.disableWalAutoCheckpoint) + try { + await db.ensureWal() + const migrator = getMigrator(db) + await migrator.migrateToLatestOrThrow() + } finally { + db.close() + } + } + + async destroy(did: string) { + const blobstore = this.resources.blobstore(did) + if (blobstore instanceof DiskBlobStore) { + await blobstore.deleteAll() + } else { + const blobRows = await this.read(did, (store) => + store.db.db.selectFrom('blob').select('cid').execute(), + ) + const cids = blobRows.map((row) => CID.parse(row.cid)) + await Promise.allSettled( + chunkArray(cids, 500).map((chunk) => blobstore.deleteMany(chunk)), + ) + } + + const { directory } = await this.getLocation(did) + await rmIfExists(directory, true) + } + + async reserveKeypair(did?: string): Promise { + let keyLoc: string | undefined + if (did) { + keyLoc = path.join(this.reservedKeyDir, did) + const maybeKey = await loadKey(keyLoc) + if (maybeKey) { + return maybeKey.did() + } + } + const keypair = await crypto.Secp256k1Keypair.create({ exportable: true }) + const keyDid = keypair.did() + keyLoc = keyLoc ?? path.join(this.reservedKeyDir, keyDid) + await mkdir(this.reservedKeyDir, { recursive: true }) + await fs.writeFile(keyLoc, await keypair.export()) + return keyDid + } + + async getReservedKeypair( + signingKeyOrDid: string, + ): Promise { + return loadKey(path.join(this.reservedKeyDir, signingKeyOrDid)) + } + + async clearReservedKeypair(keyDid: string, did?: string) { + await rmIfExists(path.join(this.reservedKeyDir, keyDid)) + if (did) { + await rmIfExists(path.join(this.reservedKeyDir, did)) + } + } + + async storePlcOp(did: string, op: Uint8Array) { + const { directory } = await this.getLocation(did) + const opLoc = path.join(directory, `did-op`) + await fs.writeFile(opLoc, op) + } + + async getPlcOp(did: string): Promise { + const { directory } = await this.getLocation(did) + const opLoc = path.join(directory, `did-op`) + return await fs.readFile(opLoc) + } + + async clearPlcOp(did: string) { + const { directory } = await this.getLocation(did) + const opLoc = path.join(directory, `did-op`) + await rmIfExists(opLoc) + } +} + +const loadKey = async (loc: string): Promise => { + const privKey = await readIfExists(loc) + if (!privKey) return undefined + return crypto.Secp256k1Keypair.import(privKey, { exportable: true }) +} + +const createActorTransactor = ( + did: string, + db: ActorDb, + keypair: Keypair, + resources: ActorStoreResources, +): ActorStoreTransactor => { + const { blobstore, backgroundQueue } = resources + const userBlobstore = blobstore(did) + return { + did, + db, + repo: new RepoTransactor(db, did, keypair, userBlobstore, backgroundQueue), + record: new RecordTransactor(db, userBlobstore), + pref: new PreferenceTransactor(db), + } +} + +const createActorReader = ( + did: string, + db: ActorDb, + resources: ActorStoreResources, + getKeypair: () => Promise, +): ActorStoreReader => { + const { blobstore } = resources + return { + did, + db, + repo: new RepoReader(db, blobstore(did)), + record: new RecordReader(db), + pref: new PreferenceReader(db), + transact: async (fn: ActorStoreTransactFn): Promise => { + const keypair = await getKeypair() + return db.transaction((dbTxn) => { + const store = createActorTransactor(did, dbTxn, keypair, resources) + return fn(store) + }) + }, + } +} + +export type ActorStoreReadFn = (fn: ActorStoreReader) => Promise +export type ActorStoreTransactFn = (fn: ActorStoreTransactor) => Promise + +export type ActorStoreReader = { + did: string + db: ActorDb + repo: RepoReader + record: RecordReader + pref: PreferenceReader + transact: (fn: ActorStoreTransactFn) => Promise +} + +export type ActorStoreTransactor = { + did: string + db: ActorDb + repo: RepoTransactor + record: RecordTransactor + pref: PreferenceTransactor +} diff --git a/packages/pds/src/actor-store/migrate.ts b/packages/pds/src/actor-store/migrate.ts new file mode 100644 index 00000000000..a54fabfea3e --- /dev/null +++ b/packages/pds/src/actor-store/migrate.ts @@ -0,0 +1,39 @@ +import { sql } from 'kysely' +import AppContext from '../context' +import PQueue from 'p-queue' + +export const forEachActorStore = async ( + ctx: AppContext, + opts: { concurrency?: number }, + fn: (ctx: AppContext, did: string) => Promise, +) => { + const { concurrency = 1 } = opts + + const queue = new PQueue({ concurrency }) + const actorQb = ctx.accountManager.db.db + .selectFrom('actor') + .selectAll() + .limit(2 * concurrency) + let cursor: { createdAt: string; did: string } | undefined + do { + const actors = cursor + ? await actorQb + .where( + sql`("createdAt", "did")`, + '>', + sql`(${cursor.createdAt}, ${cursor.did})`, + ) + .execute() + : await actorQb.execute() + queue.addAll( + actors.map(({ did }) => { + return () => fn(ctx, did) + }), + ) + cursor = actors.at(-1) + await queue.onEmpty() // wait for all remaining items to be in process, then move on to next page + } while (cursor) + + // finalize remaining work + await queue.onIdle() +} diff --git a/packages/pds/src/actor-store/preference/reader.ts b/packages/pds/src/actor-store/preference/reader.ts new file mode 100644 index 00000000000..2325350ff82 --- /dev/null +++ b/packages/pds/src/actor-store/preference/reader.ts @@ -0,0 +1,22 @@ +import { ActorDb } from '../db' + +export class PreferenceReader { + constructor(public db: ActorDb) {} + + async getPreferences(namespace?: string): Promise { + const prefsRes = await this.db.db + .selectFrom('account_pref') + .orderBy('id') + .selectAll() + .execute() + return prefsRes + .filter((pref) => !namespace || prefMatchNamespace(namespace, pref.name)) + .map((pref) => JSON.parse(pref.valueJson)) + } +} + +export type AccountPreference = Record & { $type: string } + +export const prefMatchNamespace = (namespace: string, fullname: string) => { + return fullname === namespace || fullname.startsWith(`${namespace}.`) +} diff --git a/packages/pds/src/actor-store/preference/transactor.ts b/packages/pds/src/actor-store/preference/transactor.ts new file mode 100644 index 00000000000..cfb8bd383ea --- /dev/null +++ b/packages/pds/src/actor-store/preference/transactor.ts @@ -0,0 +1,44 @@ +import { InvalidRequestError } from '@atproto/xrpc-server' +import { + PreferenceReader, + AccountPreference, + prefMatchNamespace, +} from './reader' + +export class PreferenceTransactor extends PreferenceReader { + async putPreferences( + values: AccountPreference[], + namespace: string, + ): Promise { + this.db.assertTransaction() + if (!values.every((value) => prefMatchNamespace(namespace, value.$type))) { + throw new InvalidRequestError( + `Some preferences are not in the ${namespace} namespace`, + ) + } + // get all current prefs for user and prep new pref rows + const allPrefs = await this.db.db + .selectFrom('account_pref') + .select(['id', 'name']) + .execute() + const putPrefs = values.map((value) => { + return { + name: value.$type, + valueJson: JSON.stringify(value), + } + }) + const allPrefIdsInNamespace = allPrefs + .filter((pref) => prefMatchNamespace(namespace, pref.name)) + .map((pref) => pref.id) + // replace all prefs in given namespace + if (allPrefIdsInNamespace.length) { + await this.db.db + .deleteFrom('account_pref') + .where('id', 'in', allPrefIdsInNamespace) + .execute() + } + if (putPrefs.length) { + await this.db.db.insertInto('account_pref').values(putPrefs).execute() + } + } +} diff --git a/packages/pds/src/services/record/index.ts b/packages/pds/src/actor-store/record/reader.ts similarity index 53% rename from packages/pds/src/services/record/index.ts rename to packages/pds/src/actor-store/record/reader.ts index de2491c97f3..ed8d231f3cf 100644 --- a/packages/pds/src/services/record/index.ts +++ b/packages/pds/src/actor-store/record/reader.ts @@ -1,91 +1,20 @@ -import { CID } from 'multiformats/cid' +import * as syntax from '@atproto/syntax' import { AtUri, ensureValidAtUri } from '@atproto/syntax' -import * as ident from '@atproto/syntax' -import { cborToLexRecord, WriteOpAction } from '@atproto/repo' -import { dbLogger as log } from '../../logger' -import Database from '../../db' +import { cborToLexRecord } from '@atproto/repo' +import { CID } from 'multiformats/cid' import { notSoftDeletedClause } from '../../db/util' -import { Backlink } from '../../db/tables/backlink' import { ids } from '../../lexicon/lexicons' +import { ActorDb, Backlink } from '../db' +import { StatusAttr } from '../../lexicon/types/com/atproto/admin/defs' +import { RepoRecord } from '@atproto/lexicon' -export class RecordService { - constructor(public db: Database) {} - - static creator() { - return (db: Database) => new RecordService(db) - } - - async indexRecord( - uri: AtUri, - cid: CID, - obj: unknown, - action: WriteOpAction.Create | WriteOpAction.Update = WriteOpAction.Create, - repoRev?: string, - timestamp?: string, - ) { - this.db.assertTransaction() - log.debug({ uri }, 'indexing record') - const record = { - uri: uri.toString(), - cid: cid.toString(), - did: uri.host, - collection: uri.collection, - rkey: uri.rkey, - repoRev: repoRev ?? null, - indexedAt: timestamp || new Date().toISOString(), - } - if (!record.did.startsWith('did:')) { - throw new Error('Expected indexed URI to contain DID') - } else if (record.collection.length < 1) { - throw new Error('Expected indexed URI to contain a collection') - } else if (record.rkey.length < 1) { - throw new Error('Expected indexed URI to contain a record key') - } - - // Track current version of record - await this.db.db - .insertInto('record') - .values(record) - .onConflict((oc) => - oc.column('uri').doUpdateSet({ - cid: record.cid, - repoRev: repoRev ?? null, - indexedAt: record.indexedAt, - }), - ) - .execute() - - // Maintain backlinks - const backlinks = getBacklinks(uri, obj) - if (action === WriteOpAction.Update) { - // On update just recreate backlinks from scratch for the record, so we can clear out - // the old ones. E.g. for weird cases like updating a follow to be for a different did. - await this.removeBacklinksByUri(uri) - } - await this.addBacklinks(backlinks) - - log.info({ uri }, 'indexed record') - } - - async deleteRecord(uri: AtUri) { - this.db.assertTransaction() - log.debug({ uri }, 'deleting indexed record') - const deleteQuery = this.db.db - .deleteFrom('record') - .where('uri', '=', uri.toString()) - const backlinkQuery = this.db.db - .deleteFrom('backlink') - .where('uri', '=', uri.toString()) - await Promise.all([deleteQuery.execute(), backlinkQuery.execute()]) +export class RecordReader { + constructor(public db: ActorDb) {} - log.info({ uri }, 'deleted indexed record') - } - - async listCollectionsForDid(did: string): Promise { + async listCollections(): Promise { const collections = await this.db.db .selectFrom('record') .select('collection') - .where('did', '=', did) .groupBy('collection') .execute() @@ -93,7 +22,6 @@ export class RecordService { } async listRecordsForCollection(opts: { - did: string collection: string limit: number reverse: boolean @@ -103,7 +31,6 @@ export class RecordService { includeSoftDeleted?: boolean }): Promise<{ uri: string; cid: string; value: object }[]> { const { - did, collection, limit, reverse, @@ -116,12 +43,7 @@ export class RecordService { const { ref } = this.db.db.dynamic let builder = this.db.db .selectFrom('record') - .innerJoin('ipld_block', (join) => - join - .onRef('ipld_block.cid', '=', 'record.cid') - .on('ipld_block.creator', '=', did), - ) - .where('record.did', '=', did) + .innerJoin('repo_block', 'repo_block.cid', 'record.cid') .where('record.collection', '=', collection) .if(!includeSoftDeleted, (qb) => qb.where(notSoftDeletedClause(ref('record'))), @@ -169,11 +91,7 @@ export class RecordService { const { ref } = this.db.db.dynamic let builder = this.db.db .selectFrom('record') - .innerJoin('ipld_block', (join) => - join - .onRef('ipld_block.cid', '=', 'record.cid') - .on('ipld_block.creator', '=', uri.host), - ) + .innerJoin('repo_block', 'repo_block.cid', 'record.cid') .where('record.uri', '=', uri.toString()) .selectAll() .if(!includeSoftDeleted, (qb) => @@ -213,56 +131,67 @@ export class RecordService { return !!record } - async deleteForActor(did: string) { - // Not done in transaction because it would be too long, prone to contention. - // Also, this can safely be run multiple times if it fails. - await this.db.db.deleteFrom('record').where('did', '=', did).execute() - } - - async removeBacklinksByUri(uri: AtUri) { - await this.db.db - .deleteFrom('backlink') + async getRecordTakedownStatus(uri: AtUri): Promise { + const res = await this.db.db + .selectFrom('record') + .select('takedownRef') .where('uri', '=', uri.toString()) - .execute() + .executeTakeFirst() + if (!res) return null + return res.takedownRef + ? { applied: true, ref: res.takedownRef } + : { applied: false } } - async addBacklinks(backlinks: Backlink[]) { - if (backlinks.length === 0) return - await this.db.db - .insertInto('backlink') - .values(backlinks) - .onConflict((oc) => oc.doNothing()) - .execute() + async getCurrentRecordCid(uri: AtUri): Promise { + const res = await this.db.db + .selectFrom('record') + .select('cid') + .where('uri', '=', uri.toString()) + .executeTakeFirst() + return res ? CID.parse(res.cid) : null } async getRecordBacklinks(opts: { - did: string collection: string path: string linkTo: string }) { - const { did, collection, path, linkTo } = opts + const { collection, path, linkTo } = opts return await this.db.db .selectFrom('record') .innerJoin('backlink', 'backlink.uri', 'record.uri') .where('backlink.path', '=', path) - .if(linkTo.startsWith('at://'), (q) => - q.where('backlink.linkToUri', '=', linkTo), - ) - .if(!linkTo.startsWith('at://'), (q) => - q.where('backlink.linkToDid', '=', linkTo), - ) - .where('record.did', '=', did) + .where('backlink.linkTo', '=', linkTo) .where('record.collection', '=', collection) .selectAll('record') .execute() } + + // @NOTE this logic is a placeholder until we allow users to specify these constraints themselves. + // Ensures that we don't end-up with duplicate likes, reposts, and follows from race conditions. + + async getBacklinkConflicts(uri: AtUri, record: RepoRecord): Promise { + const recordBacklinks = getBacklinks(uri, record) + const conflicts = await Promise.all( + recordBacklinks.map((backlink) => + this.getRecordBacklinks({ + collection: uri.collection, + path: backlink.path, + linkTo: backlink.linkTo, + }), + ), + ) + return conflicts + .flat() + .map(({ rkey }) => AtUri.make(uri.hostname, uri.collection, rkey)) + } } // @NOTE in the future this can be replaced with a more generic routine that pulls backlinks based on lex docs. // For now we just want to ensure we're tracking links from follows, blocks, likes, and reposts. -function getBacklinks(uri: AtUri, record: unknown): Backlink[] { +export const getBacklinks = (uri: AtUri, record: RepoRecord): Backlink[] => { if ( record?.['$type'] === ids.AppBskyGraphFollow || record?.['$type'] === ids.AppBskyGraphBlock @@ -272,7 +201,7 @@ function getBacklinks(uri: AtUri, record: unknown): Backlink[] { return [] } try { - ident.ensureValidDid(subject) + syntax.ensureValidDid(subject) } catch { return [] } @@ -280,8 +209,7 @@ function getBacklinks(uri: AtUri, record: unknown): Backlink[] { { uri: uri.toString(), path: 'subject', - linkToDid: subject, - linkToUri: null, + linkTo: subject, }, ] } @@ -290,7 +218,7 @@ function getBacklinks(uri: AtUri, record: unknown): Backlink[] { record?.['$type'] === ids.AppBskyFeedRepost ) { const subject = record['subject'] - if (typeof subject['uri'] !== 'string') { + if (typeof subject?.['uri'] !== 'string') { return [] } try { @@ -302,8 +230,7 @@ function getBacklinks(uri: AtUri, record: unknown): Backlink[] { { uri: uri.toString(), path: 'subject.uri', - linkToUri: subject.uri, - linkToDid: null, + linkTo: subject['uri'], }, ] } diff --git a/packages/pds/src/actor-store/record/transactor.ts b/packages/pds/src/actor-store/record/transactor.ts new file mode 100644 index 00000000000..7e67caec752 --- /dev/null +++ b/packages/pds/src/actor-store/record/transactor.ts @@ -0,0 +1,106 @@ +import { CID } from 'multiformats/cid' +import { AtUri } from '@atproto/syntax' +import { BlobStore, WriteOpAction } from '@atproto/repo' +import { dbLogger as log } from '../../logger' +import { ActorDb, Backlink } from '../db' +import { RecordReader, getBacklinks } from './reader' +import { StatusAttr } from '../../lexicon/types/com/atproto/admin/defs' +import { RepoRecord } from '@atproto/lexicon' + +export class RecordTransactor extends RecordReader { + constructor(public db: ActorDb, public blobstore: BlobStore) { + super(db) + } + + async indexRecord( + uri: AtUri, + cid: CID, + record: RepoRecord | null, + action: WriteOpAction.Create | WriteOpAction.Update = WriteOpAction.Create, + repoRev: string, + timestamp?: string, + ) { + log.debug({ uri }, 'indexing record') + const row = { + uri: uri.toString(), + cid: cid.toString(), + collection: uri.collection, + rkey: uri.rkey, + repoRev: repoRev, + indexedAt: timestamp || new Date().toISOString(), + } + if (!uri.hostname.startsWith('did:')) { + throw new Error('Expected indexed URI to contain DID') + } else if (row.collection.length < 1) { + throw new Error('Expected indexed URI to contain a collection') + } else if (row.rkey.length < 1) { + throw new Error('Expected indexed URI to contain a record key') + } + + // Track current version of record + await this.db.db + .insertInto('record') + .values(row) + .onConflict((oc) => + oc.column('uri').doUpdateSet({ + cid: row.cid, + repoRev: repoRev, + indexedAt: row.indexedAt, + }), + ) + .execute() + + if (record !== null) { + // Maintain backlinks + const backlinks = getBacklinks(uri, record) + if (action === WriteOpAction.Update) { + // On update just recreate backlinks from scratch for the record, so we can clear out + // the old ones. E.g. for weird cases like updating a follow to be for a different did. + await this.removeBacklinksByUri(uri) + } + await this.addBacklinks(backlinks) + } + + log.info({ uri }, 'indexed record') + } + + async deleteRecord(uri: AtUri) { + log.debug({ uri }, 'deleting indexed record') + const deleteQuery = this.db.db + .deleteFrom('record') + .where('uri', '=', uri.toString()) + const backlinkQuery = this.db.db + .deleteFrom('backlink') + .where('uri', '=', uri.toString()) + await Promise.all([deleteQuery.execute(), backlinkQuery.execute()]) + + log.info({ uri }, 'deleted indexed record') + } + + async removeBacklinksByUri(uri: AtUri) { + await this.db.db + .deleteFrom('backlink') + .where('uri', '=', uri.toString()) + .execute() + } + + async addBacklinks(backlinks: Backlink[]) { + if (backlinks.length === 0) return + await this.db.db + .insertInto('backlink') + .values(backlinks) + .onConflict((oc) => oc.doNothing()) + .execute() + } + + async updateRecordTakedownStatus(uri: AtUri, takedown: StatusAttr) { + const takedownRef = takedown.applied + ? takedown.ref ?? new Date().toISOString() + : null + await this.db.db + .updateTable('record') + .set({ takedownRef }) + .where('uri', '=', uri.toString()) + .executeTakeFirst() + } +} diff --git a/packages/pds/src/actor-store/repo/reader.ts b/packages/pds/src/actor-store/repo/reader.ts new file mode 100644 index 00000000000..07be5f83eff --- /dev/null +++ b/packages/pds/src/actor-store/repo/reader.ts @@ -0,0 +1,17 @@ +import { BlobStore } from '@atproto/repo' +import { SqlRepoReader } from './sql-repo-reader' +import { BlobReader } from '../blob/reader' +import { ActorDb } from '../db' +import { RecordReader } from '../record/reader' + +export class RepoReader { + blob: BlobReader + record: RecordReader + storage: SqlRepoReader + + constructor(public db: ActorDb, public blobstore: BlobStore) { + this.blob = new BlobReader(db, blobstore) + this.record = new RecordReader(db) + this.storage = new SqlRepoReader(db) + } +} diff --git a/packages/pds/src/actor-store/repo/sql-repo-reader.ts b/packages/pds/src/actor-store/repo/sql-repo-reader.ts new file mode 100644 index 00000000000..90b61b08e68 --- /dev/null +++ b/packages/pds/src/actor-store/repo/sql-repo-reader.ts @@ -0,0 +1,149 @@ +import { + BlockMap, + CidSet, + ReadableBlockstore, + writeCarStream, +} from '@atproto/repo' +import { chunkArray } from '@atproto/common' +import { CID } from 'multiformats/cid' +import { ActorDb } from '../db' +import { sql } from 'kysely' + +export class SqlRepoReader extends ReadableBlockstore { + cache: BlockMap = new BlockMap() + now: string + + constructor(public db: ActorDb) { + super() + } + + async getRoot(): Promise { + const root = await this.getRootDetailed() + return root?.cid ?? null + } + + async getRootDetailed(): Promise<{ cid: CID; rev: string }> { + const res = await this.db.db + .selectFrom('repo_root') + .selectAll() + .executeTakeFirstOrThrow() + return { + cid: CID.parse(res.cid), + rev: res.rev, + } + } + + async getBytes(cid: CID): Promise { + const cached = this.cache.get(cid) + if (cached) return cached + const found = await this.db.db + .selectFrom('repo_block') + .where('repo_block.cid', '=', cid.toString()) + .select('content') + .executeTakeFirst() + if (!found) return null + this.cache.set(cid, found.content) + return found.content + } + + async has(cid: CID): Promise { + const got = await this.getBytes(cid) + return !!got + } + + async getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }> { + const cached = this.cache.getMany(cids) + if (cached.missing.length < 1) return cached + const missing = new CidSet(cached.missing) + const missingStr = cached.missing.map((c) => c.toString()) + const blocks = new BlockMap() + await Promise.all( + chunkArray(missingStr, 500).map(async (batch) => { + const res = await this.db.db + .selectFrom('repo_block') + .where('repo_block.cid', 'in', batch) + .select(['repo_block.cid as cid', 'repo_block.content as content']) + .execute() + for (const row of res) { + const cid = CID.parse(row.cid) + blocks.set(cid, row.content) + missing.delete(cid) + } + }), + ) + this.cache.addMap(blocks) + blocks.addMap(cached.blocks) + return { blocks, missing: missing.toList() } + } + + async getCarStream(since?: string) { + const root = await this.getRoot() + if (!root) { + throw new RepoRootNotFoundError() + } + return writeCarStream(root, async (car) => { + let cursor: RevCursor | undefined = undefined + const writeRows = async ( + rows: { cid: string; content: Uint8Array }[], + ) => { + for (const row of rows) { + await car.put({ + cid: CID.parse(row.cid), + bytes: row.content, + }) + } + } + // allow us to write to car while fetching the next page + let writePromise: Promise = Promise.resolve() + do { + const res = await this.getBlockRange(since, cursor) + await writePromise + writePromise = writeRows(res) + const lastRow = res.at(-1) + if (lastRow && lastRow.repoRev) { + cursor = { + cid: CID.parse(lastRow.cid), + rev: lastRow.repoRev, + } + } else { + cursor = undefined + } + } while (cursor) + // ensure we flush the last page of blocks + await writePromise + }) + } + + async getBlockRange(since?: string, cursor?: RevCursor) { + const { ref } = this.db.db.dynamic + let builder = this.db.db + .selectFrom('repo_block') + .select(['cid', 'repoRev', 'content']) + .orderBy('repoRev', 'desc') + .orderBy('cid', 'desc') + .limit(500) + if (cursor) { + // use this syntax to ensure we hit the index + builder = builder.where( + sql`((${ref('repoRev')}, ${ref('cid')}) < (${ + cursor.rev + }, ${cursor.cid.toString()}))`, + ) + } + if (since) { + builder = builder.where('repoRev', '>', since) + } + return builder.execute() + } + + async destroy(): Promise { + throw new Error('Destruction of SQL repo storage not allowed at runtime') + } +} + +type RevCursor = { + cid: CID + rev: string +} + +export class RepoRootNotFoundError extends Error {} diff --git a/packages/pds/src/actor-store/repo/sql-repo-transactor.ts b/packages/pds/src/actor-store/repo/sql-repo-transactor.ts new file mode 100644 index 00000000000..0be7e0ebcb3 --- /dev/null +++ b/packages/pds/src/actor-store/repo/sql-repo-transactor.ts @@ -0,0 +1,107 @@ +import { CommitData, RepoStorage, BlockMap } from '@atproto/repo' +import { chunkArray } from '@atproto/common' +import { CID } from 'multiformats/cid' +import { ActorDb, RepoBlock } from '../db' +import { SqlRepoReader } from './sql-repo-reader' + +export class SqlRepoTransactor extends SqlRepoReader implements RepoStorage { + cache: BlockMap = new BlockMap() + now: string + + constructor(public db: ActorDb, public did: string, now?: string) { + super(db) + this.now = now ?? new Date().toISOString() + } + + // proactively cache all blocks from a particular commit (to prevent multiple roundtrips) + async cacheRev(rev: string): Promise { + const res = await this.db.db + .selectFrom('repo_block') + .where('repoRev', '=', rev) + .select(['repo_block.cid', 'repo_block.content']) + .limit(15) + .execute() + for (const row of res) { + this.cache.set(CID.parse(row.cid), row.content) + } + } + + async putBlock(cid: CID, block: Uint8Array, rev: string): Promise { + await this.db.db + .insertInto('repo_block') + .values({ + cid: cid.toString(), + repoRev: rev, + size: block.length, + content: block, + }) + .onConflict((oc) => oc.doNothing()) + .execute() + this.cache.set(cid, block) + } + + async putMany(toPut: BlockMap, rev: string): Promise { + const blocks: RepoBlock[] = [] + toPut.forEach((bytes, cid) => { + blocks.push({ + cid: cid.toString(), + repoRev: rev, + size: bytes.length, + content: bytes, + }) + }) + await Promise.all( + chunkArray(blocks, 50).map((batch) => + this.db.db + .insertInto('repo_block') + .values(batch) + .onConflict((oc) => oc.doNothing()) + .execute(), + ), + ) + } + + async deleteMany(cids: CID[]) { + if (cids.length < 1) return + const cidStrs = cids.map((c) => c.toString()) + await this.db.db + .deleteFrom('repo_block') + .where('cid', 'in', cidStrs) + .execute() + } + + async applyCommit(commit: CommitData, isCreate?: boolean) { + await Promise.all([ + this.updateRoot(commit.cid, commit.rev, isCreate), + this.putMany(commit.newBlocks, commit.rev), + this.deleteMany(commit.removedCids.toList()), + ]) + } + + async updateRoot(cid: CID, rev: string, isCreate = false): Promise { + if (isCreate) { + await this.db.db + .insertInto('repo_root') + .values({ + did: this.did, + cid: cid.toString(), + rev: rev, + indexedAt: this.now, + }) + .execute() + } else { + await this.db.db + .updateTable('repo_root') + .set({ + cid: cid.toString(), + rev: rev, + indexedAt: this.now, + }) + .execute() + } + } + + async destroy(): Promise { + throw new Error('Destruction of SQL repo storage not allowed at runtime') + } +} diff --git a/packages/pds/src/actor-store/repo/transactor.ts b/packages/pds/src/actor-store/repo/transactor.ts new file mode 100644 index 00000000000..a095e46cd4e --- /dev/null +++ b/packages/pds/src/actor-store/repo/transactor.ts @@ -0,0 +1,181 @@ +import { CID } from 'multiformats/cid' +import * as crypto from '@atproto/crypto' +import { BlobStore, CommitData, Repo, WriteOpAction } from '@atproto/repo' +import { InvalidRequestError } from '@atproto/xrpc-server' +import { AtUri } from '@atproto/syntax' +import { SqlRepoTransactor } from './sql-repo-transactor' +import { + BadCommitSwapError, + BadRecordSwapError, + PreparedCreate, + PreparedWrite, +} from '../../repo/types' +import { BlobTransactor } from '../blob/transactor' +import { createWriteToOp, writeToOp } from '../../repo' +import { BackgroundQueue } from '../../background' +import { ActorDb } from '../db' +import { RecordTransactor } from '../record/transactor' +import { RepoReader } from './reader' + +export class RepoTransactor extends RepoReader { + blob: BlobTransactor + record: RecordTransactor + storage: SqlRepoTransactor + now: string + + constructor( + public db: ActorDb, + public did: string, + public signingKey: crypto.Keypair, + public blobstore: BlobStore, + public backgroundQueue: BackgroundQueue, + now?: string, + ) { + super(db, blobstore) + this.blob = new BlobTransactor(db, blobstore, backgroundQueue) + this.record = new RecordTransactor(db, blobstore) + this.now = now ?? new Date().toISOString() + this.storage = new SqlRepoTransactor(db, this.did, this.now) + } + + async createRepo(writes: PreparedCreate[]): Promise { + this.db.assertTransaction() + const writeOps = writes.map(createWriteToOp) + const commit = await Repo.formatInitCommit( + this.storage, + this.did, + this.signingKey, + writeOps, + ) + await Promise.all([ + this.storage.applyCommit(commit, true), + this.indexWrites(writes, commit.rev), + this.blob.processWriteBlobs(commit.rev, writes), + ]) + return commit + } + + async processWrites(writes: PreparedWrite[], swapCommitCid?: CID) { + this.db.assertTransaction() + const commit = await this.formatCommit(writes, swapCommitCid) + await Promise.all([ + // persist the commit to repo storage + this.storage.applyCommit(commit), + // & send to indexing + this.indexWrites(writes, commit.rev), + // process blobs + this.blob.processWriteBlobs(commit.rev, writes), + ]) + return commit + } + + async formatCommit( + writes: PreparedWrite[], + swapCommit?: CID, + ): Promise { + // this is not in a txn, so this won't actually hold the lock, + // we just check if it is currently held by another txn + const currRoot = await this.storage.getRootDetailed() + if (!currRoot) { + throw new InvalidRequestError(`No repo root found for ${this.did}`) + } + if (swapCommit && !currRoot.cid.equals(swapCommit)) { + throw new BadCommitSwapError(currRoot.cid) + } + // cache last commit since there's likely overlap + await this.storage.cacheRev(currRoot.rev) + const newRecordCids: CID[] = [] + const delAndUpdateUris: AtUri[] = [] + for (const write of writes) { + const { action, uri, swapCid } = write + if (action !== WriteOpAction.Delete) { + newRecordCids.push(write.cid) + } + if (action !== WriteOpAction.Create) { + delAndUpdateUris.push(uri) + } + if (swapCid === undefined) { + continue + } + const record = await this.record.getRecord(uri, null, true) + const currRecord = record && CID.parse(record.cid) + if (action === WriteOpAction.Create && swapCid !== null) { + throw new BadRecordSwapError(currRecord) // There should be no current record for a create + } + if (action === WriteOpAction.Update && swapCid === null) { + throw new BadRecordSwapError(currRecord) // There should be a current record for an update + } + if (action === WriteOpAction.Delete && swapCid === null) { + throw new BadRecordSwapError(currRecord) // There should be a current record for a delete + } + if ((currRecord || swapCid) && !currRecord?.equals(swapCid)) { + throw new BadRecordSwapError(currRecord) + } + } + + const repo = await Repo.load(this.storage, currRoot.cid) + const writeOps = writes.map(writeToOp) + const commit = await repo.formatCommit(writeOps, this.signingKey) + + // find blocks that would be deleted but are referenced by another record + const dupeRecordCids = await this.getDuplicateRecordCids( + commit.removedCids.toList(), + delAndUpdateUris, + ) + for (const cid of dupeRecordCids) { + commit.removedCids.delete(cid) + } + + // find blocks that are relevant to ops but not included in diff + // (for instance a record that was moved but cid stayed the same) + const newRecordBlocks = commit.newBlocks.getMany(newRecordCids) + if (newRecordBlocks.missing.length > 0) { + const missingBlocks = await this.storage.getBlocks( + newRecordBlocks.missing, + ) + commit.newBlocks.addMap(missingBlocks.blocks) + } + return commit + } + + async indexWrites(writes: PreparedWrite[], rev: string) { + this.db.assertTransaction() + await Promise.all( + writes.map(async (write) => { + if ( + write.action === WriteOpAction.Create || + write.action === WriteOpAction.Update + ) { + await this.record.indexRecord( + write.uri, + write.cid, + write.record, + write.action, + rev, + this.now, + ) + } else if (write.action === WriteOpAction.Delete) { + await this.record.deleteRecord(write.uri) + } + }), + ) + } + + async getDuplicateRecordCids( + cids: CID[], + touchedUris: AtUri[], + ): Promise { + if (touchedUris.length === 0 || cids.length === 0) { + return [] + } + const cidStrs = cids.map((c) => c.toString()) + const uriStrs = touchedUris.map((u) => u.toString()) + const res = await this.db.db + .selectFrom('record') + .where('cid', 'in', cidStrs) + .where('uri', 'not in', uriStrs) + .select('cid') + .execute() + return res.map((row) => CID.parse(row.cid)) + } +} diff --git a/packages/pds/src/api/app/bsky/actor/getPreferences.ts b/packages/pds/src/api/app/bsky/actor/getPreferences.ts index cb595508764..d89666358ae 100644 --- a/packages/pds/src/api/app/bsky/actor/getPreferences.ts +++ b/packages/pds/src/api/app/bsky/actor/getPreferences.ts @@ -7,10 +7,9 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.authVerifier.access, handler: async ({ auth }) => { const requester = auth.credentials.did - const { services, db } = ctx - let preferences = await services - .account(db) - .getPreferences(requester, 'app.bsky') + let preferences = await ctx.actorStore.read(requester, (store) => + store.pref.getPreferences('app.bsky'), + ) if (auth.credentials.scope !== AuthScope.Access) { // filter out personal details for app passwords preferences = preferences.filter( diff --git a/packages/pds/src/api/app/bsky/actor/getProfile.ts b/packages/pds/src/api/app/bsky/actor/getProfile.ts index 9dd4b028983..4c4f0958abe 100644 --- a/packages/pds/src/api/app/bsky/actor/getProfile.ts +++ b/packages/pds/src/api/app/bsky/actor/getProfile.ts @@ -1,9 +1,12 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { authPassthru } from '../../../../api/com/atproto/admin/util' +import { authPassthru } from '../../../proxy' import { OutputSchema } from '../../../../lexicon/types/app/bsky/actor/getProfile' -import { handleReadAfterWrite } from '../util/read-after-write' -import { LocalRecords } from '../../../../services/local' +import { + LocalViewer, + handleReadAfterWrite, + LocalRecords, +} from '../../../../read-after-write' export default function (server: Server, ctx: AppContext) { server.app.bsky.actor.getProfile({ @@ -27,12 +30,10 @@ export default function (server: Server, ctx: AppContext) { } const getProfileMunge = async ( - ctx: AppContext, + localViewer: LocalViewer, original: OutputSchema, local: LocalRecords, ): Promise => { if (!local.profile) return original - return ctx.services - .local(ctx.db) - .updateProfileDetailed(original, local.profile.record) + return localViewer.updateProfileDetailed(original, local.profile.record) } diff --git a/packages/pds/src/api/app/bsky/actor/getProfiles.ts b/packages/pds/src/api/app/bsky/actor/getProfiles.ts index 0af2bff9564..bc78a26044e 100644 --- a/packages/pds/src/api/app/bsky/actor/getProfiles.ts +++ b/packages/pds/src/api/app/bsky/actor/getProfiles.ts @@ -1,8 +1,11 @@ import AppContext from '../../../../context' import { Server } from '../../../../lexicon' import { OutputSchema } from '../../../../lexicon/types/app/bsky/actor/getProfiles' -import { LocalRecords } from '../../../../services/local' -import { handleReadAfterWrite } from '../util/read-after-write' +import { + LocalViewer, + handleReadAfterWrite, + LocalRecords, +} from '../../../../read-after-write' export default function (server: Server, ctx: AppContext) { server.app.bsky.actor.getProfiles({ @@ -26,7 +29,7 @@ export default function (server: Server, ctx: AppContext) { } const getProfilesMunge = async ( - ctx: AppContext, + localViewer: LocalViewer, original: OutputSchema, local: LocalRecords, requester: string, @@ -35,9 +38,7 @@ const getProfilesMunge = async ( if (!localProf) return original const profiles = original.profiles.map((prof) => { if (prof.did !== requester) return prof - return ctx.services - .local(ctx.db) - .updateProfileDetailed(prof, localProf.record) + return localViewer.updateProfileDetailed(prof, localProf.record) }) return { ...original, diff --git a/packages/pds/src/api/app/bsky/actor/putPreferences.ts b/packages/pds/src/api/app/bsky/actor/putPreferences.ts index 7f7bdd86e39..f6274d42278 100644 --- a/packages/pds/src/api/app/bsky/actor/putPreferences.ts +++ b/packages/pds/src/api/app/bsky/actor/putPreferences.ts @@ -1,7 +1,7 @@ +import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { UserPreference } from '../../../../services/account' -import { InvalidRequestError } from '@atproto/xrpc-server' +import { AccountPreference } from '../../../../actor-store/preference/reader' export default function (server: Server, ctx: AppContext) { server.app.bsky.actor.putPreferences({ @@ -9,19 +9,16 @@ export default function (server: Server, ctx: AppContext) { handler: async ({ auth, input }) => { const { preferences } = input.body const requester = auth.credentials.did - const { services, db } = ctx - const checkedPreferences: UserPreference[] = [] + const checkedPreferences: AccountPreference[] = [] for (const pref of preferences) { if (typeof pref.$type === 'string') { - checkedPreferences.push(pref as UserPreference) + checkedPreferences.push(pref as AccountPreference) } else { throw new InvalidRequestError('Preference is missing a $type') } } - await db.transaction(async (tx) => { - await services - .account(tx) - .putPreferences(requester, checkedPreferences, 'app.bsky') + await ctx.actorStore.transact(requester, async (actorTxn) => { + await actorTxn.pref.putPreferences(checkedPreferences, 'app.bsky') }) }, }) diff --git a/packages/pds/src/api/app/bsky/feed/getActorLikes.ts b/packages/pds/src/api/app/bsky/feed/getActorLikes.ts index 4029dbc74dd..d0d18787f9d 100644 --- a/packages/pds/src/api/app/bsky/feed/getActorLikes.ts +++ b/packages/pds/src/api/app/bsky/feed/getActorLikes.ts @@ -1,9 +1,12 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' +import { authPassthru } from '../../../proxy' import { OutputSchema } from '../../../../lexicon/types/app/bsky/feed/getAuthorFeed' -import { handleReadAfterWrite } from '../util/read-after-write' -import { authPassthru } from '../../../../api/com/atproto/admin/util' -import { LocalRecords } from '../../../../services/local' +import { + LocalViewer, + handleReadAfterWrite, + LocalRecords, +} from '../../../../read-after-write' export default function (server: Server, ctx: AppContext) { server.app.bsky.feed.getActorLikes({ @@ -28,12 +31,11 @@ export default function (server: Server, ctx: AppContext) { } const getAuthorMunge = async ( - ctx: AppContext, + localViewer: LocalViewer, original: OutputSchema, local: LocalRecords, requester: string, ): Promise => { - const localSrvc = ctx.services.local(ctx.db) const localProf = local.profile let feed = original.feed // first update any out of date profile pictures in feed @@ -44,7 +46,7 @@ const getAuthorMunge = async ( ...item, post: { ...item.post, - author: localSrvc.updateProfileViewBasic( + author: localViewer.updateProfileViewBasic( item.post.author, localProf.record, ), diff --git a/packages/pds/src/api/app/bsky/feed/getAuthorFeed.ts b/packages/pds/src/api/app/bsky/feed/getAuthorFeed.ts index d887d5bae89..26c001990e3 100644 --- a/packages/pds/src/api/app/bsky/feed/getAuthorFeed.ts +++ b/packages/pds/src/api/app/bsky/feed/getAuthorFeed.ts @@ -1,10 +1,13 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' +import { authPassthru } from '../../../proxy' import { OutputSchema } from '../../../../lexicon/types/app/bsky/feed/getAuthorFeed' -import { handleReadAfterWrite } from '../util/read-after-write' -import { authPassthru } from '../../../../api/com/atproto/admin/util' -import { LocalRecords } from '../../../../services/local' import { isReasonRepost } from '../../../../lexicon/types/app/bsky/feed/defs' +import { + LocalViewer, + handleReadAfterWrite, + LocalRecords, +} from '../../../../read-after-write' export default function (server: Server, ctx: AppContext) { server.app.bsky.feed.getAuthorFeed({ @@ -28,12 +31,11 @@ export default function (server: Server, ctx: AppContext) { } const getAuthorMunge = async ( - ctx: AppContext, + localViewer: LocalViewer, original: OutputSchema, local: LocalRecords, requester: string, ): Promise => { - const localSrvc = ctx.services.local(ctx.db) const localProf = local.profile // only munge on own feed if (!isUsersFeed(original, requester)) { @@ -48,7 +50,7 @@ const getAuthorMunge = async ( ...item, post: { ...item.post, - author: localSrvc.updateProfileViewBasic( + author: localViewer.updateProfileViewBasic( item.post.author, localProf.record, ), @@ -59,7 +61,7 @@ const getAuthorMunge = async ( } }) } - feed = await localSrvc.formatAndInsertPostsInFeed(feed, local.posts) + feed = await localViewer.formatAndInsertPostsInFeed(feed, local.posts) return { ...original, feed, diff --git a/packages/pds/src/api/app/bsky/feed/getPostThread.ts b/packages/pds/src/api/app/bsky/feed/getPostThread.ts index 01670e9e9ad..93cd5fd641f 100644 --- a/packages/pds/src/api/app/bsky/feed/getPostThread.ts +++ b/packages/pds/src/api/app/bsky/feed/getPostThread.ts @@ -3,6 +3,7 @@ import { AppBskyFeedGetPostThread } from '@atproto/api' import { Headers } from '@atproto/xrpc' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' +import { authPassthru } from '../../../proxy' import { ThreadViewPost, isThreadViewPost, @@ -13,16 +14,13 @@ import { QueryParams, } from '../../../../lexicon/types/app/bsky/feed/getPostThread' import { - LocalRecords, - LocalService, - RecordDescript, -} from '../../../../services/local' -import { + LocalViewer, getLocalLag, getRepoRev, handleReadAfterWrite, -} from '../util/read-after-write' -import { authPassthru } from '../../../com/atproto/admin/util' + LocalRecords, + RecordDescript, +} from '../../../../read-after-write' export default function (server: Server, ctx: AppContext) { server.app.bsky.feed.getPostThread({ @@ -57,12 +55,18 @@ export default function (server: Server, ctx: AppContext) { ) } catch (err) { if (err instanceof AppBskyFeedGetPostThread.NotFoundError) { - const local = await readAfterWriteNotFound( - ctx, - params, - requester, - err.headers, - ) + const headers = err.headers + const keypair = await ctx.actorStore.keypair(requester) + const local = await ctx.actorStore.read(requester, (store) => { + const localViewer = ctx.localViewer(store, keypair) + return readAfterWriteNotFound( + ctx, + localViewer, + params, + requester, + headers, + ) + }) if (local === null) { throw err } else { @@ -88,7 +92,7 @@ export default function (server: Server, ctx: AppContext) { // ---------------- const getPostThreadMunge = async ( - ctx: AppContext, + localViewer: LocalViewer, original: OutputSchema, local: LocalRecords, ): Promise => { @@ -98,7 +102,7 @@ const getPostThreadMunge = async ( return original } const thread = await addPostsToThread( - ctx.services.local(ctx.db), + localViewer, original.thread, local.posts, ) @@ -109,7 +113,7 @@ const getPostThreadMunge = async ( } const addPostsToThread = async ( - localSrvc: LocalService, + localViewer: LocalViewer, original: ThreadViewPost, posts: RecordDescript[], ) => { @@ -117,7 +121,7 @@ const addPostsToThread = async ( if (inThread.length === 0) return original let thread: ThreadViewPost = original for (const record of inThread) { - thread = await insertIntoThreadReplies(localSrvc, thread, record) + thread = await insertIntoThreadReplies(localViewer, thread, record) } return thread } @@ -135,12 +139,12 @@ const findPostsInThread = ( } const insertIntoThreadReplies = async ( - localSrvc: LocalService, + localViewer: LocalViewer, view: ThreadViewPost, descript: RecordDescript, ): Promise => { if (descript.record.reply?.parent.uri === view.post.uri) { - const postView = await threadPostView(localSrvc, descript) + const postView = await threadPostView(localViewer, descript) if (!postView) return view const replies = [postView, ...(view.replies ?? [])] return { @@ -152,7 +156,7 @@ const insertIntoThreadReplies = async ( const replies = await Promise.all( view.replies.map(async (reply) => isThreadViewPost(reply) - ? await insertIntoThreadReplies(localSrvc, reply, descript) + ? await insertIntoThreadReplies(localViewer, reply, descript) : reply, ), ) @@ -163,10 +167,10 @@ const insertIntoThreadReplies = async ( } const threadPostView = async ( - localSrvc: LocalService, + localViewer: LocalViewer, descript: RecordDescript, ): Promise => { - const postView = await localSrvc.getPost(descript) + const postView = await localViewer.getPost(descript) if (!postView) return null return { $type: 'app.bsky.feed.defs#threadViewPost', @@ -179,6 +183,7 @@ const threadPostView = async ( const readAfterWriteNotFound = async ( ctx: AppContext, + localViewer: LocalViewer, params: QueryParams, requester: string, headers?: Headers, @@ -190,14 +195,13 @@ const readAfterWriteNotFound = async ( if (uri.hostname !== requester) { return null } - const localSrvc = ctx.services.local(ctx.db) - const local = await localSrvc.getRecordsSinceRev(requester, rev) + const local = await localViewer.getRecordsSinceRev(rev) const found = local.posts.find((p) => p.uri.toString() === uri.toString()) if (!found) return null - let thread = await threadPostView(localSrvc, found) + let thread = await threadPostView(localViewer, found) if (!thread) return null const rest = local.posts.filter((p) => p.uri.toString() !== uri.toString()) - thread = await addPostsToThread(localSrvc, thread, rest) + thread = await addPostsToThread(localViewer, thread, rest) const highestParent = getHighestParent(thread) if (highestParent) { try { diff --git a/packages/pds/src/api/app/bsky/feed/getTimeline.ts b/packages/pds/src/api/app/bsky/feed/getTimeline.ts index 33cdafb9f25..d8515bcbfbb 100644 --- a/packages/pds/src/api/app/bsky/feed/getTimeline.ts +++ b/packages/pds/src/api/app/bsky/feed/getTimeline.ts @@ -1,8 +1,11 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { OutputSchema } from '../../../../lexicon/types/app/bsky/feed/getTimeline' -import { handleReadAfterWrite } from '../util/read-after-write' -import { LocalRecords } from '../../../../services/local' +import { + LocalViewer, + handleReadAfterWrite, + LocalRecords, +} from '../../../../read-after-write' export default function (server: Server, ctx: AppContext) { server.app.bsky.feed.getTimeline({ @@ -19,13 +22,14 @@ export default function (server: Server, ctx: AppContext) { } const getTimelineMunge = async ( - ctx: AppContext, + localViewer: LocalViewer, original: OutputSchema, local: LocalRecords, ): Promise => { - const feed = await ctx.services - .local(ctx.db) - .formatAndInsertPostsInFeed([...original.feed], local.posts) + const feed = await localViewer.formatAndInsertPostsInFeed( + [...original.feed], + local.posts, + ) return { ...original, feed, diff --git a/packages/pds/src/api/app/bsky/graph/getFollowers.ts b/packages/pds/src/api/app/bsky/graph/getFollowers.ts index fd3a7612454..e3e5d8ba64e 100644 --- a/packages/pds/src/api/app/bsky/graph/getFollowers.ts +++ b/packages/pds/src/api/app/bsky/graph/getFollowers.ts @@ -1,6 +1,6 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { authPassthru } from '../../../../api/com/atproto/admin/util' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.app.bsky.graph.getFollowers({ diff --git a/packages/pds/src/api/app/bsky/graph/getFollows.ts b/packages/pds/src/api/app/bsky/graph/getFollows.ts index 31936dc879e..821b83fa359 100644 --- a/packages/pds/src/api/app/bsky/graph/getFollows.ts +++ b/packages/pds/src/api/app/bsky/graph/getFollows.ts @@ -1,6 +1,6 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { authPassthru } from '../../../../api/com/atproto/admin/util' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.app.bsky.graph.getFollows({ diff --git a/packages/pds/src/api/com/atproto/admin/deleteAccount.ts b/packages/pds/src/api/com/atproto/admin/deleteAccount.ts new file mode 100644 index 00000000000..420fbaecb23 --- /dev/null +++ b/packages/pds/src/api/com/atproto/admin/deleteAccount.ts @@ -0,0 +1,19 @@ +import { AuthRequiredError } from '@atproto/xrpc-server' +import { Server } from '../../../../lexicon' +import AppContext from '../../../../context' + +export default function (server: Server, ctx: AppContext) { + server.com.atproto.admin.deleteAccount({ + auth: ctx.authVerifier.role, + handler: async ({ input, auth }) => { + if (!auth.credentials.admin) { + throw new AuthRequiredError('Must be an admin to delete an account') + } + const { did } = input.body + await ctx.actorStore.destroy(did) + await ctx.accountManager.deleteAccount(did) + await ctx.sequencer.sequenceTombstone(did) + await ctx.sequencer.deleteAllForUser(did) + }, + }) +} diff --git a/packages/pds/src/api/com/atproto/admin/disableAccountInvites.ts b/packages/pds/src/api/com/atproto/admin/disableAccountInvites.ts index 17e0ac01198..a29508776e9 100644 --- a/packages/pds/src/api/com/atproto/admin/disableAccountInvites.ts +++ b/packages/pds/src/api/com/atproto/admin/disableAccountInvites.ts @@ -1,4 +1,4 @@ -import { AuthRequiredError } from '@atproto/xrpc-server' +import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' @@ -6,15 +6,16 @@ export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.disableAccountInvites({ auth: ctx.authVerifier.role, handler: async ({ input, auth }) => { + if (ctx.cfg.entryway) { + throw new InvalidRequestError( + 'Account invites are managed by the entryway service', + ) + } if (!auth.credentials.admin) { throw new AuthRequiredError('Insufficient privileges') } - const { account, note } = input.body - await ctx.db.db - .updateTable('user_account') - .where('did', '=', account) - .set({ invitesDisabled: 1, inviteNote: note?.trim() || null }) - .execute() + const { account } = input.body + await ctx.accountManager.setAccountInvitesDisabled(account, true) }, }) } diff --git a/packages/pds/src/api/com/atproto/admin/disableInviteCodes.ts b/packages/pds/src/api/com/atproto/admin/disableInviteCodes.ts index d9d8516b88f..abb8b34a8c7 100644 --- a/packages/pds/src/api/com/atproto/admin/disableInviteCodes.ts +++ b/packages/pds/src/api/com/atproto/admin/disableInviteCodes.ts @@ -6,6 +6,11 @@ export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.disableInviteCodes({ auth: ctx.authVerifier.role, handler: async ({ input, auth }) => { + if (ctx.cfg.entryway) { + throw new InvalidRequestError( + 'Account invites are managed by the entryway service', + ) + } if (!auth.credentials.admin) { throw new AuthRequiredError('Insufficient privileges') } @@ -13,20 +18,7 @@ export default function (server: Server, ctx: AppContext) { if (accounts.includes('admin')) { throw new InvalidRequestError('cannot disable admin invite codes') } - if (codes.length > 0) { - await ctx.db.db - .updateTable('invite_code') - .set({ disabled: 1 }) - .where('code', 'in', codes) - .execute() - } - if (accounts.length > 0) { - await ctx.db.db - .updateTable('invite_code') - .set({ disabled: 1 }) - .where('forUser', 'in', accounts) - .execute() - } + await ctx.accountManager.disableInviteCodes({ codes, accounts }) }, }) } diff --git a/packages/pds/src/api/com/atproto/admin/emitModerationEvent.ts b/packages/pds/src/api/com/atproto/admin/emitModerationEvent.ts index d615dd0766d..cea8a9fb664 100644 --- a/packages/pds/src/api/com/atproto/admin/emitModerationEvent.ts +++ b/packages/pds/src/api/com/atproto/admin/emitModerationEvent.ts @@ -1,6 +1,6 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { authPassthru } from './util' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.emitModerationEvent({ diff --git a/packages/pds/src/api/com/atproto/admin/enableAccountInvites.ts b/packages/pds/src/api/com/atproto/admin/enableAccountInvites.ts index 073404455f1..a067923b861 100644 --- a/packages/pds/src/api/com/atproto/admin/enableAccountInvites.ts +++ b/packages/pds/src/api/com/atproto/admin/enableAccountInvites.ts @@ -1,4 +1,4 @@ -import { AuthRequiredError } from '@atproto/xrpc-server' +import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' @@ -6,15 +6,16 @@ export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.enableAccountInvites({ auth: ctx.authVerifier.role, handler: async ({ input, auth }) => { + if (ctx.cfg.entryway) { + throw new InvalidRequestError( + 'Account invites are managed by the entryway service', + ) + } if (!auth.credentials.admin) { throw new AuthRequiredError('Insufficient privileges') } - const { account, note } = input.body - await ctx.db.db - .updateTable('user_account') - .where('did', '=', account) - .set({ invitesDisabled: 0, inviteNote: note?.trim() || null }) - .execute() + const { account } = input.body + await ctx.accountManager.setAccountInvitesDisabled(account, false) }, }) } diff --git a/packages/pds/src/api/com/atproto/admin/getAccountInfo.ts b/packages/pds/src/api/com/atproto/admin/getAccountInfo.ts index cf751d08df4..9953dc9d56b 100644 --- a/packages/pds/src/api/com/atproto/admin/getAccountInfo.ts +++ b/packages/pds/src/api/com/atproto/admin/getAccountInfo.ts @@ -2,6 +2,7 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { InvalidRequestError } from '@atproto/xrpc-server' import { ensureValidAdminAud } from '../../../../auth-verifier' +import { INVALID_HANDLE } from '@atproto/syntax' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.getAccountInfo({ @@ -9,13 +10,29 @@ export default function (server: Server, ctx: AppContext) { handler: async ({ params, auth }) => { // any admin role auth can get account info, but verify aud on service jwt ensureValidAdminAud(auth, params.did) - const view = await ctx.services.account(ctx.db).adminView(params.did) - if (!view) { + const [account, invites, invitedBy] = await Promise.all([ + ctx.accountManager.getAccount(params.did, true), + ctx.accountManager.getAccountInvitesCodes(params.did), + ctx.accountManager.getInvitedByForAccounts([params.did]), + ]) + if (!account) { throw new InvalidRequestError('Account not found', 'NotFound') } + const managesOwnInvites = !ctx.cfg.entryway return { encoding: 'application/json', - body: view, + body: { + did: account.did, + handle: account.handle ?? INVALID_HANDLE, + email: account.email ?? undefined, + indexedAt: account.createdAt, + emailConfirmedAt: account.emailConfirmedAt ?? undefined, + invitedBy: managesOwnInvites ? invitedBy[params.did] : undefined, + invites: managesOwnInvites ? invites : undefined, + invitesDisabled: managesOwnInvites + ? account.invitesDisabled === 1 + : undefined, + }, } }, }) diff --git a/packages/pds/src/api/com/atproto/admin/getInviteCodes.ts b/packages/pds/src/api/com/atproto/admin/getInviteCodes.ts index 2c584281d5a..7647dc08813 100644 --- a/packages/pds/src/api/com/atproto/admin/getInviteCodes.ts +++ b/packages/pds/src/api/com/atproto/admin/getInviteCodes.ts @@ -7,13 +7,20 @@ import { GenericKeyset, paginate, } from '../../../../db/pagination' +import { selectInviteCodesQb } from '../../../../account-manager/helpers/invite' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.getInviteCodes({ auth: ctx.authVerifier.role, handler: async ({ params }) => { + if (ctx.cfg.entryway) { + throw new InvalidRequestError( + 'Account invites are managed by the entryway service', + ) + } const { sort, limit, cursor } = params - const ref = ctx.db.db.dynamic.ref + const db = ctx.accountManager.db + const ref = db.db.dynamic.ref let keyset if (sort === 'recent') { keyset = new TimeCodeKeyset(ref('createdAt'), ref('code')) @@ -23,9 +30,7 @@ export default function (server: Server, ctx: AppContext) { throw new InvalidRequestError(`unknown sort method: ${sort}`) } - const accntSrvc = ctx.services.account(ctx.db) - - let builder = accntSrvc.selectInviteCodesQb() + let builder = selectInviteCodesQb(db) builder = paginate(builder, { limit, cursor, @@ -35,7 +40,7 @@ export default function (server: Server, ctx: AppContext) { const res = await builder.execute() const codes = res.map((row) => row.code) - const uses = await accntSrvc.getInviteCodesUses(codes) + const uses = await ctx.accountManager.getInviteCodesUses(codes) const resultCursor = keyset.packFromResult(res) const codeDetails = res.map((row) => ({ diff --git a/packages/pds/src/api/com/atproto/admin/getModerationEvent.ts b/packages/pds/src/api/com/atproto/admin/getModerationEvent.ts index 6984a2acbec..3ac6e0f72be 100644 --- a/packages/pds/src/api/com/atproto/admin/getModerationEvent.ts +++ b/packages/pds/src/api/com/atproto/admin/getModerationEvent.ts @@ -1,6 +1,6 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { authPassthru } from './util' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.getModerationEvent({ diff --git a/packages/pds/src/api/com/atproto/admin/getRecord.ts b/packages/pds/src/api/com/atproto/admin/getRecord.ts index 30075a1d2ab..9b6860ca1f2 100644 --- a/packages/pds/src/api/com/atproto/admin/getRecord.ts +++ b/packages/pds/src/api/com/atproto/admin/getRecord.ts @@ -1,6 +1,6 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { authPassthru } from './util' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.getRecord({ diff --git a/packages/pds/src/api/com/atproto/admin/getRepo.ts b/packages/pds/src/api/com/atproto/admin/getRepo.ts index 3eb2e7c14c8..f70ddc7e0fe 100644 --- a/packages/pds/src/api/com/atproto/admin/getRepo.ts +++ b/packages/pds/src/api/com/atproto/admin/getRepo.ts @@ -1,6 +1,6 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { authPassthru } from './util' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.getRepo({ diff --git a/packages/pds/src/api/com/atproto/admin/getSubjectStatus.ts b/packages/pds/src/api/com/atproto/admin/getSubjectStatus.ts index 20ded7bc747..ad212391c58 100644 --- a/packages/pds/src/api/com/atproto/admin/getSubjectStatus.ts +++ b/packages/pds/src/api/com/atproto/admin/getSubjectStatus.ts @@ -11,8 +11,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.authVerifier.roleOrAdminService, handler: async ({ params, auth }) => { const { did, uri, blob } = params - const modSrvc = ctx.services.moderation(ctx.db) - let body: OutputSchema | null + let body: OutputSchema | null = null if (blob) { if (!did) { throw new InvalidRequestError( @@ -20,14 +19,52 @@ export default function (server: Server, ctx: AppContext) { ) } ensureValidAdminAud(auth, did) - body = await modSrvc.getBlobTakedownState(did, CID.parse(blob)) + const takedown = await ctx.actorStore.read(did, (store) => + store.repo.blob.getBlobTakedownStatus(CID.parse(blob)), + ) + if (takedown) { + body = { + subject: { + $type: 'com.atproto.admin.defs#repoBlobRef', + did: did, + cid: blob, + }, + takedown, + } + } } else if (uri) { const parsedUri = new AtUri(uri) ensureValidAdminAud(auth, parsedUri.hostname) - body = await modSrvc.getRecordTakedownState(parsedUri) + const [takedown, cid] = await ctx.actorStore.read( + parsedUri.hostname, + (store) => + Promise.all([ + store.record.getRecordTakedownStatus(parsedUri), + store.record.getCurrentRecordCid(parsedUri), + ]), + ) + if (cid && takedown) { + body = { + subject: { + $type: 'com.atproto.repo.strongRef', + uri: parsedUri.toString(), + cid: cid.toString(), + }, + takedown, + } + } } else if (did) { ensureValidAdminAud(auth, did) - body = await modSrvc.getRepoTakedownState(did) + const takedown = await ctx.accountManager.getAccountTakedownStatus(did) + if (takedown) { + body = { + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: did, + }, + takedown, + } + } } else { throw new InvalidRequestError('No provided subject') } diff --git a/packages/pds/src/api/com/atproto/admin/index.ts b/packages/pds/src/api/com/atproto/admin/index.ts index 3ff1bcdb517..569ee875957 100644 --- a/packages/pds/src/api/com/atproto/admin/index.ts +++ b/packages/pds/src/api/com/atproto/admin/index.ts @@ -16,6 +16,7 @@ import getInviteCodes from './getInviteCodes' import updateAccountHandle from './updateAccountHandle' import updateAccountEmail from './updateAccountEmail' import sendEmail from './sendEmail' +import deleteAccount from './deleteAccount' import queryModerationStatuses from './queryModerationStatuses' export default function (server: Server, ctx: AppContext) { @@ -36,4 +37,5 @@ export default function (server: Server, ctx: AppContext) { updateAccountHandle(server, ctx) updateAccountEmail(server, ctx) sendEmail(server, ctx) + deleteAccount(server, ctx) } diff --git a/packages/pds/src/api/com/atproto/admin/queryModerationEvents.ts b/packages/pds/src/api/com/atproto/admin/queryModerationEvents.ts index 55fee1d61b1..4ccb0ac9f6b 100644 --- a/packages/pds/src/api/com/atproto/admin/queryModerationEvents.ts +++ b/packages/pds/src/api/com/atproto/admin/queryModerationEvents.ts @@ -1,6 +1,6 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { authPassthru } from './util' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.queryModerationEvents({ diff --git a/packages/pds/src/api/com/atproto/admin/queryModerationStatuses.ts b/packages/pds/src/api/com/atproto/admin/queryModerationStatuses.ts index e9f068018f4..4f6c85e17d2 100644 --- a/packages/pds/src/api/com/atproto/admin/queryModerationStatuses.ts +++ b/packages/pds/src/api/com/atproto/admin/queryModerationStatuses.ts @@ -1,6 +1,6 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { authPassthru } from './util' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.queryModerationStatuses({ diff --git a/packages/pds/src/api/com/atproto/admin/searchRepos.ts b/packages/pds/src/api/com/atproto/admin/searchRepos.ts index bf1ab92e3c3..4125b84eed9 100644 --- a/packages/pds/src/api/com/atproto/admin/searchRepos.ts +++ b/packages/pds/src/api/com/atproto/admin/searchRepos.ts @@ -1,13 +1,11 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { authPassthru } from './util' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.searchRepos({ auth: ctx.authVerifier.role, handler: async ({ req, params }) => { - // @TODO merge invite details to this list view. could also add - // support for invitedBy param, which is not supported by appview. const { data: result } = await ctx.appViewAgent.com.atproto.admin.searchRepos( params, diff --git a/packages/pds/src/api/com/atproto/admin/sendEmail.ts b/packages/pds/src/api/com/atproto/admin/sendEmail.ts index e4defa466a8..6be104e8a02 100644 --- a/packages/pds/src/api/com/atproto/admin/sendEmail.ts +++ b/packages/pds/src/api/com/atproto/admin/sendEmail.ts @@ -1,7 +1,7 @@ import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { authPassthru } from './util' +import { authPassthru, resultPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.sendEmail({ @@ -17,19 +17,27 @@ export default function (server: Server, ctx: AppContext) { senderDid, subject = 'Message from Bluesky moderator', } = input.body - const userInfo = await ctx.db.db - .selectFrom('user_account') - .where('did', '=', recipientDid) - .select('email') - .executeTakeFirst() - - if (!userInfo) { + const account = await ctx.accountManager.getAccount(recipientDid) + if (!account) { throw new InvalidRequestError('Recipient not found') } + if (ctx.entrywayAgent) { + return resultPassthru( + await ctx.entrywayAgent.com.atproto.admin.sendEmail( + input.body, + authPassthru(req, true), + ), + ) + } + + if (!account.email) { + throw new InvalidRequestError('account does not have an email address') + } + await ctx.moderationMailer.send( { content }, - { subject, to: userInfo.email }, + { subject, to: account.email }, ) await ctx.appViewAgent.api.com.atproto.admin.emitModerationEvent( { diff --git a/packages/pds/src/api/com/atproto/admin/updateAccountEmail.ts b/packages/pds/src/api/com/atproto/admin/updateAccountEmail.ts index 04bb8bdfb1a..6e42905b83d 100644 --- a/packages/pds/src/api/com/atproto/admin/updateAccountEmail.ts +++ b/packages/pds/src/api/com/atproto/admin/updateAccountEmail.ts @@ -1,23 +1,33 @@ import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.updateAccountEmail({ auth: ctx.authVerifier.role, - handler: async ({ input, auth }) => { + handler: async ({ input, auth, req }) => { if (!auth.credentials.admin) { throw new AuthRequiredError('Insufficient privileges') } - await ctx.db.transaction(async (dbTxn) => { - const accntService = ctx.services.account(dbTxn) - const account = await accntService.getAccount(input.body.account) - if (!account) { - throw new InvalidRequestError( - `Account does not exist: ${input.body.account}`, - ) - } - await accntService.updateEmail(account.did, input.body.email) + const account = await ctx.accountManager.getAccount(input.body.account) + if (!account) { + throw new InvalidRequestError( + `Account does not exist: ${input.body.account}`, + ) + } + + if (ctx.entrywayAgent) { + await ctx.entrywayAgent.com.atproto.admin.updateAccountEmail( + input.body, + authPassthru(req, true), + ) + return + } + + await ctx.accountManager.updateEmail({ + did: account.did, + email: input.body.email, }) }, }) diff --git a/packages/pds/src/api/com/atproto/admin/updateAccountHandle.ts b/packages/pds/src/api/com/atproto/admin/updateAccountHandle.ts index 368c2dae586..b13d1ee54aa 100644 --- a/packages/pds/src/api/com/atproto/admin/updateAccountHandle.ts +++ b/packages/pds/src/api/com/atproto/admin/updateAccountHandle.ts @@ -2,10 +2,6 @@ import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import { normalizeAndValidateHandle } from '../../../../handle' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { - HandleSequenceToken, - UserAlreadyExistsError, -} from '../../../../services/account' import { httpLogger } from '../../../../logger' export default function (server: Server, ctx: AppContext) { @@ -15,6 +11,7 @@ export default function (server: Server, ctx: AppContext) { if (!auth.credentials.admin) { throw new AuthRequiredError('Insufficient privileges') } + const { did } = input.body const handle = await normalizeAndValidateHandle({ ctx, @@ -23,34 +20,32 @@ export default function (server: Server, ctx: AppContext) { allowReserved: true, }) - const existingAccnt = await ctx.services.account(ctx.db).getAccount(did) - if (!existingAccnt) { - throw new InvalidRequestError(`Account not found: ${did}`) - } + // Pessimistic check to handle spam: also enforced by updateHandle() and the db. + const account = await ctx.accountManager.getAccount(handle) - let seqHandleTok: HandleSequenceToken - if (existingAccnt.handle === handle) { - seqHandleTok = { handle, did } + if (account) { + if (account.did !== did) { + throw new InvalidRequestError(`Handle already taken: ${handle}`) + } } else { - seqHandleTok = await ctx.db.transaction(async (dbTxn) => { - let tok: HandleSequenceToken - try { - tok = await ctx.services.account(dbTxn).updateHandle(did, handle) - } catch (err) { - if (err instanceof UserAlreadyExistsError) { - throw new InvalidRequestError(`Handle already taken: ${handle}`) - } - throw err + if (ctx.cfg.entryway) { + // the pds defers to the entryway for updating the handle in the user's did doc. + // here was just check that the handle is already bidirectionally confirmed. + // @TODO if handle is taken according to this PDS, should we force-update? + const doc = await ctx.idResolver.did + .resolveAtprotoData(did, true) + .catch(() => undefined) + if (doc?.handle !== handle) { + throw new InvalidRequestError('Handle does not match DID doc') } + } else { await ctx.plcClient.updateHandle(did, ctx.plcRotationKey, handle) - return tok - }) + } + await ctx.accountManager.updateHandle(did, handle) } try { - await ctx.db.transaction(async (dbTxn) => { - await ctx.services.account(dbTxn).sequenceHandle(seqHandleTok) - }) + await ctx.sequencer.sequenceHandleUpdate(did, handle) } catch (err) { httpLogger.error( { err, did, handle }, diff --git a/packages/pds/src/api/com/atproto/admin/updateSubjectStatus.ts b/packages/pds/src/api/com/atproto/admin/updateSubjectStatus.ts index 920debba986..649f906cd7e 100644 --- a/packages/pds/src/api/com/atproto/admin/updateSubjectStatus.ts +++ b/packages/pds/src/api/com/atproto/admin/updateSubjectStatus.ts @@ -23,24 +23,22 @@ export default function (server: Server, ctx: AppContext) { const { subject, takedown } = input.body if (takedown) { - const modSrvc = ctx.services.moderation(ctx.db) - const authSrvc = ctx.services.auth(ctx.db) if (isRepoRef(subject)) { ensureValidAdminAud(auth, subject.did) - await Promise.all([ - modSrvc.updateRepoTakedownState(subject.did, takedown), - authSrvc.revokeRefreshTokensByDid(subject.did), - ]) + await ctx.accountManager.takedownAccount(subject.did, takedown) } else if (isStrongRef(subject)) { const uri = new AtUri(subject.uri) ensureValidAdminAud(auth, uri.hostname) - await modSrvc.updateRecordTakedownState(uri, takedown) + await ctx.actorStore.transact(uri.hostname, (store) => + store.record.updateRecordTakedownStatus(uri, takedown), + ) } else if (isRepoBlobRef(subject)) { ensureValidAdminAud(auth, subject.did) - await modSrvc.updateBlobTakedownState( - subject.did, - CID.parse(subject.cid), - takedown, + await ctx.actorStore.transact(subject.did, (store) => + store.repo.blob.updateBlobTakedownStatus( + CID.parse(subject.cid), + takedown, + ), ) } else { throw new InvalidRequestError('Invalid subject') diff --git a/packages/pds/src/api/com/atproto/identity/resolveHandle.ts b/packages/pds/src/api/com/atproto/identity/resolveHandle.ts index f708f3f93d8..472d7ccb234 100644 --- a/packages/pds/src/api/com/atproto/identity/resolveHandle.ts +++ b/packages/pds/src/api/com/atproto/identity/resolveHandle.ts @@ -18,7 +18,7 @@ export default function (server: Server, ctx: AppContext) { } let did: string | undefined - const user = await ctx.services.account(ctx.db).getAccount(handle, true) + const user = await ctx.accountManager.getAccount(handle, true) if (user) { did = user.did diff --git a/packages/pds/src/api/com/atproto/identity/updateHandle.ts b/packages/pds/src/api/com/atproto/identity/updateHandle.ts index 44a2aaded72..32018601756 100644 --- a/packages/pds/src/api/com/atproto/identity/updateHandle.ts +++ b/packages/pds/src/api/com/atproto/identity/updateHandle.ts @@ -1,13 +1,10 @@ import { InvalidRequestError } from '@atproto/xrpc-server' +import { DAY, MINUTE } from '@atproto/common' import { normalizeAndValidateHandle } from '../../../../handle' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { - HandleSequenceToken, - UserAlreadyExistsError, -} from '../../../../services/account' import { httpLogger } from '../../../../logger' -import { DAY, MINUTE } from '@atproto/common' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.identity.updateHandle({ @@ -24,8 +21,20 @@ export default function (server: Server, ctx: AppContext) { calcKey: ({ auth }) => auth.credentials.did, }, ], - handler: async ({ auth, input }) => { + handler: async ({ auth, input, req }) => { const requester = auth.credentials.did + + if (ctx.entrywayAgent) { + // the full flow is: + // -> entryway(identity.updateHandle) [update handle, submit plc op] + // -> pds(admin.updateAccountHandle) [track handle, sequence handle update] + await ctx.entrywayAgent.com.atproto.identity.updateHandle( + { did: requester, handle: input.body.handle }, + authPassthru(req, true), + ) + return + } + const handle = await normalizeAndValidateHandle({ ctx, handle: input.body.handle, @@ -33,40 +42,19 @@ export default function (server: Server, ctx: AppContext) { }) // Pessimistic check to handle spam: also enforced by updateHandle() and the db. - const handleDid = await ctx.services.account(ctx.db).getHandleDid(handle) + const account = await ctx.accountManager.getAccount(handle) - let seqHandleTok: HandleSequenceToken - if (handleDid) { - if (handleDid !== requester) { + if (account) { + if (account.did !== requester) { throw new InvalidRequestError(`Handle already taken: ${handle}`) } - seqHandleTok = { did: requester, handle: handle } } else { - seqHandleTok = await ctx.db.transaction(async (dbTxn) => { - let tok: HandleSequenceToken - try { - tok = await ctx.services - .account(dbTxn) - .updateHandle(requester, handle) - } catch (err) { - if (err instanceof UserAlreadyExistsError) { - throw new InvalidRequestError(`Handle already taken: ${handle}`) - } - throw err - } - await ctx.plcClient.updateHandle( - requester, - ctx.plcRotationKey, - handle, - ) - return tok - }) + await ctx.plcClient.updateHandle(requester, ctx.plcRotationKey, handle) + await ctx.accountManager.updateHandle(requester, handle) } try { - await ctx.db.transaction(async (dbTxn) => { - await ctx.services.account(dbTxn).sequenceHandle(seqHandleTok) - }) + await ctx.sequencer.sequenceHandleUpdate(requester, handle) } catch (err) { httpLogger.error( { err, did: requester, handle }, diff --git a/packages/pds/src/api/com/atproto/index.ts b/packages/pds/src/api/com/atproto/index.ts index a5c26c80495..c7d4f217f88 100644 --- a/packages/pds/src/api/com/atproto/index.ts +++ b/packages/pds/src/api/com/atproto/index.ts @@ -6,6 +6,7 @@ import moderation from './moderation' import repo from './repo' import serverMethods from './server' import sync from './sync' +import temp from './temp' export default function (server: Server, ctx: AppContext) { admin(server, ctx) @@ -14,4 +15,5 @@ export default function (server: Server, ctx: AppContext) { repo(server, ctx) serverMethods(server, ctx) sync(server, ctx) + temp(server, ctx) } diff --git a/packages/pds/src/api/com/atproto/repo/applyWrites.ts b/packages/pds/src/api/com/atproto/repo/applyWrites.ts index b2dcaaab918..1fd1bbb531e 100644 --- a/packages/pds/src/api/com/atproto/repo/applyWrites.ts +++ b/packages/pds/src/api/com/atproto/repo/applyWrites.ts @@ -14,7 +14,6 @@ import { PreparedWrite, } from '../../../../repo' import AppContext from '../../../../context' -import { ConcurrentWriteError } from '../../../../services/repo' const ratelimitPoints = ({ input }: { input: HandlerInput }) => { let points = 0 @@ -49,7 +48,7 @@ export default function (server: Server, ctx: AppContext) { handler: async ({ input, auth }) => { const tx = input.body const { repo, validate, swapCommit } = tx - const did = await ctx.services.account(ctx.db).getDidForActor(repo) + const did = await ctx.accountManager.getDidForActor(repo) if (!did) { throw new InvalidRequestError(`Could not find repo: ${repo}`) @@ -108,19 +107,20 @@ export default function (server: Server, ctx: AppContext) { const swapCommitCid = swapCommit ? CID.parse(swapCommit) : undefined - try { - await ctx.services - .repo(ctx.db) - .processWrites({ did, writes, swapCommitCid }, 10) - } catch (err) { - if (err instanceof BadCommitSwapError) { - throw new InvalidRequestError(err.message, 'InvalidSwap') - } else if (err instanceof ConcurrentWriteError) { - throw new InvalidRequestError(err.message, 'ConcurrentWrites') - } else { - throw err + const commit = await ctx.actorStore.transact(did, async (actorTxn) => { + try { + return await actorTxn.repo.processWrites(writes, swapCommitCid) + } catch (err) { + if (err instanceof BadCommitSwapError) { + throw new InvalidRequestError(err.message, 'InvalidSwap') + } else { + throw err + } } - } + }) + + await ctx.sequencer.sequenceCommit(did, commit, writes) + await ctx.accountManager.updateRepoRoot(did, commit.cid, commit.rev) }, }) } diff --git a/packages/pds/src/api/com/atproto/repo/createRecord.ts b/packages/pds/src/api/com/atproto/repo/createRecord.ts index 0c12ecc2285..8d7aaedd11c 100644 --- a/packages/pds/src/api/com/atproto/repo/createRecord.ts +++ b/packages/pds/src/api/com/atproto/repo/createRecord.ts @@ -1,18 +1,14 @@ import { CID } from 'multiformats/cid' import { InvalidRequestError, AuthRequiredError } from '@atproto/xrpc-server' import { InvalidRecordKeyError } from '@atproto/syntax' -import { prepareCreate } from '../../../../repo' +import { prepareCreate, prepareDelete } from '../../../../repo' import { Server } from '../../../../lexicon' import { BadCommitSwapError, InvalidRecordError, PreparedCreate, - prepareDelete, } from '../../../../repo' import AppContext from '../../../../context' -import { ids } from '../../../../lexicon/lexicons' -import Database from '../../../../db' -import { ConcurrentWriteError } from '../../../../services/repo' export default function (server: Server, ctx: AppContext) { server.com.atproto.repo.createRecord({ @@ -32,7 +28,7 @@ export default function (server: Server, ctx: AppContext) { handler: async ({ input, auth }) => { const { repo, collection, rkey, record, swapCommit, validate } = input.body - const did = await ctx.services.account(ctx.db).getDidForActor(repo) + const did = await ctx.accountManager.getDidForActor(repo) if (!did) { throw new InvalidRequestError(`Could not find repo: ${repo}`) @@ -66,24 +62,40 @@ export default function (server: Server, ctx: AppContext) { throw err } - const backlinkDeletions = validate - ? await getBacklinkDeletions(ctx.db, ctx, write) - : [] + const { commit, writes } = await ctx.actorStore.transact( + did, + async (actorTxn) => { + const backlinkConflicts = validate + ? await actorTxn.record.getBacklinkConflicts( + write.uri, + write.record, + ) + : [] + const backlinkDeletions = backlinkConflicts.map((uri) => + prepareDelete({ + did: uri.hostname, + collection: uri.collection, + rkey: uri.rkey, + }), + ) + const writes = [...backlinkDeletions, write] + try { + const commit = await actorTxn.repo.processWrites( + writes, + swapCommitCid, + ) + return { commit, writes } + } catch (err) { + if (err instanceof BadCommitSwapError) { + throw new InvalidRequestError(err.message, 'InvalidSwap') + } + throw err + } + }, + ) - const writes = [...backlinkDeletions, write] - - try { - await ctx.services - .repo(ctx.db) - .processWrites({ did, writes, swapCommitCid }, 10) - } catch (err) { - if (err instanceof BadCommitSwapError) { - throw new InvalidRequestError(err.message, 'InvalidSwap') - } else if (err instanceof ConcurrentWriteError) { - throw new InvalidRequestError(err.message, 'ConcurrentWrites') - } - throw err - } + await ctx.sequencer.sequenceCommit(did, commit, writes) + await ctx.accountManager.updateRepoRoot(did, commit.cid, commit.rev) return { encoding: 'application/json', @@ -92,50 +104,3 @@ export default function (server: Server, ctx: AppContext) { }, }) } - -// @NOTE this logic a placeholder until we allow users to specify these constraints themselves. -// Ensures that we don't end-up with duplicate likes, reposts, and follows from race conditions. - -async function getBacklinkDeletions( - tx: Database, - ctx: AppContext, - write: PreparedCreate, -) { - const recordTxn = ctx.services.record(tx) - const { - record, - uri: { host: did, collection }, - } = write - const toDelete = ({ rkey }: { rkey: string }) => - prepareDelete({ did, collection, rkey }) - - if ( - (collection === ids.AppBskyGraphFollow || - collection === ids.AppBskyGraphBlock) && - typeof record['subject'] === 'string' - ) { - const backlinks = await recordTxn.getRecordBacklinks({ - did, - collection, - path: 'subject', - linkTo: record['subject'], - }) - return backlinks.map(toDelete) - } - - if ( - (collection === ids.AppBskyFeedLike || - collection === ids.AppBskyFeedRepost) && - typeof record['subject']?.['uri'] === 'string' - ) { - const backlinks = await recordTxn.getRecordBacklinks({ - did, - collection, - path: 'subject.uri', - linkTo: record['subject']['uri'], - }) - return backlinks.map(toDelete) - } - - return [] -} diff --git a/packages/pds/src/api/com/atproto/repo/deleteRecord.ts b/packages/pds/src/api/com/atproto/repo/deleteRecord.ts index efc5a64fadf..5a98c0d9963 100644 --- a/packages/pds/src/api/com/atproto/repo/deleteRecord.ts +++ b/packages/pds/src/api/com/atproto/repo/deleteRecord.ts @@ -4,7 +4,6 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { BadCommitSwapError, BadRecordSwapError } from '../../../../repo' import { CID } from 'multiformats/cid' -import { ConcurrentWriteError } from '../../../../services/repo' export default function (server: Server, ctx: AppContext) { server.com.atproto.repo.deleteRecord({ @@ -23,7 +22,7 @@ export default function (server: Server, ctx: AppContext) { ], handler: async ({ input, auth }) => { const { repo, collection, rkey, swapCommit, swapRecord } = input.body - const did = await ctx.services.account(ctx.db).getDidForActor(repo) + const did = await ctx.accountManager.getDidForActor(repo) if (!did) { throw new InvalidRequestError(`Could not find repo: ${repo}`) @@ -41,30 +40,28 @@ export default function (server: Server, ctx: AppContext) { rkey, swapCid: swapRecordCid, }) - const record = await ctx.services - .record(ctx.db) - .getRecord(write.uri, null, true) - if (!record) { - return // No-op if record already doesn't exist - } - - const writes = [write] - - try { - await ctx.services - .repo(ctx.db) - .processWrites({ did, writes, swapCommitCid }, 10) - } catch (err) { - if ( - err instanceof BadCommitSwapError || - err instanceof BadRecordSwapError - ) { - throw new InvalidRequestError(err.message, 'InvalidSwap') - } else if (err instanceof ConcurrentWriteError) { - throw new InvalidRequestError(err.message, 'ConcurrentWrites') - } else { - throw err + const commit = await ctx.actorStore.transact(did, async (actorTxn) => { + const record = await actorTxn.record.getRecord(write.uri, null, true) + if (!record) { + return null // No-op if record already doesn't exist } + try { + return await actorTxn.repo.processWrites([write], swapCommitCid) + } catch (err) { + if ( + err instanceof BadCommitSwapError || + err instanceof BadRecordSwapError + ) { + throw new InvalidRequestError(err.message, 'InvalidSwap') + } else { + throw err + } + } + }) + + if (commit !== null) { + await ctx.sequencer.sequenceCommit(did, commit, [write]) + await ctx.accountManager.updateRepoRoot(did, commit.cid, commit.rev) } }, }) diff --git a/packages/pds/src/api/com/atproto/repo/describeRepo.ts b/packages/pds/src/api/com/atproto/repo/describeRepo.ts index b340314ef77..ff671d8c537 100644 --- a/packages/pds/src/api/com/atproto/repo/describeRepo.ts +++ b/packages/pds/src/api/com/atproto/repo/describeRepo.ts @@ -2,12 +2,13 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import * as id from '@atproto/identity' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' +import { INVALID_HANDLE } from '@atproto/syntax' export default function (server: Server, ctx: AppContext) { server.com.atproto.repo.describeRepo(async ({ params }) => { const { repo } = params - const account = await ctx.services.account(ctx.db).getAccount(repo) + const account = await ctx.accountManager.getAccount(repo) if (account === null) { throw new InvalidRequestError(`Could not find user: ${repo}`) } @@ -22,14 +23,14 @@ export default function (server: Server, ctx: AppContext) { const handle = id.getHandle(didDoc) const handleIsCorrect = handle === account.handle - const collections = await ctx.services - .record(ctx.db) - .listCollectionsForDid(account.did) + const collections = await ctx.actorStore.read(account.did, (store) => + store.record.listCollections(), + ) return { encoding: 'application/json', body: { - handle: account.handle, + handle: account.handle ?? INVALID_HANDLE, did: account.did, didDoc, collections, diff --git a/packages/pds/src/api/com/atproto/repo/getRecord.ts b/packages/pds/src/api/com/atproto/repo/getRecord.ts index 4a333cf0648..68f73d38b12 100644 --- a/packages/pds/src/api/com/atproto/repo/getRecord.ts +++ b/packages/pds/src/api/com/atproto/repo/getRecord.ts @@ -6,14 +6,14 @@ import { InvalidRequestError } from '@atproto/xrpc-server' export default function (server: Server, ctx: AppContext) { server.com.atproto.repo.getRecord(async ({ params }) => { const { repo, collection, rkey, cid } = params - const did = await ctx.services.account(ctx.db).getDidForActor(repo) + const did = await ctx.accountManager.getDidForActor(repo) // fetch from pds if available, if not then fetch from appview if (did) { const uri = AtUri.make(did, collection, rkey) - const record = await ctx.services - .record(ctx.db) - .getRecord(uri, cid || null) + const record = await ctx.actorStore.read(did, (store) => + store.record.getRecord(uri, cid ?? null), + ) if (!record || record.takedownRef !== null) { throw new InvalidRequestError(`Could not locate record: ${uri}`) } diff --git a/packages/pds/src/api/com/atproto/repo/listRecords.ts b/packages/pds/src/api/com/atproto/repo/listRecords.ts index 8c2669ff010..e8440835c9a 100644 --- a/packages/pds/src/api/com/atproto/repo/listRecords.ts +++ b/packages/pds/src/api/com/atproto/repo/listRecords.ts @@ -15,20 +15,21 @@ export default function (server: Server, ctx: AppContext) { reverse = false, } = params - const did = await ctx.services.account(ctx.db).getDidForActor(repo) + const did = await ctx.accountManager.getDidForActor(repo) if (!did) { throw new InvalidRequestError(`Could not find repo: ${repo}`) } - const records = await ctx.services.record(ctx.db).listRecordsForCollection({ - did, - collection, - limit, - reverse, - cursor, - rkeyStart, - rkeyEnd, - }) + const records = await ctx.actorStore.read(did, (store) => + store.record.listRecordsForCollection({ + collection, + limit, + reverse, + cursor, + rkeyStart, + rkeyEnd, + }), + ) const lastRecord = records.at(-1) const lastUri = lastRecord && new AtUri(lastRecord?.uri) diff --git a/packages/pds/src/api/com/atproto/repo/putRecord.ts b/packages/pds/src/api/com/atproto/repo/putRecord.ts index 9d7d4332b2e..a0cc047e4ee 100644 --- a/packages/pds/src/api/com/atproto/repo/putRecord.ts +++ b/packages/pds/src/api/com/atproto/repo/putRecord.ts @@ -1,6 +1,7 @@ import { CID } from 'multiformats/cid' import { AtUri } from '@atproto/syntax' import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' +import { CommitData } from '@atproto/repo' import { Server } from '../../../../lexicon' import { prepareUpdate, prepareCreate } from '../../../../repo' import AppContext from '../../../../context' @@ -11,7 +12,6 @@ import { PreparedCreate, PreparedUpdate, } from '../../../../repo' -import { ConcurrentWriteError } from '../../../../services/repo' export default function (server: Server, ctx: AppContext) { server.com.atproto.repo.putRecord({ @@ -38,7 +38,7 @@ export default function (server: Server, ctx: AppContext) { swapCommit, swapRecord, } = input.body - const did = await ctx.services.account(ctx.db).getDidForActor(repo) + const did = await ctx.accountManager.getDidForActor(repo) if (!did) { throw new InvalidRequestError(`Could not find repo: ${repo}`) @@ -57,47 +57,59 @@ export default function (server: Server, ctx: AppContext) { const swapRecordCid = typeof swapRecord === 'string' ? CID.parse(swapRecord) : swapRecord - const current = await ctx.services - .record(ctx.db) - .getRecord(uri, null, true) - const writeInfo = { + const { commit, write } = await ctx.actorStore.transact( did, - collection, - rkey, - record, - swapCid: swapRecordCid, - validate, - } + async (actorTxn) => { + const current = await actorTxn.record.getRecord(uri, null, true) + const writeInfo = { + did, + collection, + rkey, + record, + swapCid: swapRecordCid, + validate, + } - let write: PreparedCreate | PreparedUpdate - try { - write = current - ? await prepareUpdate(writeInfo) - : await prepareCreate(writeInfo) - } catch (err) { - if (err instanceof InvalidRecordError) { - throw new InvalidRequestError(err.message) - } - throw err - } + let write: PreparedCreate | PreparedUpdate + try { + write = current + ? await prepareUpdate(writeInfo) + : await prepareCreate(writeInfo) + } catch (err) { + if (err instanceof InvalidRecordError) { + throw new InvalidRequestError(err.message) + } + throw err + } - const writes = [write] + // no-op + if (current && current.cid === write.cid.toString()) { + return { + commit: null, + write, + } + } + + let commit: CommitData + try { + commit = await actorTxn.repo.processWrites([write], swapCommitCid) + } catch (err) { + if ( + err instanceof BadCommitSwapError || + err instanceof BadRecordSwapError + ) { + throw new InvalidRequestError(err.message, 'InvalidSwap') + } else { + throw err + } + } + return { commit, write } + }, + ) - try { - await ctx.services - .repo(ctx.db) - .processWrites({ did, writes, swapCommitCid }, 10) - } catch (err) { - if ( - err instanceof BadCommitSwapError || - err instanceof BadRecordSwapError - ) { - throw new InvalidRequestError(err.message, 'InvalidSwap') - } else if (err instanceof ConcurrentWriteError) { - throw new InvalidRequestError(err.message, 'ConcurrentWrites') - } else { - throw err - } + if (commit !== null) { + await ctx.sequencer.sequenceCommit(did, commit, [write]) + await ctx.accountManager.updateRepoRoot(did, commit.cid, commit.rev) } return { diff --git a/packages/pds/src/api/com/atproto/repo/uploadBlob.ts b/packages/pds/src/api/com/atproto/repo/uploadBlob.ts index ba93648470a..f3cc9974d49 100644 --- a/packages/pds/src/api/com/atproto/repo/uploadBlob.ts +++ b/packages/pds/src/api/com/atproto/repo/uploadBlob.ts @@ -6,9 +6,10 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.authVerifier.accessCheckTakedown, handler: async ({ auth, input }) => { const requester = auth.credentials.did - const blob = await ctx.services - .repo(ctx.db) - .blobs.addUntetheredBlob(requester, input.encoding, input.body) + + const blob = await ctx.actorStore.transact(requester, (actorTxn) => { + return actorTxn.repo.blob.addUntetheredBlob(input.encoding, input.body) + }) return { encoding: 'application/json', diff --git a/packages/pds/src/api/com/atproto/server/confirmEmail.ts b/packages/pds/src/api/com/atproto/server/confirmEmail.ts index fe1f05d87f8..cbe7e5a0f74 100644 --- a/packages/pds/src/api/com/atproto/server/confirmEmail.ts +++ b/packages/pds/src/api/com/atproto/server/confirmEmail.ts @@ -1,34 +1,33 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { InvalidRequestError } from '@atproto/xrpc-server' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.confirmEmail({ auth: ctx.authVerifier.accessCheckTakedown, - handler: async ({ auth, input }) => { + handler: async ({ auth, input, req }) => { const did = auth.credentials.did - const { token, email } = input.body - const user = await ctx.services.account(ctx.db).getAccount(did) + const user = await ctx.accountManager.getAccount(did) if (!user) { throw new InvalidRequestError('user not found', 'AccountNotFound') } + if (ctx.entrywayAgent) { + await ctx.entrywayAgent.com.atproto.server.confirmEmail( + input.body, + authPassthru(req, true), + ) + return + } + + const { token, email } = input.body + if (user.email !== email.toLowerCase()) { throw new InvalidRequestError('invalid email', 'InvalidEmail') } - await ctx.services - .account(ctx.db) - .assertValidToken(did, 'confirm_email', token) - - await ctx.db.transaction(async (dbTxn) => { - await ctx.services.account(dbTxn).deleteEmailToken(did, 'confirm_email') - await dbTxn.db - .updateTable('user_account') - .set({ emailConfirmedAt: new Date().toISOString() }) - .where('did', '=', did) - .execute() - }) + await ctx.accountManager.confirmEmail({ did, token }) }, }) } diff --git a/packages/pds/src/api/com/atproto/server/createAccount.ts b/packages/pds/src/api/com/atproto/server/createAccount.ts index e747eb4e9cc..adfdab3b0f8 100644 --- a/packages/pds/src/api/com/atproto/server/createAccount.ts +++ b/packages/pds/src/api/com/atproto/server/createAccount.ts @@ -1,16 +1,16 @@ -import { MINUTE } from '@atproto/common' -import { AtprotoData } from '@atproto/identity' +import { DidDocument, MINUTE, check } from '@atproto/common' +import { AtprotoData, ensureAtpDocument } from '@atproto/identity' import { InvalidRequestError } from '@atproto/xrpc-server' -import * as plc from '@did-plc/lib' +import { ExportableKeypair, Keypair, Secp256k1Keypair } from '@atproto/crypto' import disposable from 'disposable-email' -import { normalizeAndValidateHandle } from '../../../../handle' -import * as scrypt from '../../../../db/scrypt' +import { + baseNormalizeAndValidate, + normalizeAndValidateHandle, +} from '../../../../handle' +import * as plc from '@did-plc/lib' import { Server } from '../../../../lexicon' import { InputSchema as CreateAccountInput } from '../../../../lexicon/types/com/atproto/server/createAccount' -import { countAll } from '../../../../db/util' -import { UserAlreadyExistsError } from '../../../../services/account' import AppContext from '../../../../context' -import Database from '../../../../db' import { didDocForSession } from './util' export default function (server: Server, ctx: AppContext) { @@ -20,72 +20,18 @@ export default function (server: Server, ctx: AppContext) { points: 100, }, handler: async ({ input, req }) => { - const { email, password, inviteCode } = input.body - if (!email) { - throw new InvalidRequestError('Missing input: "email"') - } else if (!password) { - throw new InvalidRequestError('Missing input: "password"') - } else if (input.body.plcOp) { - throw new InvalidRequestError('Unsupported input: "plcOp"') - } - - if (ctx.cfg.invites.required && !inviteCode) { - throw new InvalidRequestError( - 'No invite code provided', - 'InvalidInviteCode', - ) - } + const { did, handle, email, password, inviteCode, signingKey, plcOp } = + ctx.entrywayAgent + ? await validateInputsForEntrywayPds(ctx, input.body) + : await validateInputsForLocalPds(ctx, input.body) - if (!disposable.validate(email)) { - throw new InvalidRequestError( - 'This email address is not supported, please use a different email.', + let didDoc: DidDocument | undefined + let creds: { accessJwt: string; refreshJwt: string } + await ctx.actorStore.create(did, signingKey) + try { + const commit = await ctx.actorStore.transact(did, (actorTxn) => + actorTxn.repo.createRepo([]), ) - } - - // normalize & ensure valid handle - const handle = await normalizeAndValidateHandle({ - ctx, - handle: input.body.handle, - did: input.body.did, - }) - - // check that the invite code still has uses - if (ctx.cfg.invites.required && inviteCode) { - await ensureCodeIsAvailable(ctx.db, inviteCode) - } - - // determine the did & any plc ops we need to send - // if the provided did document is poorly setup, we throw - const { did, plcOp } = await getDidAndPlcOp(ctx, handle, input.body) - - const now = new Date().toISOString() - const passwordScrypt = await scrypt.genSaltAndHash(password) - - const result = await ctx.db.transaction(async (dbTxn) => { - const actorTxn = ctx.services.account(dbTxn) - const repoTxn = ctx.services.repo(dbTxn) - - // it's a bit goofy that we run this logic twice, - // but we run it once for a sanity check before doing scrypt & plc ops - // & a second time for locking + integrity check - if (ctx.cfg.invites.required && inviteCode) { - await ensureCodeIsAvailable(dbTxn, inviteCode, true) - } - - // Register user before going out to PLC to get a real did - try { - await actorTxn.registerUser({ email, handle, did, passwordScrypt }) - } catch (err) { - if (err instanceof UserAlreadyExistsError) { - const got = await actorTxn.getAccount(handle, true) - if (got) { - throw new InvalidRequestError(`Handle already taken: ${handle}`) - } else { - throw new InvalidRequestError(`Email already taken: ${email}`) - } - } - throw err - } // Generate a real did with PLC if (plcOp) { @@ -100,158 +46,245 @@ export default function (server: Server, ctx: AppContext) { } } - // insert invite code use - if (ctx.cfg.invites.required && inviteCode) { - await dbTxn.db - .insertInto('invite_code_use') - .values({ - code: inviteCode, - usedBy: did, - usedAt: now, - }) - .execute() - } - - const { access, refresh } = await ctx.services - .auth(dbTxn) - .createSession(did, null) - - // Setup repo root - await repoTxn.createRepo(did, [], now) - - return { + creds = await ctx.accountManager.createAccount({ did, - accessJwt: access.jwt, - refreshJwt: refresh.jwt, - } - }) + handle, + email, + password, + repoCid: commit.cid, + repoRev: commit.rev, + inviteCode, + }) - const didDoc = await didDocForSession(ctx, result.did, true) + await ctx.sequencer.sequenceCommit(did, commit, []) + await ctx.accountManager.updateRepoRoot(did, commit.cid, commit.rev) + didDoc = await didDocForSession(ctx, did, true) + await ctx.actorStore.clearReservedKeypair(signingKey.did(), did) + } catch (err) { + // this will only be reached if the actor store _did not_ exist before + await ctx.actorStore.destroy(did) + throw err + } return { encoding: 'application/json', body: { handle, - did: result.did, + did: did, didDoc, - accessJwt: result.accessJwt, - refreshJwt: result.refreshJwt, + accessJwt: creds.accessJwt, + refreshJwt: creds.refreshJwt, }, } }, }) } -export const ensureCodeIsAvailable = async ( - db: Database, - inviteCode: string, - withLock = false, -): Promise => { - const { ref } = db.db.dynamic - const invite = await db.db - .selectFrom('invite_code') - .selectAll() - .whereNotExists((qb) => - qb - .selectFrom('repo_root') - .selectAll() - .where('takedownRef', 'is not', null) - .whereRef('did', '=', ref('invite_code.forUser')), +const validateInputsForEntrywayPds = async ( + ctx: AppContext, + input: CreateAccountInput, +) => { + const { did, plcOp } = input + const handle = baseNormalizeAndValidate(input.handle) + if (!did || !input.plcOp) { + throw new InvalidRequestError( + 'non-entryway pds requires bringing a DID and plcOp', ) - .where('code', '=', inviteCode) - .if(withLock && db.dialect === 'pg', (qb) => qb.forUpdate().skipLocked()) - .executeTakeFirst() - - if (!invite || invite.disabled) { + } + if (!check.is(plcOp, plc.def.operation)) { + throw new InvalidRequestError('invalid plc operation', 'IncompatibleDidDoc') + } + const plcRotationKey = ctx.cfg.entryway?.plcRotationKey + if (!plcRotationKey || !plcOp.rotationKeys.includes(plcRotationKey)) { throw new InvalidRequestError( - 'Provided invite code not available', - 'InvalidInviteCode', + 'PLC DID does not include service rotation key', + 'IncompatibleDidDoc', ) } + await plc.assureValidOp(plcOp) + const doc = plc.formatDidDoc({ did, ...plcOp }) + const data = ensureAtpDocument(doc) - const uses = await db.db - .selectFrom('invite_code_use') - .select(countAll.as('count')) - .where('code', '=', inviteCode) - .executeTakeFirstOrThrow() + let signingKey: ExportableKeypair | undefined + if (input.did) { + signingKey = await ctx.actorStore.getReservedKeypair(input.did) + } + if (!signingKey) { + signingKey = await ctx.actorStore.getReservedKeypair(data.signingKey) + } + if (!signingKey) { + throw new InvalidRequestError('reserved signing key does not exist') + } - if (invite.availableUses <= uses.count) { + validateAtprotoData(data, { + handle, + pds: ctx.cfg.service.publicUrl, + signingKey: signingKey.did(), + }) + + return { + did, + handle, + email: undefined, + password: undefined, + inviteCode: undefined, + signingKey, + plcOp, + } +} + +const validateInputsForLocalPds = async ( + ctx: AppContext, + input: CreateAccountInput, +) => { + const { email, password, inviteCode } = input + if (input.plcOp) { + throw new InvalidRequestError('Unsupported input: "plcOp"') + } + + if (ctx.cfg.invites.required && !inviteCode) { throw new InvalidRequestError( - 'Provided invite code not available', + 'No invite code provided', 'InvalidInviteCode', ) } + + if (!email) { + throw new InvalidRequestError('Email is required') + } else if (!disposable.validate(email)) { + throw new InvalidRequestError( + 'This email address is not supported, please use a different email.', + ) + } + + // normalize & ensure valid handle + const handle = await normalizeAndValidateHandle({ + ctx, + handle: input.handle, + did: input.did, + }) + + // check that the invite code still has uses + if (ctx.cfg.invites.required && inviteCode) { + await ctx.accountManager.ensureInviteIsAvailable(inviteCode) + } + + // check that the handle and email are available + const [handleAccnt, emailAcct] = await Promise.all([ + ctx.accountManager.getAccount(handle), + ctx.accountManager.getAccountByEmail(email), + ]) + if (handleAccnt) { + throw new InvalidRequestError(`Handle already taken: ${handle}`) + } else if (emailAcct) { + throw new InvalidRequestError(`Email already taken: ${email}`) + } + + // determine the did & any plc ops we need to send + // if the provided did document is poorly setup, we throw + const signingKey = await Secp256k1Keypair.create({ exportable: true }) + const { did, plcOp } = input.did + ? await validateExistingDid(ctx, handle, input.did, signingKey) + : await createDidAndPlcOp(ctx, handle, input, signingKey) + + return { did, handle, email, password, inviteCode, signingKey, plcOp } } -const getDidAndPlcOp = async ( +const createDidAndPlcOp = async ( ctx: AppContext, handle: string, input: CreateAccountInput, + signingKey: Keypair, ): Promise<{ did: string plcOp: plc.Operation | null }> => { // if the user is not bringing a DID, then we format a create op for PLC - // but we don't send until we ensure the username & email are available - if (!input.did) { - const rotationKeys = [ctx.plcRotationKey.did()] - if (ctx.cfg.identity.recoveryDidKey) { - rotationKeys.unshift(ctx.cfg.identity.recoveryDidKey) - } - if (input.recoveryKey) { - rotationKeys.unshift(input.recoveryKey) - } - const plcCreate = await plc.createOp({ - signingKey: ctx.repoSigningKey.did(), - rotationKeys, - handle, - pds: ctx.cfg.service.publicUrl, - signer: ctx.plcRotationKey, - }) - return { - did: plcCreate.did, - plcOp: plcCreate.op, - } + const rotationKeys = [ctx.plcRotationKey.did()] + if (ctx.cfg.identity.recoveryDidKey) { + rotationKeys.unshift(ctx.cfg.identity.recoveryDidKey) + } + if (input.recoveryKey) { + rotationKeys.unshift(input.recoveryKey) + } + const plcCreate = await plc.createOp({ + signingKey: signingKey.did(), + rotationKeys, + handle, + pds: ctx.cfg.service.publicUrl, + signer: ctx.plcRotationKey, + }) + return { + did: plcCreate.did, + plcOp: plcCreate.op, } +} +const validateExistingDid = async ( + ctx: AppContext, + handle: string, + did: string, + signingKey: Keypair, +): Promise<{ + did: string + plcOp: plc.Operation | null +}> => { // if the user is bringing their own did: // resolve the user's did doc data, including rotationKeys if did:plc // determine if we have the capability to make changes to their DID let atpData: AtprotoData try { - atpData = await ctx.idResolver.did.resolveAtprotoData(input.did) + atpData = await ctx.idResolver.did.resolveAtprotoData(did) } catch (err) { throw new InvalidRequestError( - `could not resolve valid DID document :${input.did}`, + `could not resolve valid DID document :${did}`, 'UnresolvableDid', ) } - if (atpData.handle !== handle) { + validateAtprotoData(atpData, { + handle, + pds: ctx.cfg.service.publicUrl, + signingKey: signingKey.did(), + }) + + if (did.startsWith('did:plc')) { + const data = await ctx.plcClient.getDocumentData(did) + if (!data.rotationKeys.includes(ctx.plcRotationKey.did())) { + throw new InvalidRequestError( + 'PLC DID does not include service rotation key', + 'IncompatibleDidDoc', + ) + } + } + + return { did: did, plcOp: null } +} + +const validateAtprotoData = ( + data: AtprotoData, + expected: { + handle: string + pds: string + signingKey: string + }, +) => { + // if the user is bringing their own did: + // resolve the user's did doc data, including rotationKeys if did:plc + // determine if we have the capability to make changes to their DID + if (data.handle !== expected.handle) { throw new InvalidRequestError( 'provided handle does not match DID document handle', 'IncompatibleDidDoc', ) - } else if (atpData.pds !== ctx.cfg.service.publicUrl) { + } else if (data.pds !== expected.pds) { throw new InvalidRequestError( 'DID document pds endpoint does not match service endpoint', 'IncompatibleDidDoc', ) - } else if (atpData.signingKey !== ctx.repoSigningKey.did()) { + } else if (data.signingKey !== expected.signingKey) { throw new InvalidRequestError( 'DID document signing key does not match service signing key', 'IncompatibleDidDoc', ) } - - if (input.did.startsWith('did:plc')) { - const data = await ctx.plcClient.getDocumentData(input.did) - if (!data.rotationKeys.includes(ctx.plcRotationKey.did())) { - throw new InvalidRequestError( - 'PLC DID does not include service rotation key', - 'IncompatibleDidDoc', - ) - } - } - - return { did: input.did, plcOp: null } } diff --git a/packages/pds/src/api/com/atproto/server/createAppPassword.ts b/packages/pds/src/api/com/atproto/server/createAppPassword.ts index 43ea255cab0..e278f77f310 100644 --- a/packages/pds/src/api/com/atproto/server/createAppPassword.ts +++ b/packages/pds/src/api/com/atproto/server/createAppPassword.ts @@ -1,14 +1,25 @@ import AppContext from '../../../../context' import { Server } from '../../../../lexicon' +import { authPassthru, resultPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.createAppPassword({ auth: ctx.authVerifier.accessNotAppPassword, - handler: async ({ auth, input }) => { + handler: async ({ auth, input, req }) => { + if (ctx.entrywayAgent) { + return resultPassthru( + await ctx.entrywayAgent.com.atproto.server.createAppPassword( + input.body, + authPassthru(req, true), + ), + ) + } + const { name } = input.body - const appPassword = await ctx.services - .account(ctx.db) - .createAppPassword(auth.credentials.did, name) + const appPassword = await ctx.accountManager.createAppPassword( + auth.credentials.did, + name, + ) return { encoding: 'application/json', body: appPassword, diff --git a/packages/pds/src/api/com/atproto/server/createInviteCode.ts b/packages/pds/src/api/com/atproto/server/createInviteCode.ts index e1b71320795..6c00074e31d 100644 --- a/packages/pds/src/api/com/atproto/server/createInviteCode.ts +++ b/packages/pds/src/api/com/atproto/server/createInviteCode.ts @@ -1,4 +1,4 @@ -import { AuthRequiredError } from '@atproto/xrpc-server' +import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { genInvCode } from './util' @@ -6,27 +6,23 @@ import { genInvCode } from './util' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.createInviteCode({ auth: ctx.authVerifier.role, - handler: async ({ input, req, auth }) => { + handler: async ({ input, auth }) => { if (!auth.credentials.admin) { throw new AuthRequiredError('Insufficient privileges') } + if (ctx.cfg.entryway) { + throw new InvalidRequestError( + 'Account invites are managed by the entryway service', + ) + } const { useCount, forAccount = 'admin' } = input.body const code = genInvCode(ctx.cfg) - await ctx.db.db - .insertInto('invite_code') - .values({ - code: code, - availableUses: useCount, - disabled: 0, - forUser: forAccount, - createdBy: 'admin', - createdAt: new Date().toISOString(), - }) - .execute() - - req.log.info({ useCount, code, forAccount }, 'created invite code') + await ctx.accountManager.createInviteCodes( + [{ account: forAccount, codes: [code] }], + useCount, + ) return { encoding: 'application/json', diff --git a/packages/pds/src/api/com/atproto/server/createInviteCodes.ts b/packages/pds/src/api/com/atproto/server/createInviteCodes.ts index f5f9cb972fc..30d0b83d772 100644 --- a/packages/pds/src/api/com/atproto/server/createInviteCodes.ts +++ b/packages/pds/src/api/com/atproto/server/createInviteCodes.ts @@ -1,48 +1,32 @@ -import { chunkArray } from '@atproto/common' -import { AuthRequiredError } from '@atproto/xrpc-server' +import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { genInvCodes } from './util' -import { InviteCode } from '../../../../db/tables/invite-code' import { AccountCodes } from '../../../../lexicon/types/com/atproto/server/createInviteCodes' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.createInviteCodes({ auth: ctx.authVerifier.role, - handler: async ({ input, req, auth }) => { + handler: async ({ input, auth }) => { if (!auth.credentials.admin) { throw new AuthRequiredError('Insufficient privileges') } + if (ctx.cfg.entryway) { + throw new InvalidRequestError( + 'Account invites are managed by the entryway service', + ) + } + const { codeCount, useCount } = input.body const forAccounts = input.body.forAccounts ?? ['admin'] - const vals: InviteCode[] = [] const accountCodes: AccountCodes[] = [] for (const account of forAccounts) { const codes = genInvCodes(ctx.cfg, codeCount) - for (const code of codes) { - vals.push({ - code: code, - availableUses: useCount, - disabled: 0 as const, - forUser: account, - createdBy: 'admin', - createdAt: new Date().toISOString(), - }) - } accountCodes.push({ account, codes }) } - await Promise.all( - chunkArray(vals, 500).map((chunk) => - ctx.db.db.insertInto('invite_code').values(chunk).execute(), - ), - ) - - req.log.info( - { useCount, codes: accountCodes, forAccounts, codeCount }, - 'created invite codes', - ) + await ctx.accountManager.createInviteCodes(accountCodes, useCount) return { encoding: 'application/json', diff --git a/packages/pds/src/api/com/atproto/server/createSession.ts b/packages/pds/src/api/com/atproto/server/createSession.ts index 64872d5aae1..70e42fdde62 100644 --- a/packages/pds/src/api/com/atproto/server/createSession.ts +++ b/packages/pds/src/api/com/atproto/server/createSession.ts @@ -1,9 +1,11 @@ import { DAY, MINUTE } from '@atproto/common' +import { INVALID_HANDLE } from '@atproto/syntax' import { AuthRequiredError } from '@atproto/xrpc-server' import AppContext from '../../../../context' import { softDeleted } from '../../../../db/util' import { Server } from '../../../../lexicon' import { didDocForSession } from './util' +import { authPassthru, resultPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.createSession({ @@ -19,27 +21,34 @@ export default function (server: Server, ctx: AppContext) { calcKey: ({ input, req }) => `${input.body.identifier}-${req.ip}`, }, ], - handler: async ({ input }) => { + handler: async ({ input, req }) => { + if (ctx.entrywayAgent) { + return resultPassthru( + await ctx.entrywayAgent.com.atproto.server.createSession( + input.body, + authPassthru(req, true), + ), + ) + } + const { password } = input.body const identifier = input.body.identifier.toLowerCase() - const authService = ctx.services.auth(ctx.db) - const actorService = ctx.services.account(ctx.db) const user = identifier.includes('@') - ? await actorService.getAccountByEmail(identifier, true) - : await actorService.getAccount(identifier, true) + ? await ctx.accountManager.getAccountByEmail(identifier, true) + : await ctx.accountManager.getAccount(identifier, true) if (!user) { throw new AuthRequiredError('Invalid identifier or password') } let appPasswordName: string | null = null - const validAccountPass = await actorService.verifyAccountPassword( + const validAccountPass = await ctx.accountManager.verifyAccountPassword( user.did, password, ) if (!validAccountPass) { - appPasswordName = await actorService.verifyAppPassword( + appPasswordName = await ctx.accountManager.verifyAppPassword( user.did, password, ) @@ -55,8 +64,8 @@ export default function (server: Server, ctx: AppContext) { ) } - const [{ access, refresh }, didDoc] = await Promise.all([ - authService.createSession(user.did, appPasswordName), + const [{ accessJwt, refreshJwt }, didDoc] = await Promise.all([ + ctx.accountManager.createSession(user.did, appPasswordName), didDocForSession(ctx, user.did), ]) @@ -65,11 +74,11 @@ export default function (server: Server, ctx: AppContext) { body: { did: user.did, didDoc, - handle: user.handle, - email: user.email, + handle: user.handle ?? INVALID_HANDLE, + email: user.email ?? undefined, emailConfirmed: !!user.emailConfirmedAt, - accessJwt: access.jwt, - refreshJwt: refresh.jwt, + accessJwt, + refreshJwt, }, } }, diff --git a/packages/pds/src/api/com/atproto/server/deleteAccount.ts b/packages/pds/src/api/com/atproto/server/deleteAccount.ts index 2088c387339..9dc1de0df07 100644 --- a/packages/pds/src/api/com/atproto/server/deleteAccount.ts +++ b/packages/pds/src/api/com/atproto/server/deleteAccount.ts @@ -1,9 +1,8 @@ -import { AuthRequiredError } from '@atproto/xrpc-server' +import { MINUTE } from '@atproto/common' +import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { MINUTE } from '@atproto/common' - -const REASON_ACCT_DELETION = 'account_deletion' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.deleteAccount({ @@ -13,41 +12,37 @@ export default function (server: Server, ctx: AppContext) { }, handler: async ({ input, req }) => { const { did, password, token } = input.body - const validPass = await ctx.services - .account(ctx.db) - .verifyAccountPassword(did, password) - if (!validPass) { - throw new AuthRequiredError('Invalid did or password') + + const account = await ctx.accountManager.getAccount(did, true) + if (!account) { + throw new InvalidRequestError('account not found') } - await ctx.services - .account(ctx.db) - .assertValidToken(did, 'delete_account', token) + if (ctx.entrywayAgent) { + await ctx.entrywayAgent.com.atproto.server.deleteAccount( + input.body, + authPassthru(req, true), + ) + return + } - await ctx.db.transaction(async (dbTxn) => { - const accountService = ctx.services.account(dbTxn) - const moderationTxn = ctx.services.moderation(dbTxn) - const currState = await moderationTxn.getRepoTakedownState(did) - // Do not disturb an existing takedown, continue with account deletion - if (currState?.takedown.applied !== true) { - await moderationTxn.updateRepoTakedownState(did, { - applied: true, - ref: REASON_ACCT_DELETION, - }) - } - await accountService.deleteEmailToken(did, 'delete_account') - }) + const validPass = await ctx.accountManager.verifyAccountPassword( + did, + password, + ) + if (!validPass) { + throw new AuthRequiredError('Invalid did or password') + } - ctx.backgroundQueue.add(async (db) => { - try { - // In the background perform the hard account deletion work - await ctx.services.record(db).deleteForActor(did) - await ctx.services.repo(db).deleteRepo(did) - await ctx.services.account(db).deleteAccount(did) - } catch (err) { - req.log.error({ did, err }, 'account deletion failed') - } - }) + await ctx.accountManager.assertValidEmailToken( + did, + 'delete_account', + token, + ) + await ctx.actorStore.destroy(did) + await ctx.accountManager.deleteAccount(did) + await ctx.sequencer.sequenceTombstone(did) + await ctx.sequencer.deleteAllForUser(did) }, }) } diff --git a/packages/pds/src/api/com/atproto/server/deleteSession.ts b/packages/pds/src/api/com/atproto/server/deleteSession.ts index 2fa93528515..22a2055048f 100644 --- a/packages/pds/src/api/com/atproto/server/deleteSession.ts +++ b/packages/pds/src/api/com/atproto/server/deleteSession.ts @@ -1,19 +1,28 @@ import { AuthScope } from '../../../../auth-verifier' import AppContext from '../../../../context' import { Server } from '../../../../lexicon' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.deleteSession(async ({ req }) => { - const result = ctx.authVerifier.validateBearerToken( + if (ctx.entrywayAgent) { + await ctx.entrywayAgent.com.atproto.server.deleteSession( + undefined, + authPassthru(req, true), + ) + return + } + + const result = await ctx.authVerifier.validateBearerToken( req, [AuthScope.Refresh], - { ignoreExpiration: true }, + { clockTolerance: Infinity }, // ignore expiration ) const id = result.payload.jti if (!id) { throw new Error('Unexpected missing refresh token id') } - await ctx.services.auth(ctx.db).revokeRefreshToken(id) + await ctx.accountManager.revokeRefreshToken(id) }) } diff --git a/packages/pds/src/api/com/atproto/server/getAccountInviteCodes.ts b/packages/pds/src/api/com/atproto/server/getAccountInviteCodes.ts index 31cfe60ad3a..4e578214fe0 100644 --- a/packages/pds/src/api/com/atproto/server/getAccountInviteCodes.ts +++ b/packages/pds/src/api/com/atproto/server/getAccountInviteCodes.ts @@ -1,82 +1,60 @@ +import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { genInvCodes } from './util' -import { InvalidRequestError } from '@atproto/xrpc-server' -import { CodeDetail } from '../../../../services/account' +import { CodeDetail } from '../../../../account-manager/helpers/invite' +import { authPassthru, resultPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.getAccountInviteCodes({ auth: ctx.authVerifier.accessNotAppPassword, - handler: async ({ params, auth }) => { + handler: async ({ params, auth, req }) => { + if (ctx.entrywayAgent) { + return resultPassthru( + await ctx.entrywayAgent.com.atproto.server.getAccountInviteCodes( + params, + authPassthru(req), + ), + ) + } + const requester = auth.credentials.did const { includeUsed, createAvailable } = params - const accntSrvc = ctx.services.account(ctx.db) - - const [user, userCodes] = await Promise.all([ - ctx.db.db - .selectFrom('user_account') - .where('did', '=', requester) - .select(['invitesDisabled', 'createdAt']) - .executeTakeFirstOrThrow(), - accntSrvc.getAccountInviteCodes(requester), + const [account, userCodes] = await Promise.all([ + ctx.accountManager.getAccount(requester), + ctx.accountManager.getAccountInvitesCodes(requester), ]) + if (!account) { + throw new InvalidRequestError('Account not found', 'NotFound') + } - let created: string[] = [] + let created: CodeDetail[] = [] - const now = new Date().toISOString() if ( createAvailable && ctx.cfg.invites.required && ctx.cfg.invites.interval !== null ) { - const { toCreate, total } = await calculateCodesToCreate({ + const { toCreate, total } = calculateCodesToCreate({ did: requester, - userCreatedAt: new Date(user.createdAt).getTime(), + userCreatedAt: new Date(account.createdAt).getTime(), codes: userCodes, epoch: ctx.cfg.invites.epoch, interval: ctx.cfg.invites.interval, }) if (toCreate > 0) { - created = genInvCodes(ctx.cfg, toCreate) - const rows = created.map((code) => ({ - code: code, - availableUses: 1, - disabled: user.invitesDisabled, - forUser: requester, - createdBy: requester, - createdAt: now, - })) - await ctx.db.transaction(async (dbTxn) => { - await dbTxn.db.insertInto('invite_code').values(rows).execute() - const finalRoutineInviteCodes = await dbTxn.db - .selectFrom('invite_code') - .where('forUser', '=', requester) - .where('createdBy', '!=', 'admin') // dont count admin-gifted codes aginast the user - .selectAll() - .execute() - if (finalRoutineInviteCodes.length > total) { - throw new InvalidRequestError( - 'attempted to create additional codes in another request', - 'DuplicateCreate', - ) - } - }) + const codes = genInvCodes(ctx.cfg, toCreate) + created = await ctx.accountManager.createAccountInviteCodes( + requester, + codes, + total, + account.invitesDisabled ?? 0, + ) } } - const allCodes = [ - ...userCodes, - ...created.map((code) => ({ - code: code, - available: 1, - disabled: user.invitesDisabled === 1 ? true : false, - forAccount: requester, - createdBy: requester, - createdAt: now, - uses: [], - })), - ] + const allCodes = [...userCodes, ...created] const filtered = allCodes.filter((code) => { if (code.disabled) return false @@ -102,13 +80,13 @@ export default function (server: Server, ctx: AppContext) { * we allow a max of 5 open codes at a given time * note: even if a user is disabled from future invites, we still create the invites for bookkeeping, we just immediately disable them as well */ -const calculateCodesToCreate = async (opts: { +const calculateCodesToCreate = (opts: { did: string userCreatedAt: number codes: CodeDetail[] epoch: number interval: number -}): Promise<{ toCreate: number; total: number }> => { +}): { toCreate: number; total: number } => { // for the sake of generating routine interval codes, we do not count explicitly gifted admin codes const routineCodes = opts.codes.filter((code) => code.createdBy !== 'admin') const unusedRoutineCodes = routineCodes.filter( diff --git a/packages/pds/src/api/com/atproto/server/getSession.ts b/packages/pds/src/api/com/atproto/server/getSession.ts index bf271a9c02e..8b81068a147 100644 --- a/packages/pds/src/api/com/atproto/server/getSession.ts +++ b/packages/pds/src/api/com/atproto/server/getSession.ts @@ -1,15 +1,26 @@ import { InvalidRequestError } from '@atproto/xrpc-server' +import { INVALID_HANDLE } from '@atproto/syntax' import AppContext from '../../../../context' import { Server } from '../../../../lexicon' +import { authPassthru, resultPassthru } from '../../../proxy' import { didDocForSession } from './util' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.getSession({ auth: ctx.authVerifier.access, - handler: async ({ auth }) => { + handler: async ({ auth, req }) => { + if (ctx.entrywayAgent) { + return resultPassthru( + await ctx.entrywayAgent.com.atproto.server.getSession( + undefined, + authPassthru(req), + ), + ) + } + const did = auth.credentials.did const [user, didDoc] = await Promise.all([ - ctx.services.account(ctx.db).getAccount(did), + ctx.accountManager.getAccount(did), didDocForSession(ctx, did), ]) if (!user) { @@ -20,10 +31,10 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: { - handle: user.handle, + handle: user.handle ?? INVALID_HANDLE, did: user.did, + email: user.email ?? undefined, didDoc, - email: user.email, emailConfirmed: !!user.emailConfirmedAt, }, } diff --git a/packages/pds/src/api/com/atproto/server/index.ts b/packages/pds/src/api/com/atproto/server/index.ts index 210d0f45461..f5ab1245c1c 100644 --- a/packages/pds/src/api/com/atproto/server/index.ts +++ b/packages/pds/src/api/com/atproto/server/index.ts @@ -7,6 +7,7 @@ import createAccount from './createAccount' import createInviteCode from './createInviteCode' import createInviteCodes from './createInviteCodes' import getAccountInviteCodes from './getAccountInviteCodes' +import reserveSigningKey from './reserveSigningKey' import requestDelete from './requestAccountDelete' import deleteAccount from './deleteAccount' @@ -35,6 +36,7 @@ export default function (server: Server, ctx: AppContext) { createInviteCode(server, ctx) createInviteCodes(server, ctx) getAccountInviteCodes(server, ctx) + reserveSigningKey(server, ctx) requestDelete(server, ctx) deleteAccount(server, ctx) requestPasswordReset(server, ctx) diff --git a/packages/pds/src/api/com/atproto/server/listAppPasswords.ts b/packages/pds/src/api/com/atproto/server/listAppPasswords.ts index 75cd6392c9f..c17a1ab5e69 100644 --- a/packages/pds/src/api/com/atproto/server/listAppPasswords.ts +++ b/packages/pds/src/api/com/atproto/server/listAppPasswords.ts @@ -1,13 +1,23 @@ import AppContext from '../../../../context' import { Server } from '../../../../lexicon' +import { authPassthru, resultPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.listAppPasswords({ auth: ctx.authVerifier.access, - handler: async ({ auth }) => { - const passwords = await ctx.services - .account(ctx.db) - .listAppPasswords(auth.credentials.did) + handler: async ({ auth, req }) => { + if (ctx.entrywayAgent) { + return resultPassthru( + await ctx.entrywayAgent.com.atproto.server.listAppPasswords( + undefined, + authPassthru(req), + ), + ) + } + + const passwords = await ctx.accountManager.listAppPasswords( + auth.credentials.did, + ) return { encoding: 'application/json', body: { passwords }, diff --git a/packages/pds/src/api/com/atproto/server/refreshSession.ts b/packages/pds/src/api/com/atproto/server/refreshSession.ts index 0ab39e4d8cc..9b9d4f6a5fd 100644 --- a/packages/pds/src/api/com/atproto/server/refreshSession.ts +++ b/packages/pds/src/api/com/atproto/server/refreshSession.ts @@ -1,15 +1,17 @@ +import { INVALID_HANDLE } from '@atproto/syntax' import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import AppContext from '../../../../context' import { softDeleted } from '../../../../db/util' import { Server } from '../../../../lexicon' import { didDocForSession } from './util' +import { authPassthru, resultPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.refreshSession({ auth: ctx.authVerifier.refresh, - handler: async ({ auth }) => { + handler: async ({ auth, req }) => { const did = auth.credentials.did - const user = await ctx.services.account(ctx.db).getAccount(did, true) + const user = await ctx.accountManager.getAccount(did, true) if (!user) { throw new InvalidRequestError( `Could not find user info for account: ${did}`, @@ -22,13 +24,18 @@ export default function (server: Server, ctx: AppContext) { ) } + if (ctx.entrywayAgent) { + return resultPassthru( + await ctx.entrywayAgent.com.atproto.server.refreshSession( + undefined, + authPassthru(req), + ), + ) + } + const [didDoc, rotated] = await Promise.all([ didDocForSession(ctx, user.did), - ctx.db.transaction((dbTxn) => { - return ctx.services - .auth(dbTxn) - .rotateRefreshToken(auth.credentials.tokenId) - }), + ctx.accountManager.rotateRefreshToken(auth.credentials.tokenId), ]) if (rotated === null) { throw new InvalidRequestError('Token has been revoked', 'ExpiredToken') @@ -39,9 +46,9 @@ export default function (server: Server, ctx: AppContext) { body: { did: user.did, didDoc, - handle: user.handle, - accessJwt: rotated.access.jwt, - refreshJwt: rotated.refresh.jwt, + handle: user.handle ?? INVALID_HANDLE, + accessJwt: rotated.accessJwt, + refreshJwt: rotated.refreshJwt, }, } }, diff --git a/packages/pds/src/api/com/atproto/server/requestAccountDelete.ts b/packages/pds/src/api/com/atproto/server/requestAccountDelete.ts index ca895852f4f..2566682d244 100644 --- a/packages/pds/src/api/com/atproto/server/requestAccountDelete.ts +++ b/packages/pds/src/api/com/atproto/server/requestAccountDelete.ts @@ -1,20 +1,34 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.requestAccountDelete({ auth: ctx.authVerifier.accessCheckTakedown, - handler: async ({ auth }) => { + handler: async ({ auth, req }) => { const did = auth.credentials.did - const user = await ctx.services.account(ctx.db).getAccount(did) - if (!user) { - throw new InvalidRequestError('user not found') + const account = await ctx.accountManager.getAccount(did) + if (!account) { + throw new InvalidRequestError('account not found') } - const token = await ctx.services - .account(ctx.db) - .createEmailToken(did, 'delete_account') - await ctx.mailer.sendAccountDelete({ token }, { to: user.email }) + + if (ctx.entrywayAgent) { + await ctx.entrywayAgent.com.atproto.server.requestAccountDelete( + undefined, + authPassthru(req), + ) + return + } + + if (!account.email) { + throw new InvalidRequestError('account does not have an email address') + } + const token = await ctx.accountManager.createEmailToken( + did, + 'delete_account', + ) + await ctx.mailer.sendAccountDelete({ token }, { to: account.email }) }, }) } diff --git a/packages/pds/src/api/com/atproto/server/requestEmailConfirmation.ts b/packages/pds/src/api/com/atproto/server/requestEmailConfirmation.ts index 97b2e53cc7a..76703d0d6d2 100644 --- a/packages/pds/src/api/com/atproto/server/requestEmailConfirmation.ts +++ b/packages/pds/src/api/com/atproto/server/requestEmailConfirmation.ts @@ -1,20 +1,34 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.requestEmailConfirmation({ auth: ctx.authVerifier.accessCheckTakedown, - handler: async ({ auth }) => { + handler: async ({ auth, req }) => { const did = auth.credentials.did - const user = await ctx.services.account(ctx.db).getAccount(did) - if (!user) { - throw new InvalidRequestError('user not found') + const account = await ctx.accountManager.getAccount(did) + if (!account) { + throw new InvalidRequestError('account not found') } - const token = await ctx.services - .account(ctx.db) - .createEmailToken(did, 'confirm_email') - await ctx.mailer.sendConfirmEmail({ token }, { to: user.email }) + + if (ctx.entrywayAgent) { + await ctx.entrywayAgent.com.atproto.server.requestEmailConfirmation( + undefined, + authPassthru(req), + ) + return + } + + if (!account.email) { + throw new InvalidRequestError('account does not have an email address') + } + const token = await ctx.accountManager.createEmailToken( + did, + 'confirm_email', + ) + await ctx.mailer.sendConfirmEmail({ token }, { to: account.email }) }, }) } diff --git a/packages/pds/src/api/com/atproto/server/requestEmailUpdate.ts b/packages/pds/src/api/com/atproto/server/requestEmailUpdate.ts index 5402fa6b887..696a7d2c018 100644 --- a/packages/pds/src/api/com/atproto/server/requestEmailUpdate.ts +++ b/packages/pds/src/api/com/atproto/server/requestEmailUpdate.ts @@ -1,23 +1,38 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' +import { authPassthru, resultPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.requestEmailUpdate({ auth: ctx.authVerifier.accessCheckTakedown, - handler: async ({ auth }) => { + handler: async ({ auth, req }) => { const did = auth.credentials.did - const user = await ctx.services.account(ctx.db).getAccount(did) - if (!user) { - throw new InvalidRequestError('user not found') + const account = await ctx.accountManager.getAccount(did) + if (!account) { + throw new InvalidRequestError('account not found') } - const tokenRequired = !!user.emailConfirmedAt + if (ctx.entrywayAgent) { + return resultPassthru( + await ctx.entrywayAgent.com.atproto.server.requestEmailUpdate( + undefined, + authPassthru(req), + ), + ) + } + + if (!account.email) { + throw new InvalidRequestError('account does not have an email address') + } + + const tokenRequired = !!account.emailConfirmedAt if (tokenRequired) { - const token = await ctx.services - .account(ctx.db) - .createEmailToken(did, 'update_email') - await ctx.mailer.sendUpdateEmail({ token }, { to: user.email }) + const token = await ctx.accountManager.createEmailToken( + did, + 'update_email', + ) + await ctx.mailer.sendUpdateEmail({ token }, { to: account.email }) } return { diff --git a/packages/pds/src/api/com/atproto/server/requestPasswordReset.ts b/packages/pds/src/api/com/atproto/server/requestPasswordReset.ts index 61b17ebb9a9..4afdb0add83 100644 --- a/packages/pds/src/api/com/atproto/server/requestPasswordReset.ts +++ b/packages/pds/src/api/com/atproto/server/requestPasswordReset.ts @@ -1,20 +1,32 @@ +import { InvalidRequestError } from '@atproto/xrpc-server' import AppContext from '../../../../context' import { Server } from '../../../../lexicon' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { - server.com.atproto.server.requestPasswordReset(async ({ input }) => { + server.com.atproto.server.requestPasswordReset(async ({ input, req }) => { const email = input.body.email.toLowerCase() - const user = await ctx.services.account(ctx.db).getAccountByEmail(email) + const account = await ctx.accountManager.getAccountByEmail(email) - if (user) { - const token = await ctx.services - .account(ctx.db) - .createEmailToken(user.did, 'reset_password') - await ctx.mailer.sendResetPassword( - { handle: user.handle, token }, - { to: user.email }, - ) + if (!account?.email) { + if (ctx.entrywayAgent) { + await ctx.entrywayAgent.com.atproto.server.requestPasswordReset( + input.body, + authPassthru(req, true), + ) + return + } + throw new InvalidRequestError('account does not have an email address') } + + const token = await ctx.accountManager.createEmailToken( + account.did, + 'reset_password', + ) + await ctx.mailer.sendResetPassword( + { identifier: account.handle ?? account.email, token }, + { to: account.email }, + ) }) } diff --git a/packages/pds/src/api/com/atproto/server/reserveSigningKey.ts b/packages/pds/src/api/com/atproto/server/reserveSigningKey.ts new file mode 100644 index 00000000000..177bf4aff78 --- /dev/null +++ b/packages/pds/src/api/com/atproto/server/reserveSigningKey.ts @@ -0,0 +1,16 @@ +import { Server } from '../../../../lexicon' +import AppContext from '../../../../context' + +export default function (server: Server, ctx: AppContext) { + server.com.atproto.server.reserveSigningKey({ + handler: async ({ input }) => { + const signingKey = await ctx.actorStore.reserveKeypair(input.body.did) + return { + encoding: 'application/json', + body: { + signingKey, + }, + } + }, + }) +} diff --git a/packages/pds/src/api/com/atproto/server/resetPassword.ts b/packages/pds/src/api/com/atproto/server/resetPassword.ts index a84b6249a3c..6194680e683 100644 --- a/packages/pds/src/api/com/atproto/server/resetPassword.ts +++ b/packages/pds/src/api/com/atproto/server/resetPassword.ts @@ -1,6 +1,7 @@ import AppContext from '../../../../context' import { Server } from '../../../../lexicon' import { MINUTE } from '@atproto/common' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.resetPassword({ @@ -10,19 +11,18 @@ export default function (server: Server, ctx: AppContext) { points: 50, }, ], - handler: async ({ input }) => { - const { token, password } = input.body + handler: async ({ input, req }) => { + if (ctx.entrywayAgent) { + await ctx.entrywayAgent.com.atproto.server.resetPassword( + input.body, + authPassthru(req, true), + ) + return + } - const did = await ctx.services - .account(ctx.db) - .assertValidTokenAndFindDid('reset_password', token) + const { token, password } = input.body - await ctx.db.transaction(async (dbTxn) => { - const accountService = ctx.services.account(ctx.db) - await accountService.updateUserPassword(did, password) - await accountService.deleteEmailToken(did, 'reset_password') - await ctx.services.auth(dbTxn).revokeRefreshTokensByDid(did) - }) + await ctx.accountManager.resetPassword({ token, password }) }, }) } diff --git a/packages/pds/src/api/com/atproto/server/revokeAppPassword.ts b/packages/pds/src/api/com/atproto/server/revokeAppPassword.ts index 43398576d6a..ce28bf07f1f 100644 --- a/packages/pds/src/api/com/atproto/server/revokeAppPassword.ts +++ b/packages/pds/src/api/com/atproto/server/revokeAppPassword.ts @@ -1,18 +1,23 @@ import AppContext from '../../../../context' import { Server } from '../../../../lexicon' +import { authPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.revokeAppPassword({ auth: ctx.authVerifier.access, - handler: async ({ auth, input }) => { + handler: async ({ auth, input, req }) => { + if (ctx.entrywayAgent) { + await ctx.entrywayAgent.com.atproto.server.revokeAppPassword( + input.body, + authPassthru(req, true), + ) + return + } + const requester = auth.credentials.did const { name } = input.body - await ctx.db.transaction(async (dbTxn) => { - await ctx.services.account(dbTxn).deleteAppPassword(requester, name) - await ctx.services - .auth(dbTxn) - .revokeAppPasswordRefreshToken(requester, name) - }) + + await ctx.accountManager.revokeAppPassword(requester, name) }, }) } diff --git a/packages/pds/src/api/com/atproto/server/updateEmail.ts b/packages/pds/src/api/com/atproto/server/updateEmail.ts index e0f9d9bc078..52d27a64ae4 100644 --- a/packages/pds/src/api/com/atproto/server/updateEmail.ts +++ b/packages/pds/src/api/com/atproto/server/updateEmail.ts @@ -2,12 +2,13 @@ import disposable from 'disposable-email' import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { UserAlreadyExistsError } from '../../../../services/account' +import { authPassthru } from '../../../proxy' +import { UserAlreadyExistsError } from '../../../../account-manager/helpers/account' export default function (server: Server, ctx: AppContext) { server.com.atproto.server.updateEmail({ auth: ctx.authVerifier.accessNotAppPassword, - handler: async ({ auth, input }) => { + handler: async ({ auth, input, req }) => { const did = auth.credentials.did const { token, email } = input.body if (!disposable.validate(email)) { @@ -15,43 +16,45 @@ export default function (server: Server, ctx: AppContext) { 'This email address is not supported, please use a different email.', ) } - const user = await ctx.services.account(ctx.db).getAccount(did) - if (!user) { - throw new InvalidRequestError('user not found') + const account = await ctx.accountManager.getAccount(did) + if (!account) { + throw new InvalidRequestError('account not found') } - // require valid token if user email is confirmed - if (user.emailConfirmedAt) { + + if (ctx.entrywayAgent) { + await ctx.entrywayAgent.com.atproto.server.updateEmail( + input.body, + authPassthru(req, true), + ) + return + } + + // require valid token if account email is confirmed + if (account.emailConfirmedAt) { if (!token) { throw new InvalidRequestError( 'confirmation token required', 'TokenRequired', ) } - await ctx.services - .account(ctx.db) - .assertValidToken(did, 'update_email', token) + await ctx.accountManager.assertValidEmailToken( + did, + 'update_email', + token, + ) } - await ctx.db.transaction(async (dbTxn) => { - const accntSrvce = ctx.services.account(dbTxn) - - if (token) { - await accntSrvce.deleteEmailToken(did, 'update_email') - } - if (user.email !== email) { - try { - await accntSrvce.updateEmail(did, email) - } catch (err) { - if (err instanceof UserAlreadyExistsError) { - throw new InvalidRequestError( - 'This email address is already in use, please use a different email.', - ) - } else { - throw err - } - } + try { + await ctx.accountManager.updateEmail({ did, email, token }) + } catch (err) { + if (err instanceof UserAlreadyExistsError) { + throw new InvalidRequestError( + 'This email address is already in use, please use a different email.', + ) + } else { + throw err } - }) + } }, }) } diff --git a/packages/pds/src/api/com/atproto/sync/deprecated/getCheckout.ts b/packages/pds/src/api/com/atproto/sync/deprecated/getCheckout.ts index 899706986a5..cb73d23eebe 100644 --- a/packages/pds/src/api/com/atproto/sync/deprecated/getCheckout.ts +++ b/packages/pds/src/api/com/atproto/sync/deprecated/getCheckout.ts @@ -1,10 +1,7 @@ import { InvalidRequestError } from '@atproto/xrpc-server' -import { byteIterableToStream } from '@atproto/common' import { Server } from '../../../../../lexicon' -import SqlRepoStorage, { - RepoRootNotFoundError, -} from '../../../../../sql-repo-storage' import AppContext from '../../../../../context' +import { getCarStream } from '../getRepo' export default function (server: Server, ctx: AppContext) { server.com.atproto.sync.getCheckout({ @@ -13,28 +10,17 @@ export default function (server: Server, ctx: AppContext) { const { did } = params // takedown check for anyone other than an admin or the user if (!ctx.authVerifier.isUserOrAdmin(auth, did)) { - const available = await ctx.services - .account(ctx.db) - .isRepoAvailable(did) + const available = await ctx.accountManager.isRepoAvailable(did) if (!available) { throw new InvalidRequestError(`Could not find repo for DID: ${did}`) } } - const storage = new SqlRepoStorage(ctx.db, did) - let carStream: AsyncIterable - try { - carStream = await storage.getCarStream() - } catch (err) { - if (err instanceof RepoRootNotFoundError) { - throw new InvalidRequestError(`Could not find repo for DID: ${did}`) - } - throw err - } + const carStream = await getCarStream(ctx, did) return { encoding: 'application/vnd.ipld.car', - body: byteIterableToStream(carStream), + body: carStream, } }, }) diff --git a/packages/pds/src/api/com/atproto/sync/deprecated/getHead.ts b/packages/pds/src/api/com/atproto/sync/deprecated/getHead.ts index c8499f92faa..45be946539a 100644 --- a/packages/pds/src/api/com/atproto/sync/deprecated/getHead.ts +++ b/packages/pds/src/api/com/atproto/sync/deprecated/getHead.ts @@ -1,6 +1,5 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../../lexicon' -import SqlRepoStorage from '../../../../../sql-repo-storage' import AppContext from '../../../../../context' export default function (server: Server, ctx: AppContext) { @@ -10,9 +9,7 @@ export default function (server: Server, ctx: AppContext) { const { did } = params // takedown check for anyone other than an admin or the user if (!ctx.authVerifier.isUserOrAdmin(auth, did)) { - const available = await ctx.services - .account(ctx.db) - .isRepoAvailable(did) + const available = await ctx.accountManager.isRepoAvailable(did) if (!available) { throw new InvalidRequestError( `Could not find root for DID: ${did}`, @@ -20,8 +17,9 @@ export default function (server: Server, ctx: AppContext) { ) } } - const storage = new SqlRepoStorage(ctx.db, did) - const root = await storage.getRoot() + const root = await ctx.actorStore.read(did, (store) => + store.repo.storage.getRoot(), + ) if (root === null) { throw new InvalidRequestError( `Could not find root for DID: ${did}`, diff --git a/packages/pds/src/api/com/atproto/sync/getBlob.ts b/packages/pds/src/api/com/atproto/sync/getBlob.ts index 7de146fcc5d..3b3d1d2f65a 100644 --- a/packages/pds/src/api/com/atproto/sync/getBlob.ts +++ b/packages/pds/src/api/com/atproto/sync/getBlob.ts @@ -2,43 +2,32 @@ import { CID } from 'multiformats/cid' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { InvalidRequestError } from '@atproto/xrpc-server' -import { notSoftDeletedClause } from '../../../../db/util' import { BlobNotFoundError } from '@atproto/repo' export default function (server: Server, ctx: AppContext) { server.com.atproto.sync.getBlob({ auth: ctx.authVerifier.optionalAccessOrRole, handler: async ({ params, res, auth }) => { - const { ref } = ctx.db.db.dynamic - const found = await ctx.db.db - .selectFrom('blob') - .selectAll() - .innerJoin('repo_root', 'repo_root.did', 'blob.creator') - .innerJoin('repo_blob', (join) => - join - .onRef('repo_blob.cid', '=', 'blob.cid') - .onRef('repo_blob.did', '=', 'blob.creator'), - ) - .where('blob.cid', '=', params.cid) - .where('blob.creator', '=', params.did) - .where(notSoftDeletedClause(ref('repo_blob'))) - .if(!ctx.authVerifier.isUserOrAdmin(auth, params.did), (qb) => - // takedown check for anyone other than an admin or the user - qb.where(notSoftDeletedClause(ref('repo_root'))), - ) - .executeTakeFirst() - if (!found) { - throw new InvalidRequestError('Blob not found') + if (!ctx.authVerifier.isUserOrAdmin(auth, params.did)) { + const available = await ctx.accountManager.isRepoAvailable(params.did) + if (!available) { + throw new InvalidRequestError('Blob not found') + } } const cid = CID.parse(params.cid) - let blobStream - try { - blobStream = await ctx.blobstore.getStream(cid) - } catch (err) { - if (err instanceof BlobNotFoundError) { - throw new InvalidRequestError('Blob not found') + const found = await ctx.actorStore.read(params.did, async (store) => { + try { + return await store.repo.blob.getBlob(cid) + } catch (err) { + if (err instanceof BlobNotFoundError) { + throw new InvalidRequestError('Blob not found') + } else { + throw err + } } - throw err + }) + if (!found) { + throw new InvalidRequestError('Blob not found') } res.setHeader('content-length', found.size) res.setHeader('x-content-type-options', 'nosniff') @@ -46,7 +35,7 @@ export default function (server: Server, ctx: AppContext) { return { // @TODO better codegen for */* mimetype encoding: (found.mimeType || 'application/octet-stream') as '*/*', - body: blobStream, + body: found.stream, } }, }) diff --git a/packages/pds/src/api/com/atproto/sync/getBlocks.ts b/packages/pds/src/api/com/atproto/sync/getBlocks.ts index 1ee859255ab..cd0d00356e0 100644 --- a/packages/pds/src/api/com/atproto/sync/getBlocks.ts +++ b/packages/pds/src/api/com/atproto/sync/getBlocks.ts @@ -3,7 +3,6 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import { byteIterableToStream } from '@atproto/common' import { blocksToCarStream } from '@atproto/repo' import { Server } from '../../../../lexicon' -import SqlRepoStorage from '../../../../sql-repo-storage' import AppContext from '../../../../context' export default function (server: Server, ctx: AppContext) { @@ -13,17 +12,16 @@ export default function (server: Server, ctx: AppContext) { const { did } = params // takedown check for anyone other than an admin or the user if (!ctx.authVerifier.isUserOrAdmin(auth, did)) { - const available = await ctx.services - .account(ctx.db) - .isRepoAvailable(did) + const available = await ctx.accountManager.isRepoAvailable(did) if (!available) { throw new InvalidRequestError(`Could not find repo for DID: ${did}`) } } const cids = params.cids.map((c) => CID.parse(c)) - const storage = new SqlRepoStorage(ctx.db, did) - const got = await storage.getBlocks(cids) + const got = await ctx.actorStore.read(did, (store) => + store.repo.storage.getBlocks(cids), + ) if (got.missing.length > 0) { const missingStr = got.missing.map((c) => c.toString()) throw new InvalidRequestError(`Could not find cids: ${missingStr}`) diff --git a/packages/pds/src/api/com/atproto/sync/getLatestCommit.ts b/packages/pds/src/api/com/atproto/sync/getLatestCommit.ts index a28c1ad0edd..9a48681c21d 100644 --- a/packages/pds/src/api/com/atproto/sync/getLatestCommit.ts +++ b/packages/pds/src/api/com/atproto/sync/getLatestCommit.ts @@ -1,6 +1,5 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' -import SqlRepoStorage from '../../../../sql-repo-storage' import AppContext from '../../../../context' export default function (server: Server, ctx: AppContext) { @@ -10,9 +9,7 @@ export default function (server: Server, ctx: AppContext) { const { did } = params // takedown check for anyone other than an admin or the user if (!ctx.authVerifier.isUserOrAdmin(auth, did)) { - const available = await ctx.services - .account(ctx.db) - .isRepoAvailable(did) + const available = await ctx.accountManager.isRepoAvailable(did) if (!available) { throw new InvalidRequestError( `Could not find root for DID: ${did}`, @@ -20,8 +17,9 @@ export default function (server: Server, ctx: AppContext) { ) } } - const storage = new SqlRepoStorage(ctx.db, did) - const root = await storage.getRootDetailed() + const root = await ctx.actorStore.read(did, (store) => + store.repo.storage.getRootDetailed(), + ) if (root === null) { throw new InvalidRequestError( `Could not find root for DID: ${did}`, diff --git a/packages/pds/src/api/com/atproto/sync/getRecord.ts b/packages/pds/src/api/com/atproto/sync/getRecord.ts index dc8c7e78f08..7d3641af0ff 100644 --- a/packages/pds/src/api/com/atproto/sync/getRecord.ts +++ b/packages/pds/src/api/com/atproto/sync/getRecord.ts @@ -1,10 +1,11 @@ +import stream from 'stream' import { CID } from 'multiformats/cid' import * as repo from '@atproto/repo' import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' -import SqlRepoStorage from '../../../../sql-repo-storage' import AppContext from '../../../../context' import { byteIterableToStream } from '@atproto/common' +import { SqlRepoReader } from '../../../../actor-store/repo/sql-repo-reader' export default function (server: Server, ctx: AppContext) { server.com.atproto.sync.getRecord({ @@ -13,24 +14,37 @@ export default function (server: Server, ctx: AppContext) { const { did, collection, rkey } = params // takedown check for anyone other than an admin or the user if (!ctx.authVerifier.isUserOrAdmin(auth, did)) { - const available = await ctx.services - .account(ctx.db) - .isRepoAvailable(did) + const available = await ctx.accountManager.isRepoAvailable(did) if (!available) { throw new InvalidRequestError(`Could not find repo for DID: ${did}`) } } - const storage = new SqlRepoStorage(ctx.db, did) - const commit = params.commit - ? CID.parse(params.commit) - : await storage.getRoot() - if (!commit) { - throw new InvalidRequestError(`Could not find repo for DID: ${did}`) + // must open up the db outside of store interface so that we can close the file handle after finished streaming + const actorDb = await ctx.actorStore.openDb(did) + + let carStream: stream.Readable + try { + const storage = new SqlRepoReader(actorDb) + const commit = params.commit + ? CID.parse(params.commit) + : await storage.getRoot() + + if (!commit) { + throw new InvalidRequestError(`Could not find repo for DID: ${did}`) + } + const carIter = repo.getRecords(storage, commit, [{ collection, rkey }]) + carStream = byteIterableToStream(carIter) + } catch (err) { + actorDb.close() + throw err } - const proof = repo.getRecords(storage, commit, [{ collection, rkey }]) + const closeDb = () => actorDb.close() + carStream.on('error', closeDb) + carStream.on('close', closeDb) + return { encoding: 'application/vnd.ipld.car', - body: byteIterableToStream(proof), + body: carStream, } }, }) diff --git a/packages/pds/src/api/com/atproto/sync/getRepo.ts b/packages/pds/src/api/com/atproto/sync/getRepo.ts index 6dc9c8af68b..ae07715d134 100644 --- a/packages/pds/src/api/com/atproto/sync/getRepo.ts +++ b/packages/pds/src/api/com/atproto/sync/getRepo.ts @@ -1,10 +1,11 @@ +import stream from 'stream' import { InvalidRequestError } from '@atproto/xrpc-server' -import { byteIterableToStream } from '@atproto/common' import { Server } from '../../../../lexicon' -import SqlRepoStorage, { - RepoRootNotFoundError, -} from '../../../../sql-repo-storage' import AppContext from '../../../../context' +import { + RepoRootNotFoundError, + SqlRepoReader, +} from '../../../../actor-store/repo/sql-repo-reader' export default function (server: Server, ctx: AppContext) { server.com.atproto.sync.getRepo({ @@ -13,29 +14,41 @@ export default function (server: Server, ctx: AppContext) { const { did, since } = params // takedown check for anyone other than an admin or the user if (!ctx.authVerifier.isUserOrAdmin(auth, did)) { - const available = await ctx.services - .account(ctx.db) - .isRepoAvailable(did) + const available = await ctx.accountManager.isRepoAvailable(did) if (!available) { throw new InvalidRequestError(`Could not find repo for DID: ${did}`) } } - const storage = new SqlRepoStorage(ctx.db, did) - let carStream: AsyncIterable - try { - carStream = await storage.getCarStream(since) - } catch (err) { - if (err instanceof RepoRootNotFoundError) { - throw new InvalidRequestError(`Could not find repo for DID: ${did}`) - } - throw err - } + const carStream = await getCarStream(ctx, did, since) return { encoding: 'application/vnd.ipld.car', - body: byteIterableToStream(carStream), + body: carStream, } }, }) } + +export const getCarStream = async ( + ctx: AppContext, + did: string, + since?: string, +): Promise => { + const actorDb = await ctx.actorStore.openDb(did) + let carStream: stream.Readable + try { + const storage = new SqlRepoReader(actorDb) + carStream = await storage.getCarStream(since) + } catch (err) { + await actorDb.close() + if (err instanceof RepoRootNotFoundError) { + throw new InvalidRequestError(`Could not find repo for DID: ${did}`) + } + throw err + } + const closeDb = () => actorDb.close() + carStream.on('error', closeDb) + carStream.on('close', closeDb) + return carStream +} diff --git a/packages/pds/src/api/com/atproto/sync/listBlobs.ts b/packages/pds/src/api/com/atproto/sync/listBlobs.ts index c6700d28dce..e71a2425bca 100644 --- a/packages/pds/src/api/com/atproto/sync/listBlobs.ts +++ b/packages/pds/src/api/com/atproto/sync/listBlobs.ts @@ -9,36 +9,21 @@ export default function (server: Server, ctx: AppContext) { const { did, since, limit, cursor } = params // takedown check for anyone other than an admin or the user if (!ctx.authVerifier.isUserOrAdmin(auth, did)) { - const available = await ctx.services - .account(ctx.db) - .isRepoAvailable(did) + const available = await ctx.accountManager.isRepoAvailable(did) if (!available) { throw new InvalidRequestError(`Could not find root for DID: ${did}`) } } - let builder = ctx.db.db - .selectFrom('repo_blob') - .where('did', '=', did) - .select('cid') - .orderBy('cid', 'asc') - .groupBy('cid') - .limit(limit) - if (since) { - builder = builder.where('repoRev', '>', since) - } - - if (cursor) { - builder = builder.where('cid', '>', cursor) - } - - const res = await builder.execute() + const blobCids = await ctx.actorStore.read(did, (store) => + store.repo.blob.listBlobs({ since, limit, cursor }), + ) return { encoding: 'application/json', body: { - cursor: res.at(-1)?.cid, - cids: res.map((row) => row.cid), + cursor: blobCids.at(-1), + cids: blobCids, }, } }, diff --git a/packages/pds/src/api/com/atproto/sync/listRepos.ts b/packages/pds/src/api/com/atproto/sync/listRepos.ts index d2829ddcc4a..8a9fe8170a4 100644 --- a/packages/pds/src/api/com/atproto/sync/listRepos.ts +++ b/packages/pds/src/api/com/atproto/sync/listRepos.ts @@ -7,21 +7,19 @@ import { notSoftDeletedClause } from '../../../../db/util' export default function (server: Server, ctx: AppContext) { server.com.atproto.sync.listRepos(async ({ params }) => { const { limit, cursor } = params - const { ref } = ctx.db.db.dynamic - let builder = ctx.db.db - .selectFrom('user_account') - .innerJoin('repo_root', 'repo_root.did', 'user_account.did') - .where(notSoftDeletedClause(ref('repo_root'))) + const db = ctx.accountManager.db + const { ref } = db.db.dynamic + let builder = db.db + .selectFrom('actor') + .innerJoin('repo_root', 'repo_root.did', 'actor.did') + .where(notSoftDeletedClause(ref('actor'))) .select([ - 'user_account.did as did', - 'repo_root.root as head', + 'actor.did as did', + 'repo_root.cid as head', 'repo_root.rev as rev', - 'user_account.createdAt as createdAt', + 'actor.createdAt as createdAt', ]) - const keyset = new TimeDidKeyset( - ref('user_account.createdAt'), - ref('user_account.did'), - ) + const keyset = new TimeDidKeyset(ref('actor.createdAt'), ref('actor.did')) builder = paginate(builder, { limit, cursor, diff --git a/packages/pds/src/api/com/atproto/sync/subscribeRepos.ts b/packages/pds/src/api/com/atproto/sync/subscribeRepos.ts index 7fc9be6f3ac..8302760a75f 100644 --- a/packages/pds/src/api/com/atproto/sync/subscribeRepos.ts +++ b/packages/pds/src/api/com/atproto/sync/subscribeRepos.ts @@ -21,7 +21,7 @@ export default function (server: Server, ctx: AppContext) { ctx.sequencer.next(cursor), ctx.sequencer.curr(), ]) - if (cursor > (curr?.seq ?? 0)) { + if (cursor > (curr ?? 0)) { throw new InvalidRequestError('Cursor in the future.', 'FutureCursor') } else if (next && next.sequencedAt < backfillTime) { // if cursor is before backfill time, find earliest cursor from backfill window diff --git a/packages/pds/src/api/com/atproto/temp/importRepo.ts b/packages/pds/src/api/com/atproto/temp/importRepo.ts new file mode 100644 index 00000000000..ff11dd5f6d1 --- /dev/null +++ b/packages/pds/src/api/com/atproto/temp/importRepo.ts @@ -0,0 +1,243 @@ +import { Readable } from 'stream' +import assert from 'assert' +import PQueue from 'p-queue' +import axios from 'axios' +import { CID } from 'multiformats/cid' +import { InvalidRequestError } from '@atproto/xrpc-server' +import { AsyncBuffer, TID, wait } from '@atproto/common' +import { AtUri } from '@atproto/syntax' +import { + Repo, + WriteOpAction, + getAndParseRecord, + readCarStream, + verifyDiff, +} from '@atproto/repo' +import { BlobRef, LexValue, RepoRecord } from '@atproto/lexicon' +import { Server } from '../../../../lexicon' +import AppContext from '../../../../context' +import { ActorStoreTransactor } from '../../../../actor-store' +import { AtprotoData } from '@atproto/identity' + +export default function (server: Server, ctx: AppContext) { + server.com.atproto.temp.importRepo({ + opts: { + blobLimit: 5 * 1024 * 1024 * 1024, // 5GB + }, + auth: ctx.authVerifier.role, + handler: async ({ params, input, req }) => { + const { did } = params + const outBuffer = new AsyncBuffer() + sendTicks(outBuffer).catch((err) => { + req.log.error({ err }, 'failed to send ticks') + }) + processImport(ctx, did, input.body, outBuffer).catch(async (err) => { + req.log.error({ did, err }, 'failed import') + try { + await ctx.actorStore.destroy(did) + } catch (err) { + req.log.error({ did, err }, 'failed to clean up actor store') + } + outBuffer.throw(err) + }) + + return { + encoding: 'text/plain', + body: Readable.from(outBuffer.events()), + } + }, + }) +} + +const sendTicks = async (outBuffer: AsyncBuffer) => { + while (!outBuffer.isClosed) { + outBuffer.push('tick\n') + await wait(1000) + } +} + +const processImport = async ( + ctx: AppContext, + did: string, + incomingCar: AsyncIterable, + outBuffer: AsyncBuffer, +) => { + const didData = await ctx.idResolver.did.resolveAtprotoData(did) + const alreadyExists = await ctx.actorStore.exists(did) + if (!alreadyExists) { + const keypair = await ctx.actorStore.getReservedKeypair(did) + if (!keypair) { + throw new InvalidRequestError('No signing key reserved') + } + await ctx.actorStore.create(did, keypair) + } + await ctx.actorStore.transact(did, async (actorStore) => { + const blobRefs = await importRepo(actorStore, incomingCar, outBuffer) + await importBlobs(actorStore, didData, blobRefs, outBuffer) + }) + outBuffer.close() +} + +const importRepo = async ( + actorStore: ActorStoreTransactor, + incomingCar: AsyncIterable, + outBuffer: AsyncBuffer, +) => { + const now = new Date().toISOString() + const rev = TID.nextStr() + const did = actorStore.repo.did + + const { roots, blocks } = await readCarStream(incomingCar) + if (roots.length !== 1) { + throw new InvalidRequestError('expected one root') + } + outBuffer.push(`read ${blocks.size} blocks\n`) + const currRoot = await actorStore.db.db + .selectFrom('repo_root') + .selectAll() + .executeTakeFirst() + const currRepo = currRoot + ? await Repo.load(actorStore.repo.storage, CID.parse(currRoot.cid)) + : null + const diff = await verifyDiff( + currRepo, + blocks, + roots[0], + undefined, + undefined, + { ensureLeaves: false }, + ) + outBuffer.push(`diffed repo and found ${diff.writes.length} writes\n`) + diff.commit.rev = rev + await actorStore.repo.storage.applyCommit(diff.commit, currRepo === null) + const recordQueue = new PQueue({ concurrency: 50 }) + let blobRefs: BlobRef[] = [] + let count = 0 + for (const write of diff.writes) { + recordQueue.add(async () => { + const uri = AtUri.make(did, write.collection, write.rkey) + if (write.action === WriteOpAction.Delete) { + await actorStore.record.deleteRecord(uri) + } else { + let parsedRecord: RepoRecord | null + try { + const parsed = await getAndParseRecord(blocks, write.cid) + parsedRecord = parsed.record + } catch { + parsedRecord = null + } + const indexRecord = actorStore.record.indexRecord( + uri, + write.cid, + parsedRecord, + write.action, + rev, + now, + ) + const recordBlobs = findBlobRefs(parsedRecord) + blobRefs = blobRefs.concat(recordBlobs) + const blobValues = recordBlobs.map((cid) => ({ + recordUri: uri.toString(), + blobCid: cid.ref.toString(), + })) + const indexRecordBlobs = + blobValues.length > 0 + ? actorStore.db.db + .insertInto('record_blob') + .values(blobValues) + .onConflict((oc) => oc.doNothing()) + .execute() + : Promise.resolve() + await Promise.all([indexRecord, indexRecordBlobs]) + } + count++ + if (count % 50 === 0) { + outBuffer.push(`indexed ${count}/${diff.writes.length} writes\n`) + } + }) + } + outBuffer.push(`indexed ${count}/${diff.writes.length} writes\n`) + await recordQueue.onIdle() + return blobRefs +} + +const importBlobs = async ( + actorStore: ActorStoreTransactor, + didData: AtprotoData, + blobRefs: BlobRef[], + outBuffer: AsyncBuffer, +) => { + let blobCount = 0 + const blobQueue = new PQueue({ concurrency: 10 }) + outBuffer.push(`fetching ${blobRefs.length} blobs\n`) + const endpoint = `${didData.pds}/xrpc/com.atproto.sync.getBlob` + for (const ref of blobRefs) { + blobQueue.add(async () => { + try { + await importBlob(actorStore, endpoint, ref) + blobCount++ + outBuffer.push(`imported ${blobCount}/${blobRefs.length} blobs\n`) + } catch (err) { + outBuffer.push(`failed to import blob: ${ref.ref.toString()}\n`) + } + }) + } + await blobQueue.onIdle() + outBuffer.push(`finished importing all blobs\n`) +} + +const importBlob = async ( + actorStore: ActorStoreTransactor, + endpoint: string, + blob: BlobRef, +) => { + const hasBlob = await actorStore.db.db + .selectFrom('blob') + .selectAll() + .where('cid', '=', blob.ref.toString()) + .executeTakeFirst() + if (hasBlob) { + return + } + const res = await axios.get(endpoint, { + params: { did: actorStore.repo.did, cid: blob.ref.toString() }, + decompress: true, + responseType: 'stream', + timeout: 5000, + }) + const mimeType = res.headers['content-type'] ?? 'application/octet-stream' + const importedRef = await actorStore.repo.blob.addUntetheredBlob( + mimeType, + res.data, + ) + assert(blob.ref.equals(importedRef.ref)) + await actorStore.repo.blob.verifyBlobAndMakePermanent({ + mimeType: blob.mimeType, + cid: blob.ref, + constraints: {}, + }) +} + +export const findBlobRefs = (val: LexValue, layer = 0): BlobRef[] => { + if (layer > 10) { + return [] + } + // walk arrays + if (Array.isArray(val)) { + return val.flatMap((item) => findBlobRefs(item, layer + 1)) + } + // objects + if (val && typeof val === 'object') { + // convert blobs, leaving the original encoding so that we don't change CIDs on re-encode + if (val instanceof BlobRef) { + return [val] + } + // retain cids & bytes + if (CID.asCID(val) || val instanceof Uint8Array) { + return [] + } + return Object.values(val).flatMap((item) => findBlobRefs(item, layer + 1)) + } + // pass through + return [] +} diff --git a/packages/pds/src/api/com/atproto/temp/index.ts b/packages/pds/src/api/com/atproto/temp/index.ts new file mode 100644 index 00000000000..8209dfe49f6 --- /dev/null +++ b/packages/pds/src/api/com/atproto/temp/index.ts @@ -0,0 +1,11 @@ +import AppContext from '../../../../context' +import { Server } from '../../../../lexicon' +import importRepo from './importRepo' +import pushBlob from './pushBlob' +import transferAccount from './transferAccount' + +export default function (server: Server, ctx: AppContext) { + importRepo(server, ctx) + pushBlob(server, ctx) + transferAccount(server, ctx) +} diff --git a/packages/pds/src/api/com/atproto/temp/pushBlob.ts b/packages/pds/src/api/com/atproto/temp/pushBlob.ts new file mode 100644 index 00000000000..74ef80e42c0 --- /dev/null +++ b/packages/pds/src/api/com/atproto/temp/pushBlob.ts @@ -0,0 +1,23 @@ +import { Server } from '../../../../lexicon' +import AppContext from '../../../../context' + +export default function (server: Server, ctx: AppContext) { + server.com.atproto.temp.pushBlob({ + auth: ctx.authVerifier.role, + handler: async ({ params, input }) => { + const { did } = params + + await ctx.actorStore.transact(did, async (actorTxn) => { + const blob = await actorTxn.repo.blob.addUntetheredBlob( + input.encoding, + input.body, + ) + await actorTxn.repo.blob.verifyBlobAndMakePermanent({ + mimeType: blob.mimeType, + cid: blob.ref, + constraints: {}, + }) + }) + }, + }) +} diff --git a/packages/pds/src/api/com/atproto/temp/transferAccount.ts b/packages/pds/src/api/com/atproto/temp/transferAccount.ts new file mode 100644 index 00000000000..0b1b765089c --- /dev/null +++ b/packages/pds/src/api/com/atproto/temp/transferAccount.ts @@ -0,0 +1,118 @@ +import { ensureAtpDocument } from '@atproto/identity' +import * as plc from '@did-plc/lib' +import { Server } from '../../../../lexicon' +import AppContext from '../../../../context' +import { check, cidForCbor } from '@atproto/common' +import { InvalidRequestError } from '@atproto/xrpc-server' +import { BlockMap, CidSet } from '@atproto/repo' + +export default function (server: Server, ctx: AppContext) { + server.com.atproto.temp.transferAccount({ + auth: ctx.authVerifier.role, + handler: async ({ input }) => { + const { did, handle } = input.body + + const signingKey = await ctx.actorStore.keypair(did) + const currRoot = await ctx.actorStore.read(did, (store) => + store.repo.storage.getRootDetailed(), + ) + + const plcOp = did.startsWith('did:plc') + ? await verifyDidAndPlcOp( + ctx, + did, + handle, + signingKey.did(), + input.body.plcOp, + ) + : null + + const { accessJwt, refreshJwt } = await ctx.accountManager.createAccount({ + did, + handle, + repoCid: currRoot.cid, + repoRev: currRoot.rev, + }) + + if (plcOp) { + try { + await ctx.plcClient.sendOperation(did, plcOp) + } catch (err) { + await ctx.accountManager.deleteAccount(did) + throw err + } + } + + await ctx.sequencer.sequenceCommit( + did, + { + cid: currRoot.cid, + rev: currRoot.rev, + since: null, + prev: null, + newBlocks: new BlockMap(), + removedCids: new CidSet(), + }, + [], + ) + + return { + encoding: 'application/json', + body: { + handle, + did: did, + accessJwt: accessJwt, + refreshJwt: refreshJwt, + }, + } + }, + }) +} + +const verifyDidAndPlcOp = async ( + ctx: AppContext, + did: string, + handle: string, + signingKey: string, + plcOp: unknown, +): Promise => { + if (!check.is(plcOp, plc.def.operation)) { + throw new InvalidRequestError('invalid plc operation', 'IncompatibleDidDoc') + } + await plc.assureValidOp(plcOp) + const prev = await ctx.plcClient.getLastOp(did) + if (!prev || prev.type === 'plc_tombstone') { + throw new InvalidRequestError( + 'no accessible prev for did', + 'IncompatibleDidDoc', + ) + } + const prevCid = await cidForCbor(prev) + if (plcOp.prev?.toString() !== prevCid.toString()) { + throw new InvalidRequestError( + 'invalid prev on plc operation', + 'IncompatibleDidDoc', + ) + } + const normalizedPrev = plc.normalizeOp(prev) + await plc.assureValidSig(normalizedPrev.rotationKeys, plcOp) + const doc = plc.formatDidDoc({ did, ...plcOp }) + const data = ensureAtpDocument(doc) + if (handle !== data.handle) { + throw new InvalidRequestError( + 'invalid handle on plc operation', + 'IncompatibleDidDoc', + ) + } else if (data.pds !== ctx.cfg.service.publicUrl) { + throw new InvalidRequestError( + 'invalid service on plc operation', + 'IncompatibleDidDoc', + ) + } else if (data.signingKey !== signingKey) { + throw new InvalidRequestError( + 'invalid signing key on plc operation', + 'IncompatibleDidDoc', + ) + } + return plcOp +} diff --git a/packages/pds/src/api/proxy.ts b/packages/pds/src/api/proxy.ts new file mode 100644 index 00000000000..554898dbbaf --- /dev/null +++ b/packages/pds/src/api/proxy.ts @@ -0,0 +1,33 @@ +import { Headers } from '@atproto/xrpc' +import { IncomingMessage } from 'node:http' + +export const resultPassthru = (result: { headers: Headers; data: T }) => { + // @TODO pass through any headers that we always want to forward along + return { + encoding: 'application/json' as const, + body: result.data, + } +} + +// Output designed to passed as second arg to AtpAgent methods. +// The encoding field here is a quirk of the AtpAgent. +export function authPassthru( + req: IncomingMessage, + withEncoding?: false, +): { headers: { authorization: string }; encoding: undefined } | undefined + +export function authPassthru( + req: IncomingMessage, + withEncoding: true, +): + | { headers: { authorization: string }; encoding: 'application/json' } + | undefined + +export function authPassthru(req: IncomingMessage, withEncoding?: boolean) { + if (req.headers.authorization) { + return { + headers: { authorization: req.headers.authorization }, + encoding: withEncoding ? 'application/json' : undefined, + } + } +} diff --git a/packages/pds/src/auth-verifier.ts b/packages/pds/src/auth-verifier.ts index dba1550ba0b..7a09135a72c 100644 --- a/packages/pds/src/auth-verifier.ts +++ b/packages/pds/src/auth-verifier.ts @@ -1,13 +1,17 @@ +import { KeyObject, createPublicKey, createSecretKey } from 'node:crypto' import { AuthRequiredError, + ForbiddenError, InvalidRequestError, verifyJwt as verifyServiceJwt, } from '@atproto/xrpc-server' import { IdResolver } from '@atproto/identity' import * as ui8 from 'uint8arrays' import express from 'express' -import * as jwt from 'jsonwebtoken' -import Database from './db' +import * as jose from 'jose' +import KeyEncoder from 'key-encoder' +import { AccountManager } from './account-manager' +import { softDeleted } from './db' type ReqCtx = { req: express.Request @@ -52,6 +56,7 @@ type AccessOutput = { type: 'access' did: string scope: AuthScope + audience: string | undefined } artifacts: string } @@ -61,6 +66,7 @@ type RefreshOutput = { type: 'refresh' did: string scope: AuthScope + audience: string | undefined tokenId: string } artifacts: string @@ -70,39 +76,44 @@ type ValidatedBearer = { did: string scope: AuthScope token: string - payload: jwt.JwtPayload + payload: jose.JWTPayload + audience: string | undefined } export type AuthVerifierOpts = { - jwtSecret: string + jwtKey: KeyObject adminPass: string moderatorPass: string triagePass: string - adminServiceDid: string + dids: { + pds: string + entryway?: string + admin: string + } } export class AuthVerifier { - private _secret: string + private _jwtKey: KeyObject private _adminPass: string private _moderatorPass: string private _triagePass: string - public adminServiceDid: string + public dids: AuthVerifierOpts['dids'] constructor( - public db: Database, + public accountManager: AccountManager, public idResolver: IdResolver, opts: AuthVerifierOpts, ) { - this._secret = opts.jwtSecret + this._jwtKey = opts.jwtKey this._adminPass = opts.adminPass this._moderatorPass = opts.moderatorPass this._triagePass = opts.triagePass - this.adminServiceDid = opts.adminServiceDid + this.dids = opts.dids } // verifiers (arrow fns to preserve scope) - access = (ctx: ReqCtx): AccessOutput => { + access = (ctx: ReqCtx): Promise => { return this.validateAccessToken(ctx.req, [ AuthScope.Access, AuthScope.AppPass, @@ -110,18 +121,16 @@ export class AuthVerifier { } accessCheckTakedown = async (ctx: ReqCtx): Promise => { - const result = this.validateAccessToken(ctx.req, [ + const result = await this.validateAccessToken(ctx.req, [ AuthScope.Access, AuthScope.AppPass, ]) - const found = await this.db.db - .selectFrom('user_account') - .innerJoin('repo_root', 'repo_root.did', 'user_account.did') - .where('user_account.did', '=', result.credentials.did) - .where('repo_root.takedownRef', 'is', null) - .select('user_account.did') - .executeTakeFirst() + const found = await this.accountManager.getAccount(result.credentials.did) if (!found) { + // will be turned into ExpiredToken for the client if proxied by entryway + throw new ForbiddenError('Account not found', 'AccountNotFound') + } + if (softDeleted(found)) { throw new AuthRequiredError( 'Account has been taken down', 'AccountTakedown', @@ -130,14 +139,16 @@ export class AuthVerifier { return result } - accessNotAppPassword = (ctx: ReqCtx): AccessOutput => { + accessNotAppPassword = (ctx: ReqCtx): Promise => { return this.validateAccessToken(ctx.req, [AuthScope.Access]) } - refresh = (ctx: ReqCtx): RefreshOutput => { - const { did, scope, token, payload } = this.validateBearerToken(ctx.req, [ - AuthScope.Refresh, - ]) + refresh = async (ctx: ReqCtx): Promise => { + const { did, scope, token, audience, payload } = + await this.validateBearerToken(ctx.req, [AuthScope.Refresh], { + // when using entryway, proxying refresh credentials + audience: this.dids.entryway ? this.dids.entryway : this.dids.pds, + }) if (!payload.jti) { throw new AuthRequiredError( 'Unexpected missing refresh token id', @@ -149,6 +160,7 @@ export class AuthVerifier { type: 'refresh', did, scope, + audience, tokenId: payload.jti, }, artifacts: token, @@ -168,7 +180,7 @@ export class AuthVerifier { } } - accessOrRole = (ctx: ReqCtx): AccessOutput | RoleOutput => { + accessOrRole = async (ctx: ReqCtx): Promise => { if (isBearerToken(ctx.req)) { return this.access(ctx) } else { @@ -176,22 +188,22 @@ export class AuthVerifier { } } - optionalAccessOrRole = ( + optionalAccessOrRole = async ( ctx: ReqCtx, - ): AccessOutput | RoleOutput | NullOutput => { + ): Promise => { if (isBearerToken(ctx.req)) { - return this.access(ctx) + return await this.access(ctx) } else { const creds = this.parseRoleCreds(ctx.req) - if (creds.status === RoleStatus.Missing) { - return { credentials: null } - } else if (creds.admin) { + if (creds.status === RoleStatus.Valid) { return { credentials: { ...creds, type: 'role', }, } + } else if (creds.status === RoleStatus.Missing) { + return { credentials: null } } else { throw new AuthRequiredError() } @@ -206,14 +218,14 @@ export class AuthVerifier { const payload = await verifyServiceJwt( jwtStr, null, - async (did: string) => { - if (did !== this.adminServiceDid) { + async (did, forceRefresh) => { + if (did !== this.dids.admin) { throw new AuthRequiredError( 'Untrusted issuer for admin actions', 'UntrustedIss', ) } - return this.idResolver.did.resolveAtprotoKey(did) + return this.idResolver.did.resolveAtprotoKey(did, forceRefresh) }, ) return { @@ -235,39 +247,53 @@ export class AuthVerifier { } } - validateBearerToken( + async validateBearerToken( req: express.Request, scopes: AuthScope[], - options?: jwt.VerifyOptions, - ): ValidatedBearer { + verifyOptions?: jose.JWTVerifyOptions, + ): Promise { const token = bearerTokenFromReq(req) if (!token) { throw new AuthRequiredError(undefined, 'AuthMissing') } - const payload = verifyJwt({ secret: this._secret, token, options }) - const sub = payload.sub + const payload = await verifyJwt({ key: this._jwtKey, token, verifyOptions }) + const { sub, aud, scope } = payload if (typeof sub !== 'string' || !sub.startsWith('did:')) { throw new InvalidRequestError('Malformed token', 'InvalidToken') } - if (scopes.length > 0 && !scopes.includes(payload.scope)) { + if ( + aud !== undefined && + (typeof aud !== 'string' || !aud.startsWith('did:')) + ) { + throw new InvalidRequestError('Malformed token', 'InvalidToken') + } + if (!isAuthScope(scope) || (scopes.length > 0 && !scopes.includes(scope))) { throw new InvalidRequestError('Bad token scope', 'InvalidToken') } - return { did: sub, - scope: payload.scope, + scope, + audience: aud, token, payload, } } - validateAccessToken(req: express.Request, scopes: AuthScope[]): AccessOutput { - const { did, scope, token } = this.validateBearerToken(req, scopes) + async validateAccessToken( + req: express.Request, + scopes: AuthScope[], + ): Promise { + const { did, scope, token, audience } = await this.validateBearerToken( + req, + scopes, + { audience: this.dids.pds }, + ) return { credentials: { type: 'access', did, scope, + audience, }, artifacts: token, } @@ -322,20 +348,17 @@ const bearerTokenFromReq = (req: express.Request) => { return header.slice(BEARER.length) } -const verifyJwt = (params: { - secret: string +const verifyJwt = async (params: { + key: KeyObject token: string - options?: jwt.VerifyOptions -}): jwt.JwtPayload => { - const { secret, token, options } = params + verifyOptions?: jose.JWTVerifyOptions +}): Promise => { + const { key, token, verifyOptions } = params try { - const payload = jwt.verify(token, secret, options) - if (typeof payload === 'string' || 'signature' in payload) { - throw new InvalidRequestError('Malformed token', 'InvalidToken') - } - return payload + const result = await jose.jwtVerify(token, key, verifyOptions) + return result.payload } catch (err) { - if (err instanceof jwt.TokenExpiredError) { + if (err?.['code'] === 'ERR_JWT_EXPIRED') { throw new InvalidRequestError('Token has expired', 'ExpiredToken') } throw new InvalidRequestError('Token could not be verified', 'InvalidToken') @@ -372,3 +395,19 @@ export const ensureValidAdminAud = ( ) } } + +const authScopes = new Set(Object.values(AuthScope)) +const isAuthScope = (val: unknown): val is AuthScope => { + return authScopes.has(val as any) +} + +export const createSecretKeyObject = (secret: string): KeyObject => { + return createSecretKey(Buffer.from(secret)) +} + +export const createPublicKeyObject = (publicKeyHex: string): KeyObject => { + const key = keyEncoder.encodePublic(publicKeyHex, 'raw', 'pem') + return createPublicKey({ format: 'pem', key }) +} + +const keyEncoder = new KeyEncoder('secp256k1') diff --git a/packages/pds/src/background.ts b/packages/pds/src/background.ts index 65d8cbd473b..d58f3bf5122 100644 --- a/packages/pds/src/background.ts +++ b/packages/pds/src/background.ts @@ -1,5 +1,4 @@ import PQueue from 'p-queue' -import Database from './db' import { dbLogger } from './logger' // A simple queue for in-process, out-of-band/backgrounded work @@ -7,14 +6,14 @@ import { dbLogger } from './logger' export class BackgroundQueue { queue = new PQueue({ concurrency: 5 }) destroyed = false - constructor(public db: Database) {} + constructor() {} add(task: Task) { if (this.destroyed) { return } this.queue - .add(() => task(this.db)) + .add(() => task()) .catch((err) => { dbLogger.error(err, 'background queue task failed') }) @@ -32,4 +31,4 @@ export class BackgroundQueue { } } -type Task = (db: Database) => Promise +type Task = () => Promise diff --git a/packages/pds/src/basic-routes.ts b/packages/pds/src/basic-routes.ts index aa094bea635..b64d1293c2f 100644 --- a/packages/pds/src/basic-routes.ts +++ b/packages/pds/src/basic-routes.ts @@ -22,10 +22,11 @@ export const createRouter = (ctx: AppContext): express.Router => { router.get('/xrpc/_health', async function (req, res) { const { version } = ctx.cfg.service try { - await sql`select 1`.execute(ctx.db.db) + await sql`select 1`.execute(ctx.accountManager.db.db) } catch (err) { req.log.error(err, 'failed health check') - return res.status(503).send({ version, error: 'Service Unavailable' }) + res.status(503).send({ version, error: 'Service Unavailable' }) + return } res.send({ version }) }) diff --git a/packages/pds/src/config/config.ts b/packages/pds/src/config/config.ts index 75227099835..c1676c25908 100644 --- a/packages/pds/src/config/config.ts +++ b/packages/pds/src/config/config.ts @@ -1,5 +1,5 @@ -import os from 'node:os' import path from 'node:path' +import assert from 'node:assert' import { DAY, HOUR, SECOND } from '@atproto/common' import { ServerEnvironment } from './env' @@ -24,29 +24,23 @@ export const envToCfg = (env: ServerEnvironment): ServerConfig => { termsOfServiceUrl: env.termsOfServiceUrl, } - let dbCfg: ServerConfig['db'] - if (env.dbSqliteLocation && env.dbPostgresUrl) { - throw new Error('Cannot set both sqlite & postgres db env vars') + const dbLoc = (name: string) => { + return env.dataDirectory ? path.join(env.dataDirectory, name) : name } - if (env.dbSqliteLocation) { - dbCfg = { - dialect: 'sqlite', - location: env.dbSqliteLocation, - } - } else if (env.dbPostgresUrl) { - dbCfg = { - dialect: 'pg', - url: env.dbPostgresUrl, - migrationUrl: env.dbPostgresMigrationUrl ?? env.dbPostgresUrl, - schema: env.dbPostgresSchema, - pool: { - idleTimeoutMs: env.dbPostgresPoolIdleTimeoutMs ?? 10000, - maxUses: env.dbPostgresPoolMaxUses ?? Infinity, - size: env.dbPostgresPoolSize ?? 10, - }, - } - } else { - throw new Error('Must configure either sqlite or postgres db') + + const disableWalAutoCheckpoint = env.disableWalAutoCheckpoint ?? false + + const dbCfg: ServerConfig['db'] = { + accountDbLoc: env.accountDbLocation ?? dbLoc('account.sqlite'), + sequencerDbLoc: env.sequencerDbLocation ?? dbLoc('sequencer.sqlite'), + didCacheDbLoc: env.didCacheDbLocation ?? dbLoc('did_cache.sqlite'), + disableWalAutoCheckpoint, + } + + const actorStoreCfg: ServerConfig['actorStore'] = { + directory: env.actorStoreDirectory ?? dbLoc('actors'), + cacheSize: env.actorStoreCacheSize ?? 100, + disableWalAutoCheckpoint, } let blobstoreCfg: ServerConfig['blobstore'] @@ -76,8 +70,7 @@ export const envToCfg = (env: ServerEnvironment): ServerConfig => { blobstoreCfg = { provider: 'disk', location: env.blobstoreDiskLocation, - tempLocation: - env.blobstoreDiskTmpLocation ?? path.join(os.tmpdir(), 'pds/blobs'), + tempLocation: env.blobstoreDiskTmpLocation, } } else { throw new Error('Must configure either S3 or disk blobstore') @@ -111,6 +104,22 @@ export const envToCfg = (env: ServerEnvironment): ServerConfig => { enableDidDocWithSession: !!env.enableDidDocWithSession, } + let entrywayCfg: ServerConfig['entryway'] = null + if (env.entrywayUrl) { + assert( + env.entrywayJwtVerifyKeyK256PublicKeyHex && + env.entrywayPlcRotationKey && + env.entrywayDid, + 'if entryway url is configured, must include all required entryway configuration', + ) + entrywayCfg = { + url: env.entrywayUrl, + did: env.entrywayDid, + jwtPublicKeyHex: env.entrywayJwtVerifyKeyK256PublicKeyHex, + plcRotationKey: env.entrywayPlcRotationKey, + } + } + // default to being required if left undefined const invitesCfg: ServerConfig['invites'] = env.inviteRequired === false @@ -156,8 +165,6 @@ export const envToCfg = (env: ServerEnvironment): ServerConfig => { const subscriptionCfg: ServerConfig['subscription'] = { maxBuffer: env.maxSubscriptionBuffer ?? 500, repoBackfillLimitMs: env.repoBackfillLimitMs ?? DAY, - sequencerLeaderEnabled: env.sequencerLeaderEnabled ?? true, - sequencerLeaderLockId: env.sequencerLeaderLockId ?? 1100, } if (!env.bskyAppViewUrl) { @@ -195,8 +202,10 @@ export const envToCfg = (env: ServerEnvironment): ServerConfig => { return { service: serviceCfg, db: dbCfg, + actorStore: actorStoreCfg, blobstore: blobstoreCfg, identity: identityCfg, + entryway: entrywayCfg, invites: invitesCfg, email: emailCfg, moderationEmail: moderationEmailCfg, @@ -210,9 +219,11 @@ export const envToCfg = (env: ServerEnvironment): ServerConfig => { export type ServerConfig = { service: ServiceConfig - db: SqliteConfig | PostgresConfig + db: DatabaseConfig + actorStore: ActorStoreConfig blobstore: S3BlobstoreConfig | DiskBlobstoreConfig identity: IdentityConfig + entryway: EntrywayConfig | null invites: InvitesConfig email: EmailConfig | null moderationEmail: EmailConfig | null @@ -233,23 +244,17 @@ export type ServiceConfig = { termsOfServiceUrl?: string } -export type SqliteConfig = { - dialect: 'sqlite' - location: string +export type DatabaseConfig = { + accountDbLoc: string + sequencerDbLoc: string + didCacheDbLoc: string + disableWalAutoCheckpoint: boolean } -export type PostgresPoolConfig = { - size: number - maxUses: number - idleTimeoutMs: number -} - -export type PostgresConfig = { - dialect: 'pg' - url: string - migrationUrl: string - pool: PostgresPoolConfig - schema?: string +export type ActorStoreConfig = { + directory: string + cacheSize: number + disableWalAutoCheckpoint: boolean } export type S3BlobstoreConfig = { @@ -267,7 +272,7 @@ export type S3BlobstoreConfig = { export type DiskBlobstoreConfig = { provider: 'disk' location: string - tempLocation: string + tempLocation?: string } export type IdentityConfig = { @@ -281,6 +286,13 @@ export type IdentityConfig = { enableDidDocWithSession: boolean } +export type EntrywayConfig = { + url: string + did: string + jwtPublicKeyHex: string + plcRotationKey: string +} + export type InvitesConfig = | { required: true @@ -299,8 +311,6 @@ export type EmailConfig = { export type SubscriptionConfig = { maxBuffer: number repoBackfillLimitMs: number - sequencerLeaderEnabled: boolean - sequencerLeaderLockId: number } export type RedisScratchConfig = { diff --git a/packages/pds/src/config/env.ts b/packages/pds/src/config/env.ts index a7f1c2636af..bfcf4e36956 100644 --- a/packages/pds/src/config/env.ts +++ b/packages/pds/src/config/env.ts @@ -10,16 +10,16 @@ export const readEnv = (): ServerEnvironment => { privacyPolicyUrl: envStr('PDS_PRIVACY_POLICY_URL'), termsOfServiceUrl: envStr('PDS_TERMS_OF_SERVICE_URL'), - // db: one required - // sqlite - dbSqliteLocation: envStr('PDS_DB_SQLITE_LOCATION'), - // postgres - dbPostgresUrl: envStr('PDS_DB_POSTGRES_URL'), - dbPostgresMigrationUrl: envStr('PDS_DB_POSTGRES_MIGRATION_URL'), - dbPostgresSchema: envStr('PDS_DB_POSTGRES_SCHEMA'), - dbPostgresPoolSize: envInt('PDS_DB_POSTGRES_POOL_SIZE'), - dbPostgresPoolMaxUses: envInt('PDS_DB_POSTGRES_POOL_MAX_USES'), - dbPostgresPoolIdleTimeoutMs: envInt('PDS_DB_POSTGRES_POOL_IDLE_TIMEOUT_MS'), + // database + dataDirectory: envStr('PDS_DATA_DIRECTORY'), + disableWalAutoCheckpoint: envBool('PDS_SQLITE_DISABLE_WAL_AUTO_CHECKPOINT'), + accountDbLocation: envStr('PDS_ACCOUNT_DB_LOCATION'), + sequencerDbLocation: envStr('PDS_SEQUENCER_DB_LOCATION'), + didCacheDbLocation: envStr('PDS_DID_CACHE_DB_LOCATION'), + + // actor store + actorStoreDirectory: envStr('PDS_ACTOR_STORE_DIRECTORY'), + actorStoreCacheSize: envInt('PDS_ACTOR_STORE_CACHE_SIZE'), // blobstore: one required // s3 @@ -43,6 +43,14 @@ export const readEnv = (): ServerEnvironment => { handleBackupNameservers: envList('PDS_HANDLE_BACKUP_NAMESERVERS'), enableDidDocWithSession: envBool('PDS_ENABLE_DID_DOC_WITH_SESSION'), + // entryway + entrywayUrl: envStr('PDS_ENTRYWAY_URL'), + entrywayDid: envStr('PDS_ENTRYWAY_DID'), + entrywayJwtVerifyKeyK256PublicKeyHex: envStr( + 'PDS_ENTRYWAY_JWT_VERIFY_KEY_K256_PUBLIC_KEY_HEX', + ), + entrywayPlcRotationKey: envStr('PDS_ENTRYWAY_PLC_ROTATION_KEY'), + // invites inviteRequired: envBool('PDS_INVITE_REQUIRED'), inviteInterval: envInt('PDS_INVITE_INTERVAL'), @@ -57,8 +65,6 @@ export const readEnv = (): ServerEnvironment => { // subscription maxSubscriptionBuffer: envInt('PDS_MAX_SUBSCRIPTION_BUFFER'), repoBackfillLimitMs: envInt('PDS_REPO_BACKFILL_LIMIT_MS'), - sequencerLeaderEnabled: envBool('PDS_SEQUENCER_LEADER_ENABLED'), - sequencerLeaderLockId: envInt('PDS_SEQUENCER_LEADER_LOCK_ID'), // appview bskyAppViewUrl: envStr('PDS_BSKY_APP_VIEW_URL'), @@ -84,13 +90,6 @@ export const readEnv = (): ServerEnvironment => { moderatorPassword: envStr('PDS_MODERATOR_PASSWORD'), triagePassword: envStr('PDS_TRIAGE_PASSWORD'), - // keys: only one of each required - // kms - repoSigningKeyKmsKeyId: envStr('PDS_REPO_SIGNING_KEY_KMS_KEY_ID'), - // memory - repoSigningKeyK256PrivateKeyHex: envStr( - 'PDS_REPO_SIGNING_KEY_K256_PRIVATE_KEY_HEX', - ), // kms plcRotationKeyKmsKeyId: envStr('PDS_PLC_ROTATION_KEY_KMS_KEY_ID'), // memory @@ -109,14 +108,16 @@ export type ServerEnvironment = { privacyPolicyUrl?: string termsOfServiceUrl?: string - // db: one required - dbSqliteLocation?: string - dbPostgresUrl?: string - dbPostgresMigrationUrl?: string - dbPostgresSchema?: string - dbPostgresPoolSize?: number - dbPostgresPoolMaxUses?: number - dbPostgresPoolIdleTimeoutMs?: number + // database + dataDirectory?: string + disableWalAutoCheckpoint?: boolean + accountDbLocation?: string + sequencerDbLocation?: string + didCacheDbLocation?: string + + // actor store + actorStoreDirectory?: string + actorStoreCacheSize?: number // blobstore: one required blobstoreS3Bucket?: string @@ -140,6 +141,12 @@ export type ServerEnvironment = { handleBackupNameservers?: string[] enableDidDocWithSession?: boolean + // entryway + entrywayUrl?: string + entrywayDid?: string + entrywayJwtVerifyKeyK256PublicKeyHex?: string + entrywayPlcRotationKey?: string + // invites inviteRequired?: boolean inviteInterval?: number @@ -154,8 +161,6 @@ export type ServerEnvironment = { // subscription maxSubscriptionBuffer?: number repoBackfillLimitMs?: number - sequencerLeaderEnabled?: boolean - sequencerLeaderLockId?: number // appview bskyAppViewUrl?: string @@ -182,8 +187,6 @@ export type ServerEnvironment = { triagePassword?: string // keys - repoSigningKeyKmsKeyId?: string - repoSigningKeyK256PrivateKeyHex?: string plcRotationKeyKmsKeyId?: string plcRotationKeyK256PrivateKeyHex?: string } diff --git a/packages/pds/src/config/secrets.ts b/packages/pds/src/config/secrets.ts index f0f876f1ccc..8e18cd830f7 100644 --- a/packages/pds/src/config/secrets.ts +++ b/packages/pds/src/config/secrets.ts @@ -1,23 +1,6 @@ import { ServerEnvironment } from './env' export const envToSecrets = (env: ServerEnvironment): ServerSecrets => { - let repoSigningKey: ServerSecrets['repoSigningKey'] - if (env.repoSigningKeyKmsKeyId && env.repoSigningKeyK256PrivateKeyHex) { - throw new Error('Cannot set both kms & memory keys for repo signing key') - } else if (env.repoSigningKeyKmsKeyId) { - repoSigningKey = { - provider: 'kms', - keyId: env.repoSigningKeyKmsKeyId, - } - } else if (env.repoSigningKeyK256PrivateKeyHex) { - repoSigningKey = { - provider: 'memory', - privateKeyHex: env.repoSigningKeyK256PrivateKeyHex, - } - } else { - throw new Error('Must configure repo signing key') - } - let plcRotationKey: ServerSecrets['plcRotationKey'] if (env.plcRotationKeyKmsKeyId && env.plcRotationKeyK256PrivateKeyHex) { throw new Error('Cannot set both kms & memory keys for plc rotation key') @@ -49,7 +32,6 @@ export const envToSecrets = (env: ServerEnvironment): ServerSecrets => { moderatorPassword: env.moderatorPassword ?? env.adminPassword, triagePassword: env.triagePassword ?? env.moderatorPassword ?? env.adminPassword, - repoSigningKey, plcRotationKey, } } @@ -59,7 +41,6 @@ export type ServerSecrets = { adminPassword: string moderatorPassword: string triagePassword: string - repoSigningKey: SigningKeyKms | SigningKeyMemory plcRotationKey: SigningKeyKms | SigningKeyMemory } diff --git a/packages/pds/src/context.ts b/packages/pds/src/context.ts index 56eead1dace..8f47992c008 100644 --- a/packages/pds/src/context.ts +++ b/packages/pds/src/context.ts @@ -6,82 +6,89 @@ import { IdResolver } from '@atproto/identity' import { AtpAgent } from '@atproto/api' import { KmsKeypair, S3BlobStore } from '@atproto/aws' import { createServiceAuthHeaders } from '@atproto/xrpc-server' -import { Database } from './db' import { ServerConfig, ServerSecrets } from './config' -import { AuthVerifier } from './auth-verifier' +import { + AuthVerifier, + createPublicKeyObject, + createSecretKeyObject, +} from './auth-verifier' import { ServerMailer } from './mailer' import { ModerationMailer } from './mailer/moderation' import { BlobStore } from '@atproto/repo' -import { Services, createServices } from './services' -import { Sequencer, SequencerLeader } from './sequencer' +import { AccountManager } from './account-manager' +import { Sequencer } from './sequencer' import { BackgroundQueue } from './background' -import DidSqlCache from './did-cache' +import { DidSqliteCache } from './did-cache' import { Crawlers } from './crawlers' -import { DiskBlobStore } from './storage' +import { DiskBlobStore } from './disk-blobstore' import { getRedisClient } from './redis' -import { RuntimeFlags } from './runtime-flags' +import { ActorStore, ActorStoreReader } from './actor-store' +import { LocalViewer } from './read-after-write/viewer' export type AppContextOptions = { - db: Database - blobstore: BlobStore + actorStore: ActorStore + blobstore: (did: string) => BlobStore + localViewer: ( + actorStore: ActorStoreReader, + actorKey: crypto.Keypair, + ) => LocalViewer mailer: ServerMailer moderationMailer: ModerationMailer - didCache: DidSqlCache + didCache: DidSqliteCache idResolver: IdResolver plcClient: plc.Client - services: Services + accountManager: AccountManager sequencer: Sequencer - sequencerLeader?: SequencerLeader backgroundQueue: BackgroundQueue - runtimeFlags: RuntimeFlags redisScratch?: Redis crawlers: Crawlers appViewAgent: AtpAgent + entrywayAgent?: AtpAgent authVerifier: AuthVerifier - repoSigningKey: crypto.Keypair plcRotationKey: crypto.Keypair cfg: ServerConfig } export class AppContext { - public db: Database - public blobstore: BlobStore + public actorStore: ActorStore + public blobstore: (did: string) => BlobStore + public localViewer: ( + actorStore: ActorStoreReader, + actorKey: crypto.Keypair, + ) => LocalViewer public mailer: ServerMailer public moderationMailer: ModerationMailer - public didCache: DidSqlCache + public didCache: DidSqliteCache public idResolver: IdResolver public plcClient: plc.Client - public services: Services + public accountManager: AccountManager public sequencer: Sequencer - public sequencerLeader?: SequencerLeader public backgroundQueue: BackgroundQueue - public runtimeFlags: RuntimeFlags public redisScratch?: Redis public crawlers: Crawlers public appViewAgent: AtpAgent + public entrywayAgent: AtpAgent | undefined public authVerifier: AuthVerifier - public repoSigningKey: crypto.Keypair public plcRotationKey: crypto.Keypair public cfg: ServerConfig constructor(opts: AppContextOptions) { - this.db = opts.db + this.actorStore = opts.actorStore this.blobstore = opts.blobstore + this.localViewer = opts.localViewer this.mailer = opts.mailer this.moderationMailer = opts.moderationMailer this.didCache = opts.didCache this.idResolver = opts.idResolver this.plcClient = opts.plcClient - this.services = opts.services + this.accountManager = opts.accountManager this.sequencer = opts.sequencer - this.sequencerLeader = opts.sequencerLeader this.backgroundQueue = opts.backgroundQueue - this.runtimeFlags = opts.runtimeFlags this.redisScratch = opts.redisScratch this.crawlers = opts.crawlers this.appViewAgent = opts.appViewAgent + this.entrywayAgent = opts.entrywayAgent this.authVerifier = opts.authVerifier - this.repoSigningKey = opts.repoSigningKey this.plcRotationKey = opts.plcRotationKey this.cfg = opts.cfg } @@ -91,26 +98,16 @@ export class AppContext { secrets: ServerSecrets, overrides?: Partial, ): Promise { - const db = - cfg.db.dialect === 'sqlite' - ? Database.sqlite(cfg.db.location) - : Database.postgres({ - url: cfg.db.url, - schema: cfg.db.schema, - poolSize: cfg.db.pool.size, - poolMaxUses: cfg.db.pool.maxUses, - poolIdleTimeoutMs: cfg.db.pool.idleTimeoutMs, - }) const blobstore = cfg.blobstore.provider === 's3' - ? new S3BlobStore({ + ? S3BlobStore.creator({ bucket: cfg.blobstore.bucket, region: cfg.blobstore.region, endpoint: cfg.blobstore.endpoint, forcePathStyle: cfg.blobstore.forcePathStyle, credentials: cfg.blobstore.credentials, }) - : await DiskBlobStore.create( + : DiskBlobStore.creator( cfg.blobstore.location, cfg.blobstore.tempLocation, ) @@ -129,11 +126,14 @@ export class AppContext { const moderationMailer = new ModerationMailer(modMailTransport, cfg) - const didCache = new DidSqlCache( - db, + const didCache = new DidSqliteCache( + cfg.db.didCacheDbLoc, cfg.identity.cacheStaleTTL, cfg.identity.cacheMaxTTL, + cfg.db.disableWalAutoCheckpoint, ) + await didCache.migrateOrThrow() + const idResolver = new IdResolver({ plcUrl: cfg.identity.plcUrl, didCache, @@ -142,38 +142,53 @@ export class AppContext { }) const plcClient = new plc.Client(cfg.identity.plcUrl) - const sequencer = new Sequencer(db) - const sequencerLeader = cfg.subscription.sequencerLeaderEnabled - ? new SequencerLeader(db, cfg.subscription.sequencerLeaderLockId) - : undefined - - const backgroundQueue = new BackgroundQueue(db) - const runtimeFlags = new RuntimeFlags(db) + const backgroundQueue = new BackgroundQueue() + const crawlers = new Crawlers( + cfg.service.hostname, + cfg.crawlers, + backgroundQueue, + ) + const sequencer = new Sequencer( + cfg.db.sequencerDbLoc, + crawlers, + undefined, + cfg.db.disableWalAutoCheckpoint, + ) const redisScratch = cfg.redis ? getRedisClient(cfg.redis.address, cfg.redis.password) : undefined - const crawlers = new Crawlers(cfg.service.hostname, cfg.crawlers) - const appViewAgent = new AtpAgent({ service: cfg.bskyAppView.url }) - const authVerifier = new AuthVerifier(db, idResolver, { - jwtSecret: secrets.jwtSecret, + const entrywayAgent = cfg.entryway + ? new AtpAgent({ service: cfg.entryway.url }) + : undefined + + const jwtSecretKey = createSecretKeyObject(secrets.jwtSecret) + const accountManager = new AccountManager( + cfg.db.accountDbLoc, + jwtSecretKey, + cfg.service.did, + cfg.db.disableWalAutoCheckpoint, + ) + await accountManager.migrateOrThrow() + + const jwtKey = cfg.entryway + ? createPublicKeyObject(cfg.entryway.jwtPublicKeyHex) + : jwtSecretKey + + const authVerifier = new AuthVerifier(accountManager, idResolver, { + jwtKey, // @TODO support multiple keys? adminPass: secrets.adminPassword, moderatorPass: secrets.moderatorPassword, triagePass: secrets.triagePassword, - adminServiceDid: cfg.bskyAppView.did, + dids: { + pds: cfg.service.did, + entryway: cfg.entryway?.did, + admin: cfg.bskyAppView.did, + }, }) - const repoSigningKey = - secrets.repoSigningKey.provider === 'kms' - ? await KmsKeypair.load({ - keyId: secrets.repoSigningKey.keyId, - }) - : await crypto.Secp256k1Keypair.import( - secrets.repoSigningKey.privateKeyHex, - ) - const plcRotationKey = secrets.plcRotationKey.provider === 'kms' ? await KmsKeypair.load({ @@ -183,36 +198,36 @@ export class AppContext { secrets.plcRotationKey.privateKeyHex, ) - const services = createServices({ - repoSigningKey, + const actorStore = new ActorStore(cfg.actorStore, { blobstore, + backgroundQueue, + }) + + const localViewer = LocalViewer.creator({ + accountManager, appViewAgent, pdsHostname: cfg.service.hostname, - jwtSecret: secrets.jwtSecret, - appViewDid: cfg.bskyAppView.did, - appViewCdnUrlPattern: cfg.bskyAppView.cdnUrlPattern, - backgroundQueue, - crawlers, + appviewDid: cfg.bskyAppView.did, + appviewCdnUrlPattern: cfg.bskyAppView.cdnUrlPattern, }) return new AppContext({ - db, + actorStore, blobstore, + localViewer, mailer, moderationMailer, didCache, idResolver, plcClient, - services, + accountManager, sequencer, - sequencerLeader, backgroundQueue, - runtimeFlags, redisScratch, crawlers, appViewAgent, + entrywayAgent, authVerifier, - repoSigningKey, plcRotationKey, cfg, ...(overrides ?? {}), @@ -224,10 +239,11 @@ export class AppContext { if (!aud) { throw new Error('Could not find bsky appview did') } + const keypair = await this.actorStore.keypair(did) return createServiceAuthHeaders({ iss: did, aud, - keypair: this.repoSigningKey, + keypair, }) } } diff --git a/packages/pds/src/crawlers.ts b/packages/pds/src/crawlers.ts index 5fd855217c1..5ca20b10621 100644 --- a/packages/pds/src/crawlers.ts +++ b/packages/pds/src/crawlers.ts @@ -1,6 +1,7 @@ import { AtpAgent } from '@atproto/api' import { crawlerLogger as log } from './logger' import { MINUTE } from '@atproto/common' +import { BackgroundQueue } from './background' const NOTIFY_THRESHOLD = 20 * MINUTE @@ -8,7 +9,11 @@ export class Crawlers { public agents: AtpAgent[] public lastNotified = 0 - constructor(public hostname: string, public crawlers: string[]) { + constructor( + public hostname: string, + public crawlers: string[], + public backgroundQueue: BackgroundQueue, + ) { this.agents = crawlers.map((service) => new AtpAgent({ service })) } @@ -18,20 +23,22 @@ export class Crawlers { return } - await Promise.all( - this.agents.map(async (agent) => { - try { - await agent.api.com.atproto.sync.requestCrawl({ - hostname: this.hostname, - }) - } catch (err) { - log.warn( - { err, cralwer: agent.service.toString() }, - 'failed to request crawl', - ) - } - }), - ) - this.lastNotified = now + this.backgroundQueue.add(async () => { + await Promise.all( + this.agents.map(async (agent) => { + try { + await agent.api.com.atproto.sync.requestCrawl({ + hostname: this.hostname, + }) + } catch (err) { + log.warn( + { err, cralwer: agent.service.toString() }, + 'failed to request crawl', + ) + } + }), + ) + this.lastNotified = now + }) } } diff --git a/packages/pds/src/db/database-schema.ts b/packages/pds/src/db/database-schema.ts deleted file mode 100644 index 26159418206..00000000000 --- a/packages/pds/src/db/database-schema.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { Kysely } from 'kysely' -import * as userAccount from './tables/user-account' -import * as userPref from './tables/user-pref' -import * as didHandle from './tables/did-handle' -import * as repoRoot from './tables/repo-root' -import * as didCache from './tables/did-cache' -import * as refreshToken from './tables/refresh-token' -import * as appPassword from './tables/app-password' -import * as record from './tables/record' -import * as backlink from './tables/backlink' -import * as ipldBlock from './tables/ipld-block' -import * as inviteCode from './tables/invite-code' -import * as blob from './tables/blob' -import * as repoBlob from './tables/repo-blob' -import * as emailToken from './tables/email-token' -import * as moderation from './tables/moderation' -import * as repoSeq from './tables/repo-seq' -import * as appMigration from './tables/app-migration' -import * as runtimeFlag from './tables/runtime-flag' - -export type DatabaseSchemaType = appMigration.PartialDB & - runtimeFlag.PartialDB & - userAccount.PartialDB & - userPref.PartialDB & - didHandle.PartialDB & - refreshToken.PartialDB & - appPassword.PartialDB & - repoRoot.PartialDB & - didCache.PartialDB & - record.PartialDB & - backlink.PartialDB & - ipldBlock.PartialDB & - inviteCode.PartialDB & - blob.PartialDB & - repoBlob.PartialDB & - emailToken.PartialDB & - moderation.PartialDB & - repoSeq.PartialDB - -export type DatabaseSchema = Kysely - -export default DatabaseSchema diff --git a/packages/pds/src/db/db.ts b/packages/pds/src/db/db.ts new file mode 100644 index 00000000000..1850e07f2d5 --- /dev/null +++ b/packages/pds/src/db/db.ts @@ -0,0 +1,137 @@ +import assert from 'assert' +import { + sql, + Kysely, + SqliteDialect, + KyselyPlugin, + PluginTransformQueryArgs, + PluginTransformResultArgs, + RootOperationNode, + QueryResult, + UnknownRow, +} from 'kysely' +import SqliteDB from 'better-sqlite3' +import { dbLogger } from '../logger' +import { retrySqlite } from './util' + +const DEFAULT_PRAGMAS = { + // strict: 'ON', // @TODO strictness should live on table defs instead +} + +export class Database { + destroyed = false + commitHooks: CommitHook[] = [] + + constructor(public db: Kysely) {} + + static sqlite( + location: string, + opts?: { pragmas?: Record }, + ): Database { + const sqliteDb = new SqliteDB(location, { + timeout: 0, // handled by application + }) + const pragmas = { + ...DEFAULT_PRAGMAS, + ...(opts?.pragmas ?? {}), + } + for (const pragma of Object.keys(pragmas)) { + sqliteDb.pragma(`${pragma} = ${pragmas[pragma]}`) + } + const db = new Kysely({ + dialect: new SqliteDialect({ + database: sqliteDb, + }), + }) + return new Database(db) + } + + async ensureWal() { + await sql`PRAGMA journal_mode = WAL`.execute(this.db) + } + + async transactionNoRetry( + fn: (db: Database) => Promise, + ): Promise { + this.assertNotTransaction() + const leakyTxPlugin = new LeakyTxPlugin() + const { hooks, txRes } = await this.db + .withPlugin(leakyTxPlugin) + .transaction() + .execute(async (txn) => { + const dbTxn = new Database(txn) + const txRes = await fn(dbTxn) + .catch(async (err) => { + leakyTxPlugin.endTx() + // ensure that all in-flight queries are flushed & the connection is open + await dbTxn.db.getExecutor().provideConnection(async () => {}) + throw err + }) + .finally(() => leakyTxPlugin.endTx()) + const hooks = dbTxn.commitHooks + return { hooks, txRes } + }) + hooks.map((hook) => hook()) + return txRes + } + + async transaction(fn: (db: Database) => Promise): Promise { + return retrySqlite(() => this.transactionNoRetry(fn)) + } + + async executeWithRetry(query: { execute: () => Promise }) { + if (this.isTransaction) { + // transaction() ensures retry on entire transaction, no need to retry individual statements. + return query.execute() + } + return retrySqlite(() => query.execute()) + } + + onCommit(fn: () => void) { + this.assertTransaction() + this.commitHooks.push(fn) + } + + get isTransaction() { + return this.db.isTransaction + } + + assertTransaction() { + assert(this.isTransaction, 'Transaction required') + } + + assertNotTransaction() { + assert(!this.isTransaction, 'Cannot be in a transaction') + } + + close(): void { + if (this.destroyed) return + this.db + .destroy() + .then(() => (this.destroyed = true)) + .catch((err) => dbLogger.error({ err }, 'error closing db')) + } +} + +type CommitHook = () => void + +class LeakyTxPlugin implements KyselyPlugin { + private txOver: boolean + + endTx() { + this.txOver = true + } + + transformQuery(args: PluginTransformQueryArgs): RootOperationNode { + if (this.txOver) { + throw new Error('tx already failed') + } + return args.node + } + + async transformResult( + args: PluginTransformResultArgs, + ): Promise> { + return args.result + } +} diff --git a/packages/pds/src/db/index.ts b/packages/pds/src/db/index.ts index f6a1cc831a3..2ccf49b90c7 100644 --- a/packages/pds/src/db/index.ts +++ b/packages/pds/src/db/index.ts @@ -1,398 +1,3 @@ -import assert from 'assert' -import { - Kysely, - SqliteDialect, - PostgresDialect, - Migrator, - sql, - KyselyPlugin, - PluginTransformQueryArgs, - PluginTransformResultArgs, - RootOperationNode, - QueryResult, - UnknownRow, -} from 'kysely' -import SqliteDB from 'better-sqlite3' -import { Pool as PgPool, Client as PgClient, types as pgTypes } from 'pg' -import EventEmitter from 'events' -import TypedEmitter from 'typed-emitter' -import { wait } from '@atproto/common' -import DatabaseSchema, { DatabaseSchemaType } from './database-schema' -import { dummyDialect } from './util' -import * as migrations from './migrations' -import { CtxMigrationProvider } from './migrations/provider' -import { dbLogger as log } from '../logger' -import { randomIntFromSeed } from '@atproto/crypto' - -export class Database { - txEvt = new EventEmitter() as TxnEmitter - txChannelEvts: ChannelEvt[] = [] - txLockNonce: string | undefined - channels: Channels - migrator: Migrator - destroyed = false - - private channelClient: PgClient | null = null - - constructor( - public db: DatabaseSchema, - public cfg: DialectConfig, - channels?: Channels, - ) { - this.migrator = new Migrator({ - db, - migrationTableSchema: cfg.dialect === 'pg' ? cfg.schema : undefined, - provider: new CtxMigrationProvider(migrations, cfg.dialect), - }) - this.channels = channels || { - new_repo_event: new EventEmitter() as ChannelEmitter, - outgoing_repo_seq: new EventEmitter() as ChannelEmitter, - } - this.txLockNonce = cfg.dialect === 'pg' ? cfg.txLockNonce : undefined - } - - static sqlite(location: string): Database { - const db = new Kysely({ - dialect: new SqliteDialect({ - database: new SqliteDB(location), - }), - }) - return new Database(db, { dialect: 'sqlite' }) - } - - static postgres(opts: PgOptions): Database { - const { schema, url, txLockNonce } = opts - const pool = - opts.pool ?? - new PgPool({ - connectionString: url, - max: opts.poolSize, - maxUses: opts.poolMaxUses, - idleTimeoutMillis: opts.poolIdleTimeoutMs, - }) - - // Select count(*) and other pg bigints as js integer - pgTypes.setTypeParser(pgTypes.builtins.INT8, (n) => parseInt(n, 10)) - - // Setup schema usage, primarily for test parallelism (each test suite runs in its own pg schema) - if (schema && !/^[a-z_]+$/i.test(schema)) { - throw new Error(`Postgres schema must only contain [A-Za-z_]: ${schema}`) - } - - pool.on('error', onPoolError) - pool.on('connect', (client) => { - client.on('error', onClientError) - // Used for trigram indexes, e.g. on actor search - client.query('SET pg_trgm.word_similarity_threshold TO .4;') - if (schema) { - // Shared objects such as extensions will go in the public schema - client.query(`SET search_path TO "${schema}",public;`) - } - }) - - const db = new Kysely({ - dialect: new PostgresDialect({ pool }), - }) - - return new Database(db, { - dialect: 'pg', - pool, - schema, - url, - txLockNonce, - }) - } - - static memory(): Database { - return Database.sqlite(':memory:') - } - - async startListeningToChannels() { - if (this.cfg.dialect !== 'pg') return - if (this.channelClient) return - this.channelClient = new PgClient(this.cfg.url) - await this.channelClient.connect() - await this.channelClient.query(`LISTEN ${this.getSchemaChannel()}`) - this.channelClient.on('notification', (msg) => { - const channel = this.channels[msg.payload ?? ''] - if (channel) { - channel.emit('message') - } - }) - this.channelClient.on('error', (err) => { - log.error({ err }, 'postgres listener errored, reconnecting') - this.channelClient?.removeAllListeners() - this.startListeningToChannels() - }) - } - - async notify(evt: ChannelEvt) { - // if in a sqlite tx, we buffer the notification until the tx successfully commits - if (this.isTransaction && this.dialect === 'sqlite') { - // no duplicate notifies in a tx per Postgres semantics - if (!this.txChannelEvts.includes(evt)) { - this.txChannelEvts.push(evt) - } - } else { - await this.sendChannelEvt(evt) - } - } - - onCommit(fn: () => void) { - this.assertTransaction() - this.txEvt.once('commit', fn) - } - - private getSchemaChannel() { - const CHANNEL_NAME = 'pds_db_channel' - if (this.cfg.dialect === 'pg' && this.cfg.schema) { - return this.cfg.schema + '_' + CHANNEL_NAME - } else { - return CHANNEL_NAME - } - } - - private async sendChannelEvt(evt: ChannelEvt) { - if (this.cfg.dialect === 'pg') { - const { ref } = this.db.dynamic - if (evt !== 'new_repo_event' && evt !== 'outgoing_repo_seq') { - throw new Error(`Invalid evt: ${evt}`) - } - await sql`NOTIFY ${ref(this.getSchemaChannel())}, ${sql.literal( - evt, - )}`.execute(this.db) - } else { - const emitter = this.channels[evt] - if (emitter) { - emitter.emit('message') - } - } - } - - async transaction(fn: (db: Database) => Promise): Promise { - let txEvts: ChannelEvt[] = [] - const leakyTxPlugin = new LeakyTxPlugin() - const { dbTxn, txRes } = await this.db - .withPlugin(leakyTxPlugin) - .transaction() - .execute(async (txn) => { - const dbTxn = new Database(txn, this.cfg, this.channels) - const txRes = await fn(dbTxn) - .catch(async (err) => { - leakyTxPlugin.endTx() - // ensure that all in-flight queries are flushed & the connection is open - await dbTxn.db.getExecutor().provideConnection(async () => {}) - throw err - }) - .finally(() => leakyTxPlugin.endTx()) - txEvts = dbTxn.txChannelEvts - return { txRes, dbTxn } - }) - dbTxn?.txEvt.emit('commit') - txEvts.forEach((evt) => this.sendChannelEvt(evt)) - return txRes - } - - async takeTxAdvisoryLock(name: string): Promise { - this.assertTransaction() - return this.txAdvisoryLock(name) - } - - async checkTxAdvisoryLock(name: string): Promise { - this.assertNotTransaction() - return this.txAdvisoryLock(name) - } - - private async txAdvisoryLock(name: string): Promise { - assert(this.dialect === 'pg', 'Postgres required') - // any lock id < 10k is reserved for session locks - const id = await randomIntFromSeed(name, Number.MAX_SAFE_INTEGER, 10000) - const res = (await sql`SELECT pg_try_advisory_xact_lock(${sql.literal( - id, - )}) as acquired`.execute(this.db)) as TxLockRes - return res.rows[0]?.acquired === true - } - - get schema(): string | undefined { - return this.cfg.dialect === 'pg' ? this.cfg.schema : undefined - } - - get dialect(): Dialect { - return this.cfg.dialect - } - - get isTransaction() { - return this.db.isTransaction - } - - assertTransaction() { - assert(this.isTransaction, 'Transaction required') - } - - assertNotTransaction() { - assert(!this.isTransaction, 'Cannot be in a transaction') - } - - async close(): Promise { - if (this.destroyed) return - if (this.channelClient) { - await this.channelClient.end() - } - await this.db.destroy() - this.destroyed = true - } - - async migrateToOrThrow(migration: string) { - if (this.schema) { - await this.db.schema.createSchema(this.schema).ifNotExists().execute() - } - const { error, results } = await this.migrator.migrateTo(migration) - if (error) { - throw error - } - if (!results) { - throw new Error('An unknown failure occurred while migrating') - } - return results - } - - async migrateToLatestOrThrow() { - if (this.schema) { - await this.db.schema.createSchema(this.schema).ifNotExists().execute() - } - const { error, results } = await this.migrator.migrateToLatest() - if (error) { - throw error - } - if (!results) { - throw new Error('An unknown failure occurred while migrating') - } - return results - } - - async maintainMaterializedViews(opts: { - views: string[] - intervalSec: number - signal: AbortSignal - }) { - assert( - this.dialect === 'pg', - 'Can only maintain materialized views on postgres', - ) - const { views, intervalSec, signal } = opts - while (!signal.aborted) { - // super basic synchronization by agreeing when the intervals land relative to unix timestamp - const now = Date.now() - const intervalMs = 1000 * intervalSec - const nextIteration = Math.ceil(now / intervalMs) - const nextInMs = nextIteration * intervalMs - now - await wait(nextInMs) - if (signal.aborted) break - await Promise.all( - views.map(async (view) => { - try { - await this.refreshMaterializedView(view) - log.info( - { view, time: new Date().toISOString() }, - 'materialized view refreshed', - ) - } catch (err) { - log.error( - { view, err, time: new Date().toISOString() }, - 'materialized view refresh failed', - ) - } - }), - ) - } - } - - async refreshMaterializedView(view: string) { - assert( - this.dialect === 'pg', - 'Can only maintain materialized views on postgres', - ) - const { ref } = this.db.dynamic - await sql`refresh materialized view concurrently ${ref(view)}`.execute( - this.db, - ) - } -} - -export default Database - -export type Dialect = 'pg' | 'sqlite' - -export type DialectConfig = PgConfig | SqliteConfig - -export type PgConfig = { - dialect: 'pg' - pool: PgPool - url: string - schema?: string - txLockNonce?: string -} - -export type SqliteConfig = { - dialect: 'sqlite' -} - -// Can use with typeof to get types for partial queries -export const dbType = new Kysely({ dialect: dummyDialect }) - -type PgOptions = { - url: string - pool?: PgPool - schema?: string - poolSize?: number - poolMaxUses?: number - poolIdleTimeoutMs?: number - txLockNonce?: string -} - -type ChannelEvents = { - message: () => void -} - -type ChannelEmitter = TypedEmitter - -type Channels = { - outgoing_repo_seq: ChannelEmitter - new_repo_event: ChannelEmitter -} - -type ChannelEvt = keyof Channels - -type TxnEmitter = TypedEmitter - -type TxnEvents = { - commit: () => void -} - -class LeakyTxPlugin implements KyselyPlugin { - private txOver: boolean - - endTx() { - this.txOver = true - } - - transformQuery(args: PluginTransformQueryArgs): RootOperationNode { - if (this.txOver) { - throw new Error('tx already failed') - } - return args.node - } - - async transformResult( - args: PluginTransformResultArgs, - ): Promise> { - return args.result - } -} - -type TxLockRes = { - rows: { acquired: true | false }[] -} - -const onPoolError = (err: Error) => log.error({ err }, 'db pool error') -const onClientError = (err: Error) => log.error({ err }, 'db client error') +export * from './db' +export * from './migrator' +export * from './util' diff --git a/packages/pds/src/db/leader.ts b/packages/pds/src/db/leader.ts deleted file mode 100644 index 67c19351fa1..00000000000 --- a/packages/pds/src/db/leader.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { PoolClient } from 'pg' -import Database from '.' - -export class Leader { - session: Session | null = null - static inProcessLocks = new Map>() // Only for sqlite in-process locking mechanism - - constructor(public id: number, public db: Database) {} - - async run( - task: (ctx: { signal: AbortSignal }) => Promise, - ): Promise> { - const session = await this.lock() - if (!session) return { ran: false } - try { - const result = await task({ signal: session.abortController.signal }) - return { ran: true, result } - } finally { - this.release() - } - } - - destroy(err?: Error) { - this.session?.abortController.abort(err) - } - - private async lock(): Promise { - if (this.session) { - return null - } - - if (this.db.cfg.dialect === 'sqlite') { - const locksForId = Leader.inProcessLocks.get(this.id) ?? new WeakSet() - if (locksForId.has(this.db)) { - return null - } else { - Leader.inProcessLocks.set(this.id, locksForId.add(this.db)) - this.session = { abortController: new AbortController() } - return this.session - } - } - - // Postgres implementation uses advisory locking, automatically released by ending connection. - - const client = await this.db.cfg.pool.connect() - try { - const lock = await client.query( - 'SELECT pg_try_advisory_lock($1) as acquired', - [this.id], - ) - if (!lock.rows[0].acquired) { - client.release() - return null - } - } catch (err) { - client.release(true) - throw err - } - - const abortController = new AbortController() - client.once('error', (err) => abortController.abort(err)) - this.session = { abortController, client } - return this.session - } - - private release() { - if (this.db.cfg.dialect === 'sqlite') { - Leader.inProcessLocks.get(this.id)?.delete(this.db) - } else { - // The flag ensures the connection is destroyed on release, not reused. - // This is required, as that is how the pg advisory lock is released. - this.session?.client?.release(true) - } - this.session = null - } -} - -type Session = { abortController: AbortController; client?: PoolClient } - -type RunResult = { ran: false } | { ran: true; result: T } - -// Mini system for coordinated app-level migrations. - -const APP_MIGRATION_LOCK_ID = 1100 - -export async function appMigration( - db: Database, - id: string, - runMigration: (tx: Database) => Promise, -) { - // Ensure migration is tracked in a table - await ensureMigrationTracked(db, id) - - // If the migration has already completed, succeed/fail with it (fast path, no locks) - const status = await checkMigrationStatus(db, id) - if (status === MigrationStatus.Succeeded) { - return - } else if (status === MigrationStatus.Failed) { - throw new Error('Migration previously failed') - } - - // Take a lock for potentially running an app migration - const disposeLock = await acquireMigrationLock(db) - try { - // If the migration has already completed, succeed/fail with it - const status = await checkMigrationStatus(db, id) - if (status === MigrationStatus.Succeeded) { - return - } else if (status === MigrationStatus.Failed) { - throw new Error('Migration previously failed') - } - // Run the migration and update migration state - try { - await db.transaction(runMigration) - await completeMigration(db, id, 1) - } catch (err) { - await completeMigration(db, id, 0) - throw err - } - } finally { - // Ensure lock is released - disposeLock() - } -} - -async function checkMigrationStatus(db: Database, id: string) { - const migration = await db.db - .selectFrom('app_migration') - .selectAll() - .where('id', '=', id) - .executeTakeFirstOrThrow() - if (!migration.completedAt) { - return MigrationStatus.Running - } - return migration.success ? MigrationStatus.Succeeded : MigrationStatus.Failed -} - -async function acquireMigrationLock(db: Database) { - if (db.cfg.dialect !== 'pg') { - throw new Error('App migrations are pg-only') - } - const client = await db.cfg.pool.connect() - const dispose = () => client.release(true) - try { - // Blocks until lock is acquired - await client.query('SELECT pg_advisory_lock($1)', [APP_MIGRATION_LOCK_ID]) - } catch (err) { - dispose() - throw err - } - return dispose -} - -async function completeMigration(db: Database, id: string, success: 0 | 1) { - await db.db - .updateTable('app_migration') - .where('id', '=', id) - .where('completedAt', 'is', null) - .set({ success, completedAt: new Date().toISOString() }) - .executeTakeFirst() -} - -async function ensureMigrationTracked(db: Database, id: string) { - await db.db - .insertInto('app_migration') - .values({ id, success: 0 }) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .execute() -} - -enum MigrationStatus { - Succeeded, - Failed, - Running, -} diff --git a/packages/pds/src/db/migrations/20230613T164932261Z-init.ts b/packages/pds/src/db/migrations/20230613T164932261Z-init.ts deleted file mode 100644 index a150ff06cf9..00000000000 --- a/packages/pds/src/db/migrations/20230613T164932261Z-init.ts +++ /dev/null @@ -1,375 +0,0 @@ -import { Kysely, sql } from 'kysely' -import { Dialect } from '..' - -// @TODO make takedownId a varchar w/o fkey? - -export async function up(db: Kysely, dialect: Dialect): Promise { - const binaryDatatype = dialect === 'sqlite' ? 'blob' : sql`bytea` - - await db.schema - .createTable('app_migration') - .addColumn('id', 'varchar', (col) => col.primaryKey()) - .addColumn('success', 'int2', (col) => col.notNull().defaultTo(0)) - .addColumn('completedAt', 'varchar', (col) => col) - .execute() - - await db.schema - .createTable('app_password') - .addColumn('did', 'varchar', (col) => col.notNull()) - .addColumn('name', 'varchar', (col) => col.notNull()) - .addColumn('passwordScrypt', 'varchar', (col) => col.notNull()) - .addColumn('createdAt', 'varchar', (col) => col.notNull()) - .addPrimaryKeyConstraint('app_password_pkey', ['did', 'name']) - .execute() - - await db.schema - .createTable('backlink') - .addColumn('uri', 'varchar', (col) => col.notNull()) - .addColumn('path', 'varchar', (col) => col.notNull()) - .addColumn('linkToUri', 'varchar') - .addColumn('linkToDid', 'varchar') - .addPrimaryKeyConstraint('backlinks_pkey', ['uri', 'path']) - .addCheckConstraint( - 'backlink_link_to_chk', - // Exactly one of linkToUri or linkToDid should be set - sql`("linkToUri" is null and "linkToDid" is not null) or ("linkToUri" is not null and "linkToDid" is null)`, - ) - .execute() - await db.schema - .createIndex('backlink_path_to_uri_idx') - .on('backlink') - .columns(['path', 'linkToUri']) - .execute() - await db.schema - .createIndex('backlink_path_to_did_idx') - .on('backlink') - .columns(['path', 'linkToDid']) - .execute() - - await db.schema - .createTable('blob') - .addColumn('creator', 'varchar', (col) => col.notNull()) - .addColumn('cid', 'varchar', (col) => col.notNull()) - .addColumn('mimeType', 'varchar', (col) => col.notNull()) - .addColumn('size', 'integer', (col) => col.notNull()) - .addColumn('tempKey', 'varchar') - .addColumn('width', 'integer') - .addColumn('height', 'integer') - .addColumn('createdAt', 'varchar', (col) => col.notNull()) - .addPrimaryKeyConstraint('blob_pkey', ['creator', 'cid']) - .execute() - - await db.schema - .createTable('delete_account_token') - .addColumn('did', 'varchar', (col) => col.primaryKey()) - .addColumn('token', 'varchar', (col) => col.notNull()) - .addColumn('requestedAt', 'varchar', (col) => col.notNull()) - .execute() - - await db.schema - .createTable('did_cache') - .addColumn('did', 'varchar', (col) => col.primaryKey()) - .addColumn('doc', 'text', (col) => col.notNull()) - .addColumn('updatedAt', 'bigint', (col) => col.notNull()) - .execute() - - await db.schema - .createTable('did_handle') - .addColumn('did', 'varchar', (col) => col.primaryKey()) - .addColumn('handle', 'varchar', (col) => col.unique()) - .execute() - await db.schema - .createIndex(`did_handle_handle_lower_idx`) - .unique() - .on('did_handle') - .expression(sql`lower("handle")`) - .execute() - - await db.schema - .createTable('invite_code') - .addColumn('code', 'varchar', (col) => col.primaryKey()) - .addColumn('availableUses', 'integer', (col) => col.notNull()) - .addColumn('disabled', 'int2', (col) => col.defaultTo(0)) - .addColumn('forUser', 'varchar', (col) => col.notNull()) - .addColumn('createdBy', 'varchar', (col) => col.notNull()) - .addColumn('createdAt', 'varchar', (col) => col.notNull()) - .execute() - - await db.schema - .createTable('invite_code_use') - .addColumn('code', 'varchar', (col) => col.notNull()) - .addColumn('usedBy', 'varchar', (col) => col.notNull()) - .addColumn('usedAt', 'varchar', (col) => col.notNull()) - .addPrimaryKeyConstraint(`invite_code_use_pkey`, ['code', 'usedBy']) - .execute() - - await db.schema - .createTable('ipld_block') - .addColumn('creator', 'varchar', (col) => col.notNull()) - .addColumn('cid', 'varchar', (col) => col.notNull()) - .addColumn('size', 'integer', (col) => col.notNull()) - .addColumn('content', binaryDatatype, (col) => col.notNull()) - .addPrimaryKeyConstraint('ipld_block_pkey', ['creator', 'cid']) - .execute() - - const moderationActionBuilder = - dialect === 'pg' - ? db.schema - .createTable('moderation_action') - .addColumn('id', 'serial', (col) => col.primaryKey()) - : db.schema - .createTable('moderation_action') - .addColumn('id', 'integer', (col) => col.autoIncrement().primaryKey()) - await moderationActionBuilder - .addColumn('action', 'varchar', (col) => col.notNull()) - .addColumn('subjectType', 'varchar', (col) => col.notNull()) - .addColumn('subjectDid', 'varchar', (col) => col.notNull()) - .addColumn('subjectUri', 'varchar') - .addColumn('subjectCid', 'varchar') - .addColumn('reason', 'text', (col) => col.notNull()) - .addColumn('createdAt', 'varchar', (col) => col.notNull()) - .addColumn('createdBy', 'varchar', (col) => col.notNull()) - .addColumn('reversedAt', 'varchar') - .addColumn('reversedBy', 'varchar') - .addColumn('reversedReason', 'text') - .addColumn('createLabelVals', 'varchar') - .addColumn('negateLabelVals', 'varchar') - .execute() - - await db.schema - .createTable('moderation_action_subject_blob') - .addColumn('actionId', 'integer', (col) => - col.notNull().references('moderation_action.id'), - ) - .addColumn('cid', 'varchar', (col) => col.notNull()) - .addColumn('recordUri', 'varchar', (col) => col.notNull()) - .addPrimaryKeyConstraint('moderation_action_subject_blob_pkey', [ - 'actionId', - 'cid', - 'recordUri', - ]) - .execute() - - const moderationReportBuilder = - dialect === 'pg' - ? db.schema - .createTable('moderation_report') - .addColumn('id', 'serial', (col) => col.primaryKey()) - : db.schema - .createTable('moderation_report') - .addColumn('id', 'integer', (col) => col.autoIncrement().primaryKey()) - await moderationReportBuilder - .addColumn('subjectType', 'varchar', (col) => col.notNull()) - .addColumn('subjectDid', 'varchar', (col) => col.notNull()) - .addColumn('subjectUri', 'varchar') - .addColumn('subjectCid', 'varchar') - .addColumn('reasonType', 'varchar', (col) => col.notNull()) - .addColumn('reason', 'text') - .addColumn('reportedByDid', 'varchar', (col) => col.notNull()) - .addColumn('createdAt', 'varchar', (col) => col.notNull()) - .execute() - - await db.schema - .createTable('moderation_report_resolution') - .addColumn('reportId', 'integer', (col) => - col.notNull().references('moderation_report.id'), - ) - .addColumn('actionId', 'integer', (col) => - col.notNull().references('moderation_action.id'), - ) - .addColumn('createdBy', 'varchar', (col) => col.notNull()) - .addColumn('createdAt', 'varchar', (col) => col.notNull()) - .addPrimaryKeyConstraint('moderation_report_resolution_pkey', [ - 'reportId', - 'actionId', - ]) - .execute() - await db.schema - .createIndex('moderation_report_resolution_action_id_idx') - .on('moderation_report_resolution') - .column('actionId') - .execute() - - await db.schema - .createTable('record') - .addColumn('uri', 'varchar', (col) => col.primaryKey()) - .addColumn('cid', 'varchar', (col) => col.notNull()) - .addColumn('did', 'varchar', (col) => col.notNull()) - .addColumn('collection', 'varchar', (col) => col.notNull()) - .addColumn('rkey', 'varchar', (col) => col.notNull()) - .addColumn('indexedAt', 'varchar', (col) => col.notNull()) - .addColumn('takedownId', 'varchar') - .execute() - await db.schema - .createIndex('record_did_cid_idx') - .on('record') - .columns(['did', 'cid']) - .execute() - await db.schema - .createIndex('record_did_collection_idx') - .on('record') - .columns(['did', 'collection']) - .execute() - - await db.schema - .createTable('refresh_token') - .addColumn('id', 'varchar', (col) => col.primaryKey()) - .addColumn('did', 'varchar', (col) => col.notNull()) - .addColumn('expiresAt', 'varchar', (col) => col.notNull()) - .addColumn('nextId', 'varchar') - .addColumn('appPasswordName', 'varchar') - .execute() - await db.schema // Aids in refresh token cleanup - .createIndex('refresh_token_did_idx') - .on('refresh_token') - .column('did') - .execute() - - await db.schema - .createTable('repo_blob') - .addColumn('cid', 'varchar', (col) => col.notNull()) - .addColumn('recordUri', 'varchar', (col) => col.notNull()) - .addColumn('commit', 'varchar', (col) => col.notNull()) - .addColumn('did', 'varchar', (col) => col.notNull()) - .addColumn('takedownId', 'varchar') - .addPrimaryKeyConstraint(`repo_blob_pkey`, ['cid', 'recordUri']) - .execute() - await db.schema // supports rebase - .createIndex('repo_blob_did_idx') - .on('repo_blob') - .column('did') - .execute() - - await db.schema - .createTable('repo_commit_block') - .addColumn('creator', 'varchar', (col) => col.notNull()) - .addColumn('commit', 'varchar', (col) => col.notNull()) - .addColumn('block', 'varchar', (col) => col.notNull()) - .addPrimaryKeyConstraint('repo_commit_block_pkey', [ - 'creator', - 'commit', - 'block', - ]) - .execute() - - await db.schema - .createTable('repo_commit_history') - .addColumn('creator', 'varchar', (col) => col.notNull()) - .addColumn('commit', 'varchar', (col) => col.notNull()) - .addColumn('prev', 'varchar') - .addPrimaryKeyConstraint('repo_commit_history_pkey', ['creator', 'commit']) - .execute() - - await db.schema - .createTable('repo_root') - .addColumn('did', 'varchar', (col) => col.primaryKey()) - .addColumn('root', 'varchar', (col) => col.notNull()) - .addColumn('indexedAt', 'varchar', (col) => col.notNull()) - .addColumn('takedownId', 'varchar') - .execute() - - // @TODO renamed indexes for consistency - const repoSeqBuilder = - dialect === 'pg' - ? db.schema - .createTable('repo_seq') - .addColumn('id', 'bigserial', (col) => col.primaryKey()) - .addColumn('seq', 'bigint', (col) => col.unique()) - : db.schema - .createTable('repo_seq') - .addColumn('id', 'integer', (col) => col.autoIncrement().primaryKey()) - .addColumn('seq', 'integer', (col) => col.unique()) - await repoSeqBuilder - .addColumn('did', 'varchar', (col) => col.notNull()) - .addColumn('eventType', 'varchar', (col) => col.notNull()) - .addColumn('event', binaryDatatype, (col) => col.notNull()) - .addColumn('invalidated', 'int2', (col) => col.notNull().defaultTo(0)) - .addColumn('sequencedAt', 'varchar', (col) => col.notNull()) - .execute() - // for filtering seqs based on did - await db.schema - .createIndex('repo_seq_did_idx') - .on('repo_seq') - .column('did') - .execute() - // for filtering seqs based on event type - await db.schema - .createIndex('repo_seq_event_type_idx') - .on('repo_seq') - .column('eventType') - .execute() - // for entering into the seq stream at a particular time - await db.schema - .createIndex('repo_seq_sequenced_at_index') - .on('repo_seq') - .column('sequencedAt') - .execute() - - await db.schema - .createTable('user_account') - .addColumn('did', 'varchar', (col) => col.primaryKey()) - .addColumn('email', 'varchar', (col) => col.notNull()) - .addColumn('passwordScrypt', 'varchar', (col) => col.notNull()) - .addColumn('createdAt', 'varchar', (col) => col.notNull()) - .addColumn('passwordResetToken', 'varchar') - .addColumn('passwordResetGrantedAt', 'varchar') - .addColumn('invitesDisabled', 'int2', (col) => col.notNull().defaultTo(0)) - .execute() - await db.schema - .createIndex(`user_account_email_lower_idx`) - .unique() - .on('user_account') - .expression(sql`lower("email")`) - .execute() - await db.schema - .createIndex('user_account_password_reset_token_idx') - .unique() - .on('user_account') - .column('passwordResetToken') - .execute() - - const userPrefBuilder = - dialect === 'pg' - ? db.schema - .createTable('user_pref') - .addColumn('id', 'bigserial', (col) => col.primaryKey()) - : db.schema - .createTable('user_pref') - .addColumn('id', 'integer', (col) => col.autoIncrement().primaryKey()) - await userPrefBuilder - .addColumn('did', 'varchar', (col) => col.notNull()) - .addColumn('name', 'varchar', (col) => col.notNull()) - .addColumn('valueJson', 'text', (col) => col.notNull()) - .execute() - await db.schema - .createIndex('user_pref_did_idx') - .on('user_pref') - .column('did') - .execute() -} - -export async function down(db: Kysely): Promise { - await db.schema.dropTable('user_pref').execute() - await db.schema.dropTable('user_account').execute() - await db.schema.dropTable('repo_seq').execute() - await db.schema.dropTable('repo_root').execute() - await db.schema.dropTable('repo_commit_history').execute() - await db.schema.dropTable('repo_commit_block').execute() - await db.schema.dropTable('repo_blob').execute() - await db.schema.dropTable('refresh_token').execute() - await db.schema.dropTable('record').execute() - await db.schema.dropTable('moderation_report_resolution').execute() - await db.schema.dropTable('moderation_report').execute() - await db.schema.dropTable('moderation_action_subject_blob').execute() - await db.schema.dropTable('moderation_action').execute() - await db.schema.dropTable('ipld_block').execute() - await db.schema.dropTable('invite_code_use').execute() - await db.schema.dropTable('invite_code').execute() - await db.schema.dropTable('did_handle').execute() - await db.schema.dropTable('did_cache').execute() - await db.schema.dropTable('delete_account_token').execute() - await db.schema.dropTable('blob').execute() - await db.schema.dropTable('backlink').execute() - await db.schema.dropTable('app_password').execute() - await db.schema.dropTable('app_migration').execute() -} diff --git a/packages/pds/src/db/migrations/20230914T014727199Z-repo-v3.ts b/packages/pds/src/db/migrations/20230914T014727199Z-repo-v3.ts deleted file mode 100644 index cd9569bc33c..00000000000 --- a/packages/pds/src/db/migrations/20230914T014727199Z-repo-v3.ts +++ /dev/null @@ -1,165 +0,0 @@ -import { Kysely, sql } from 'kysely' -import { Dialect } from '..' - -export async function up(db: Kysely, dialect: Dialect): Promise { - // sequencer leader sequence - if (dialect !== 'sqlite') { - const res = await db - .selectFrom('repo_seq') - .select('seq') - .where('seq', 'is not', null) - .orderBy('seq', 'desc') - .limit(1) - .executeTakeFirst() - const startAt = res?.seq ? res.seq + 50000 : 1 - await sql`CREATE SEQUENCE repo_seq_sequence START ${sql.literal( - startAt, - )};`.execute(db) - } - - // user account cursor idx - await db.schema - .createIndex('user_account_cursor_idx') - .on('user_account') - .columns(['createdAt', 'did']) - .execute() - - // invite note - await db.schema - .alterTable('user_account') - .addColumn('inviteNote', 'varchar') - .execute() - - // listing user invites - await db.schema - .createIndex('invite_code_for_user_idx') - .on('invite_code') - .column('forUser') - .execute() - - // mod action duration - await db.schema - .alterTable('moderation_action') - .addColumn('durationInHours', 'integer') - .execute() - await db.schema - .alterTable('moderation_action') - .addColumn('expiresAt', 'varchar') - .execute() - - // runtime flag - await db.schema - .createTable('runtime_flag') - .addColumn('name', 'varchar', (col) => col.primaryKey()) - .addColumn('value', 'varchar', (col) => col.notNull()) - .execute() - - // blob tempkey idx - await db.schema - .createIndex('blob_tempkey_idx') - .on('blob') - .column('tempKey') - .execute() - - // repo v3 - await db.schema.alterTable('repo_root').addColumn('rev', 'varchar').execute() - await db.schema.alterTable('record').addColumn('repoRev', 'varchar').execute() - await db.schema - .alterTable('ipld_block') - .addColumn('repoRev', 'varchar') - .execute() - await db.schema - .alterTable('repo_blob') - .addColumn('repoRev', 'varchar') - .execute() - await db.schema.alterTable('repo_blob').dropColumn('commit').execute() - - await db.schema - .createIndex('record_repo_rev_idx') - .on('record') - .columns(['did', 'repoRev']) - .execute() - - await db.schema - .createIndex('ipld_block_repo_rev_idx') - .on('ipld_block') - .columns(['creator', 'repoRev', 'cid']) - .execute() - - await db.schema - .createIndex('repo_blob_repo_rev_idx') - .on('repo_blob') - .columns(['did', 'repoRev']) - .execute() - - await db.schema.dropTable('repo_commit_history').execute() - await db.schema.dropTable('repo_commit_block').execute() -} - -export async function down( - db: Kysely, - dialect: Dialect, -): Promise { - // repo v3 - await db.schema - .createTable('repo_commit_block') - .addColumn('commit', 'varchar', (col) => col.notNull()) - .addColumn('block', 'varchar', (col) => col.notNull()) - .addColumn('creator', 'varchar', (col) => col.notNull()) - .addPrimaryKeyConstraint('repo_commit_block_pkey', [ - 'creator', - 'commit', - 'block', - ]) - .execute() - await db.schema - .createTable('repo_commit_history') - .addColumn('commit', 'varchar', (col) => col.notNull()) - .addColumn('prev', 'varchar') - .addColumn('creator', 'varchar', (col) => col.notNull()) - .addPrimaryKeyConstraint('repo_commit_history_pkey', ['creator', 'commit']) - .execute() - - await db.schema.dropIndex('record_repo_rev_idx').execute() - await db.schema.dropIndex('ipld_block_repo_rev_idx').execute() - await db.schema.dropIndex('repo_blob_repo_rev_idx').execute() - - await db.schema.alterTable('repo_root').dropColumn('rev').execute() - await db.schema.alterTable('record').dropColumn('repoRev').execute() - await db.schema.alterTable('ipld_block').dropColumn('repoRev').execute() - await db.schema.alterTable('repo_blob').dropColumn('repoRev').execute() - await db.schema - .alterTable('repo_blob') - .addColumn('commit', 'varchar') - .execute() - - // blob tempkey idx - await db.schema.dropIndex('blob_tempkey_idx').execute() - - // runtime flag - await db.schema.dropTable('runtime_flag').execute() - - // mod action duration - await db.schema - .alterTable('moderation_action') - .dropColumn('durationInHours') - .execute() - await db.schema - .alterTable('moderation_action') - .dropColumn('expiresAt') - .execute() - - // listing user invites - await db.schema.dropIndex('invite_code_for_user_idx').execute() - - // invite note - await db.schema.alterTable('user_account').dropColumn('inviteNote').execute() - - // user account cursor idx - await db.schema.dropIndex('user_account_cursor_idx').execute() - - // sequencer leader sequence - if (dialect !== 'sqlite') { - await sql`DROP SEQUENCE repo_seq_sequence;`.execute(db) - } -} diff --git a/packages/pds/src/db/migrations/20230926T195532354Z-email-tokens.ts b/packages/pds/src/db/migrations/20230926T195532354Z-email-tokens.ts deleted file mode 100644 index 44cefc18899..00000000000 --- a/packages/pds/src/db/migrations/20230926T195532354Z-email-tokens.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { Kysely } from 'kysely' -import { Dialect } from '..' - -export async function up(db: Kysely, dialect: Dialect): Promise { - const timestamp = dialect === 'sqlite' ? 'datetime' : 'timestamptz' - await db.schema - .createTable('email_token') - .addColumn('purpose', 'varchar', (col) => col.notNull()) - .addColumn('did', 'varchar', (col) => col.notNull()) - .addColumn('token', 'varchar', (col) => col.notNull()) - .addColumn('requestedAt', timestamp, (col) => col.notNull()) - .addPrimaryKeyConstraint('email_token_pkey', ['purpose', 'did']) - .addUniqueConstraint('email_token_purpose_token_unique', [ - 'purpose', - 'token', - ]) - .execute() - - await db.schema - .alterTable('user_account') - .addColumn('emailConfirmedAt', 'varchar') - .execute() - - await db.schema.dropIndex('user_account_password_reset_token_idx').execute() - - await db.schema - .alterTable('user_account') - .dropColumn('passwordResetToken') - .execute() - - await db.schema - .alterTable('user_account') - .dropColumn('passwordResetGrantedAt') - .execute() - - await db.schema.dropTable('delete_account_token').execute() -} - -export async function down(db: Kysely): Promise { - await db.schema.dropTable('email_token').execute() - await db.schema - .alterTable('user_account') - .dropColumn('emailConfirmedAt') - .execute() - - await db.schema - .createIndex('user_account_password_reset_token_idx') - .unique() - .on('user_account') - .column('passwordResetToken') - .execute() - - await db.schema - .alterTable('user_account') - .addColumn('passwordResetToken', 'varchar') - .execute() - - await db.schema - .alterTable('user_account') - .addColumn('passwordResetGrantedAt', 'varchar') - .execute() - - await db.schema - .createTable('delete_account_token') - .addColumn('did', 'varchar', (col) => col.primaryKey()) - .addColumn('token', 'varchar', (col) => col.notNull()) - .addColumn('requestedAt', 'varchar', (col) => col.notNull()) - .execute() -} diff --git a/packages/pds/src/db/migrations/20230929T213219699Z-takedown-id-as-int.ts b/packages/pds/src/db/migrations/20230929T213219699Z-takedown-id-as-int.ts deleted file mode 100644 index 8cacc599c60..00000000000 --- a/packages/pds/src/db/migrations/20230929T213219699Z-takedown-id-as-int.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { Kysely, sql } from 'kysely' -import { Dialect } from '..' - -export async function up(db: Kysely, dialect: Dialect): Promise { - if (dialect === 'pg') { - await sql` - alter table "repo_root" alter column "takedownId" type integer using "takedownId"::integer; - alter table "repo_blob" alter column "takedownId" type integer using "takedownId"::integer; - alter table "record" alter column "takedownId" type integer using "takedownId"::integer; - `.execute(db) - } else { - await sql`alter table "repo_root" drop column "takedownId"`.execute(db) - await sql`alter table "repo_root" add column "takedownId" integer`.execute( - db, - ) - await sql`alter table "repo_blob" drop column "takedownId"`.execute(db) - await sql`alter table "repo_blob" add column "takedownId" integer`.execute( - db, - ) - await sql`alter table "record" drop column "takedownId"`.execute(db) - await sql`alter table "record" add column "takedownId" integer`.execute(db) - } -} - -export async function down( - db: Kysely, - dialect: Dialect, -): Promise { - if (dialect === 'pg') { - await sql` - alter table "repo_root" alter column "takedownId" type varchar; - alter table "repo_blob" alter column "takedownId" type varchar; - alter table "record" alter column "takedownId" type varchar; - `.execute(db) - } else { - await sql`alter table "repo_root" drop column "takedownId"`.execute(db) - await sql`alter table "repo_root" add column "takedownId" varchar`.execute( - db, - ) - await sql`alter table "repo_blob" drop column "takedownId"`.execute(db) - await sql`alter table "repo_blob" add column "takedownId" varchar`.execute( - db, - ) - await sql`alter table "record" drop column "takedownId"`.execute(db) - await sql`alter table "record" add column "takedownId" varchar`.execute(db) - } -} diff --git a/packages/pds/src/db/migrations/20231011T155513453Z-takedown-ref.ts b/packages/pds/src/db/migrations/20231011T155513453Z-takedown-ref.ts deleted file mode 100644 index e0d4d16e1f1..00000000000 --- a/packages/pds/src/db/migrations/20231011T155513453Z-takedown-ref.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { Kysely } from 'kysely' - -export async function up(db: Kysely): Promise { - await db.schema - .alterTable('repo_root') - .addColumn('takedownRef', 'varchar') - .execute() - await db.schema.alterTable('repo_root').dropColumn('takedownId').execute() - - await db.schema - .alterTable('repo_blob') - .addColumn('takedownRef', 'varchar') - .execute() - await db.schema.alterTable('repo_blob').dropColumn('takedownId').execute() - - await db.schema - .alterTable('record') - .addColumn('takedownRef', 'varchar') - .execute() - await db.schema.alterTable('record').dropColumn('takedownId').execute() -} - -export async function down(db: Kysely): Promise { - await db.schema - .alterTable('repo_root') - .addColumn('takedownId', 'integer') - .execute() - await db.schema.alterTable('repo_root').dropColumn('takedownRef').execute() - - await db.schema - .alterTable('repo_blob') - .addColumn('takedownId', 'integer') - .execute() - await db.schema.alterTable('repo_blob').dropColumn('takedownRef').execute() - - await db.schema - .alterTable('record') - .addColumn('takedownId', 'integer') - .execute() - await db.schema.alterTable('record').dropColumn('takedownRef').execute() -} diff --git a/packages/pds/src/db/migrations/index.ts b/packages/pds/src/db/migrations/index.ts deleted file mode 100644 index 51979099feb..00000000000 --- a/packages/pds/src/db/migrations/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -// NOTE this file can be edited by hand, but it is also appended to by the migration:create command. -// It's important that every migration is exported from here with the proper name. We'd simplify -// this with kysely's FileMigrationProvider, but it doesn't play nicely with the build process. - -export * as _20230613T164932261Z from './20230613T164932261Z-init' -export * as _20230914T014727199Z from './20230914T014727199Z-repo-v3' -export * as _20230926T195532354Z from './20230926T195532354Z-email-tokens' -export * as _20230929T213219699Z from './20230929T213219699Z-takedown-id-as-int' -export * as _20231011T155513453Z from './20231011T155513453Z-takedown-ref' diff --git a/packages/pds/src/db/migrations/provider.ts b/packages/pds/src/db/migrations/provider.ts deleted file mode 100644 index f5e77eec871..00000000000 --- a/packages/pds/src/db/migrations/provider.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { Kysely, Migration, MigrationProvider } from 'kysely' - -// Passes a context argument to migrations. We use this to thread the dialect into migrations - -export class CtxMigrationProvider implements MigrationProvider { - constructor( - private migrations: Record>, - private ctx: T, - ) {} - async getMigrations(): Promise> { - const ctxMigrations: Record = {} - Object.entries(this.migrations).forEach(([name, migration]) => { - ctxMigrations[name] = { - up: async (db) => await migration.up(db, this.ctx), - down: async (db) => await migration.down?.(db, this.ctx), - } - }) - return ctxMigrations - } -} - -export interface CtxMigration { - up(db: Kysely, ctx: T): Promise - down?(db: Kysely, ctx: T): Promise -} diff --git a/packages/pds/src/db/migrator.ts b/packages/pds/src/db/migrator.ts new file mode 100644 index 00000000000..00d6cff44f2 --- /dev/null +++ b/packages/pds/src/db/migrator.ts @@ -0,0 +1,36 @@ +import { Kysely, Migrator as KyselyMigrator, Migration } from 'kysely' + +export class Migrator extends KyselyMigrator { + constructor(public db: Kysely, migrations: Record) { + super({ + db, + provider: { + async getMigrations() { + return migrations + }, + }, + }) + } + + async migrateToOrThrow(migration: string) { + const { error, results } = await this.migrateTo(migration) + if (error) { + throw error + } + if (!results) { + throw new Error('An unknown failure occurred while migrating') + } + return results + } + + async migrateToLatestOrThrow() { + const { error, results } = await this.migrateToLatest() + if (error) { + throw error + } + if (!results) { + throw new Error('An unknown failure occurred while migrating') + } + return results + } +} diff --git a/packages/pds/src/db/tables/app-migration.ts b/packages/pds/src/db/tables/app-migration.ts deleted file mode 100644 index 1c6b5680128..00000000000 --- a/packages/pds/src/db/tables/app-migration.ts +++ /dev/null @@ -1,9 +0,0 @@ -export interface AppMigration { - id: string - success: 0 | 1 - completedAt: string | null -} - -export const tableName = 'app_migration' - -export type PartialDB = { [tableName]: AppMigration } diff --git a/packages/pds/src/db/tables/did-cache.ts b/packages/pds/src/db/tables/did-cache.ts deleted file mode 100644 index fb0573c0012..00000000000 --- a/packages/pds/src/db/tables/did-cache.ts +++ /dev/null @@ -1,11 +0,0 @@ -export interface DidCache { - did: string - doc: string // json representation of DidDocument - updatedAt: number -} - -export const tableName = 'did_cache' - -export type PartialDB = { - [tableName]: DidCache -} diff --git a/packages/pds/src/db/tables/did-handle.ts b/packages/pds/src/db/tables/did-handle.ts deleted file mode 100644 index 9271d44ebd0..00000000000 --- a/packages/pds/src/db/tables/did-handle.ts +++ /dev/null @@ -1,9 +0,0 @@ -// @NOTE also used by app-view -export interface DidHandle { - did: string - handle: string -} - -export const tableName = 'did_handle' - -export type PartialDB = { [tableName]: DidHandle } diff --git a/packages/pds/src/db/tables/ipld-block.ts b/packages/pds/src/db/tables/ipld-block.ts deleted file mode 100644 index ce7bd30a51a..00000000000 --- a/packages/pds/src/db/tables/ipld-block.ts +++ /dev/null @@ -1,11 +0,0 @@ -export interface IpldBlock { - cid: string - creator: string - repoRev: string | null - size: number - content: Uint8Array -} - -export const tableName = 'ipld_block' - -export type PartialDB = { [tableName]: IpldBlock } diff --git a/packages/pds/src/db/tables/repo-blob.ts b/packages/pds/src/db/tables/repo-blob.ts deleted file mode 100644 index ddeb6c59158..00000000000 --- a/packages/pds/src/db/tables/repo-blob.ts +++ /dev/null @@ -1,11 +0,0 @@ -export interface RepoBlob { - cid: string - recordUri: string - repoRev: string | null - did: string - takedownRef: string | null -} - -export const tableName = 'repo_blob' - -export type PartialDB = { [tableName]: RepoBlob } diff --git a/packages/pds/src/db/tables/repo-seq.ts b/packages/pds/src/db/tables/repo-seq.ts deleted file mode 100644 index ffd482c327a..00000000000 --- a/packages/pds/src/db/tables/repo-seq.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Generated, GeneratedAlways, Insertable, Selectable } from 'kysely' - -export type EventType = 'append' | 'rebase' | 'handle' | 'migrate' | 'tombstone' - -export const REPO_SEQ_SEQUENCE = 'repo_seq_sequence' - -export interface RepoSeq { - id: GeneratedAlways - seq: number | null - did: string - eventType: EventType - event: Uint8Array - invalidated: Generated<0 | 1> - sequencedAt: string -} - -export type RepoSeqInsert = Insertable -export type RepoSeqEntry = Selectable - -export const tableName = 'repo_seq' - -export type PartialDB = { - [tableName]: RepoSeq -} diff --git a/packages/pds/src/db/tables/runtime-flag.ts b/packages/pds/src/db/tables/runtime-flag.ts deleted file mode 100644 index f1e701a6914..00000000000 --- a/packages/pds/src/db/tables/runtime-flag.ts +++ /dev/null @@ -1,8 +0,0 @@ -export interface RuntimeFlag { - name: string - value: string -} - -export const tableName = 'runtime_flag' - -export type PartialDB = { [tableName]: RuntimeFlag } diff --git a/packages/pds/src/db/tables/user-account.ts b/packages/pds/src/db/tables/user-account.ts deleted file mode 100644 index 808663ca468..00000000000 --- a/packages/pds/src/db/tables/user-account.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Generated, Selectable } from 'kysely' - -export interface UserAccount { - did: string - email: string - passwordScrypt: string - createdAt: string - emailConfirmedAt: string | null - invitesDisabled: Generated<0 | 1> - inviteNote: string | null -} - -export type UserAccountEntry = Selectable - -export const tableName = 'user_account' - -export type PartialDB = { [tableName]: UserAccount } diff --git a/packages/pds/src/db/tables/user-pref.ts b/packages/pds/src/db/tables/user-pref.ts deleted file mode 100644 index d72b902861b..00000000000 --- a/packages/pds/src/db/tables/user-pref.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { GeneratedAlways } from 'kysely' - -export interface UserPref { - id: GeneratedAlways - did: string - name: string - valueJson: string // json -} - -export const tableName = 'user_pref' - -export type PartialDB = { [tableName]: UserPref } diff --git a/packages/pds/src/db/types.ts b/packages/pds/src/db/types.ts deleted file mode 100644 index ce697cd7e60..00000000000 --- a/packages/pds/src/db/types.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { DynamicReferenceBuilder } from 'kysely/dist/cjs/dynamic/dynamic-reference-builder' - -export type Ref = DynamicReferenceBuilder diff --git a/packages/pds/src/db/util.ts b/packages/pds/src/db/util.ts index 09970e3be8b..17b84822753 100644 --- a/packages/pds/src/db/util.ts +++ b/packages/pds/src/db/util.ts @@ -1,22 +1,16 @@ +import { retry } from '@atproto/common' import { DummyDriver, DynamicModule, + Kysely, RawBuilder, + ReferenceExpression, SelectQueryBuilder, sql, SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler, } from 'kysely' -import DatabaseSchema from './database-schema' - -export const actorWhereClause = (actor: string) => { - if (actor.startsWith('did:')) { - return sql<0 | 1>`"did_handle"."did" = ${actor}` - } else { - return sql<0 | 1>`"did_handle"."handle" = ${actor}` - } -} // Applies to repo_root or record table export const notSoftDeletedClause = (alias: DbRef) => { @@ -30,7 +24,7 @@ export const softDeleted = (repoOrRecord: { takedownRef: string | null }) => { export const countAll = sql`count(*)` // For use with doUpdateSet() -export const excluded = (db: DatabaseSchema, col) => { +export const excluded = (db: Kysely, col) => { return sql`${db.dynamic.ref(`excluded.${col}`)}` } @@ -54,6 +48,49 @@ export const dummyDialect = { }, } +export const retrySqlite = (fn: () => Promise): Promise => { + return retry(fn, { + retryable: retryableSqlite, + getWaitMs: getWaitMsSqlite, + maxRetries: 60, // a safety measure: getWaitMsSqlite() times out before this after 5000ms of waiting. + }) +} + +const retryableSqlite = (err: unknown) => { + return typeof err?.['code'] === 'string' && RETRY_ERRORS.has(err['code']) +} + +// based on sqlite's backoff strategy https://github.com/sqlite/sqlite/blob/91c8e65dd4bf17d21fbf8f7073565fe1a71c8948/src/main.c#L1704-L1713 +const getWaitMsSqlite = (n: number, timeout = 5000) => { + if (n < 0) return null + let delay: number + let prior: number + if (n < DELAYS.length) { + delay = DELAYS[n] + prior = TOTALS[n] + } else { + delay = last(DELAYS) + prior = last(TOTALS) + delay * (n - (DELAYS.length - 1)) + } + if (prior + delay > timeout) { + delay = timeout - prior + if (delay <= 0) return null + } + return delay +} + +const last = (arr: T[]) => arr[arr.length - 1] +const DELAYS = [1, 2, 5, 10, 15, 20, 25, 25, 25, 50, 50, 100] +const TOTALS = [0, 1, 3, 8, 18, 33, 53, 78, 103, 128, 178, 228] +const RETRY_ERRORS = new Set([ + 'SQLITE_BUSY', + 'SQLITE_BUSY_SNAPSHOT', + 'SQLITE_BUSY_RECOVERY', + 'SQLITE_BUSY_TIMEOUT', +]) + +export type Ref = ReferenceExpression + export type DbRef = RawBuilder | ReturnType export type AnyQb = SelectQueryBuilder diff --git a/packages/pds/src/did-cache.ts b/packages/pds/src/did-cache.ts deleted file mode 100644 index ac719fe922c..00000000000 --- a/packages/pds/src/did-cache.ts +++ /dev/null @@ -1,107 +0,0 @@ -import PQueue from 'p-queue' -import { CacheResult, DidCache, DidDocument } from '@atproto/identity' -import Database from './db' -import { excluded } from './db/util' -import { dbLogger } from './logger' - -export class DidSqlCache implements DidCache { - public pQueue: PQueue | null //null during teardown - - constructor( - public db: Database, - public staleTTL: number, - public maxTTL: number, - ) { - this.pQueue = new PQueue() - } - - async cacheDid( - did: string, - doc: DidDocument, - prevResult?: CacheResult, - ): Promise { - if (prevResult) { - await this.db.db - .updateTable('did_cache') - .set({ doc: JSON.stringify(doc), updatedAt: Date.now() }) - .where('did', '=', did) - .where('updatedAt', '=', prevResult.updatedAt) - .execute() - } else { - await this.db.db - .insertInto('did_cache') - .values({ did, doc: JSON.stringify(doc), updatedAt: Date.now() }) - .onConflict((oc) => - oc.column('did').doUpdateSet({ - doc: excluded(this.db.db, 'doc'), - updatedAt: excluded(this.db.db, 'updatedAt'), - }), - ) - .executeTakeFirst() - } - } - - async refreshCache( - did: string, - getDoc: () => Promise, - prevResult?: CacheResult, - ): Promise { - this.pQueue?.add(async () => { - try { - const doc = await getDoc() - if (doc) { - await this.cacheDid(did, doc, prevResult) - } else { - await this.clearEntry(did) - } - } catch (err) { - dbLogger.error({ did, err }, 'refreshing did cache failed') - } - }) - } - - async checkCache(did: string): Promise { - const res = await this.db.db - .selectFrom('did_cache') - .where('did', '=', did) - .selectAll() - .executeTakeFirst() - if (!res) return null - const now = Date.now() - const updatedAt = new Date(res.updatedAt).getTime() - const expired = now > updatedAt + this.maxTTL - const stale = now > updatedAt + this.staleTTL - return { - doc: JSON.parse(res.doc) as DidDocument, - updatedAt, - did, - stale, - expired, - } - } - - async clearEntry(did: string): Promise { - await this.db.db - .deleteFrom('did_cache') - .where('did', '=', did) - .executeTakeFirst() - } - - async clear(): Promise { - await this.db.db.deleteFrom('did_cache').execute() - } - - async processAll() { - await this.pQueue?.onIdle() - } - - async destroy() { - const pQueue = this.pQueue - this.pQueue = null - pQueue?.pause() - pQueue?.clear() - await pQueue?.onIdle() - } -} - -export default DidSqlCache diff --git a/packages/pds/src/did-cache/db/index.ts b/packages/pds/src/did-cache/db/index.ts new file mode 100644 index 00000000000..25031c63431 --- /dev/null +++ b/packages/pds/src/did-cache/db/index.ts @@ -0,0 +1,21 @@ +import { Database, Migrator } from '../../db' +import { DidCacheSchema } from './schema' +import migrations from './migrations' + +export * from './schema' + +export type DidCacheDb = Database + +export const getDb = ( + location: string, + disableWalAutoCheckpoint = false, +): DidCacheDb => { + const pragmas: Record = disableWalAutoCheckpoint + ? { wal_autocheckpoint: '0', synchronous: 'NORMAL' } + : { synchronous: 'NORMAL' } + return Database.sqlite(location, { pragmas }) +} + +export const getMigrator = (db: DidCacheDb) => { + return new Migrator(db.db, migrations) +} diff --git a/packages/pds/src/did-cache/db/migrations.ts b/packages/pds/src/did-cache/db/migrations.ts new file mode 100644 index 00000000000..4cf340a2b3a --- /dev/null +++ b/packages/pds/src/did-cache/db/migrations.ts @@ -0,0 +1,17 @@ +import { Kysely } from 'kysely' + +export default { + '001': { + up: async (db: Kysely) => { + await db.schema + .createTable('did_doc') + .addColumn('did', 'varchar', (col) => col.primaryKey()) + .addColumn('doc', 'text', (col) => col.notNull()) + .addColumn('updatedAt', 'bigint', (col) => col.notNull()) + .execute() + }, + down: async (db: Kysely) => { + await db.schema.dropTable('did_doc').execute() + }, + }, +} diff --git a/packages/pds/src/did-cache/db/schema.ts b/packages/pds/src/did-cache/db/schema.ts new file mode 100644 index 00000000000..8f01d4f9987 --- /dev/null +++ b/packages/pds/src/did-cache/db/schema.ts @@ -0,0 +1,9 @@ +export interface DidDoc { + did: string + doc: string // json representation of DidDocument + updatedAt: number +} + +export type DidCacheSchema = { + did_doc: DidDoc +} diff --git a/packages/pds/src/did-cache/index.ts b/packages/pds/src/did-cache/index.ts new file mode 100644 index 00000000000..526283090c5 --- /dev/null +++ b/packages/pds/src/did-cache/index.ts @@ -0,0 +1,131 @@ +import PQueue from 'p-queue' +import { CacheResult, DidCache, DidDocument } from '@atproto/identity' +import { excluded } from '../db/util' +import { didCacheLogger } from '../logger' +import { DidCacheDb, getMigrator, getDb } from './db' + +export class DidSqliteCache implements DidCache { + db: DidCacheDb + public pQueue: PQueue | null //null during teardown + + constructor( + dbLocation: string, + public staleTTL: number, + public maxTTL: number, + disableWalAutoCheckpoint = false, + ) { + this.db = getDb(dbLocation, disableWalAutoCheckpoint) + this.pQueue = new PQueue() + } + + async cacheDid( + did: string, + doc: DidDocument, + prevResult?: CacheResult, + ): Promise { + try { + if (prevResult) { + await this.db.executeWithRetry( + this.db.db + .updateTable('did_doc') + .set({ doc: JSON.stringify(doc), updatedAt: Date.now() }) + .where('did', '=', did) + .where('updatedAt', '=', prevResult.updatedAt), + ) + } else { + await this.db.executeWithRetry( + this.db.db + .insertInto('did_doc') + .values({ did, doc: JSON.stringify(doc), updatedAt: Date.now() }) + .onConflict((oc) => + oc.column('did').doUpdateSet({ + doc: excluded(this.db.db, 'doc'), + updatedAt: excluded(this.db.db, 'updatedAt'), + }), + ), + ) + } + } catch (err) { + didCacheLogger.error({ did, doc, err }, 'failed to cache did') + } + } + + async refreshCache( + did: string, + getDoc: () => Promise, + prevResult?: CacheResult, + ): Promise { + this.pQueue?.add(async () => { + try { + const doc = await getDoc() + if (doc) { + await this.cacheDid(did, doc, prevResult) + } else { + await this.clearEntry(did) + } + } catch (err) { + didCacheLogger.error({ did, err }, 'refreshing did cache failed') + } + }) + } + + async checkCache(did: string): Promise { + try { + return await this.checkCacheInternal(did) + } catch (err) { + didCacheLogger.error({ did, err }, 'failed to check did cache') + return null + } + } + + async checkCacheInternal(did: string): Promise { + const res = await this.db.db + .selectFrom('did_doc') + .where('did', '=', did) + .selectAll() + .executeTakeFirst() + if (!res) return null + const now = Date.now() + const updatedAt = new Date(res.updatedAt).getTime() + const expired = now > updatedAt + this.maxTTL + const stale = now > updatedAt + this.staleTTL + return { + doc: JSON.parse(res.doc) as DidDocument, + updatedAt, + did, + stale, + expired, + } + } + + async clearEntry(did: string): Promise { + try { + await this.db.executeWithRetry( + this.db.db.deleteFrom('did_doc').where('did', '=', did), + ) + } catch (err) { + didCacheLogger.error({ did, err }, 'clearing did cache entry failed') + } + } + + async clear(): Promise { + await this.db.db.deleteFrom('did_doc').execute() + } + + async processAll() { + await this.pQueue?.onIdle() + } + + async migrateOrThrow() { + await this.db.ensureWal() + await getMigrator(this.db).migrateToLatestOrThrow() + } + + async destroy() { + const pQueue = this.pQueue + this.pQueue = null + pQueue?.pause() + pQueue?.clear() + await pQueue?.onIdle() + } +} diff --git a/packages/pds/src/storage/disk-blobstore.ts b/packages/pds/src/disk-blobstore.ts similarity index 53% rename from packages/pds/src/storage/disk-blobstore.ts rename to packages/pds/src/disk-blobstore.ts index 496e7b42c52..8420f0b2046 100644 --- a/packages/pds/src/storage/disk-blobstore.ts +++ b/packages/pds/src/disk-blobstore.ts @@ -1,42 +1,45 @@ import fs from 'fs/promises' import fsSync from 'fs' import stream from 'stream' -import os from 'os' import path from 'path' import { CID } from 'multiformats/cid' import { BlobNotFoundError, BlobStore } from '@atproto/repo' import { randomStr } from '@atproto/crypto' -import { httpLogger as log } from '../logger' -import { isErrnoException, fileExists } from '@atproto/common' +import { httpLogger as log } from './logger' +import { isErrnoException, fileExists, rmIfExists } from '@atproto/common' export class DiskBlobStore implements BlobStore { - location: string - tmpLocation: string - quarantineLocation: string - constructor( - location: string, - tmpLocation: string, - quarantineLocation: string, - ) { - this.location = location - this.tmpLocation = tmpLocation - this.quarantineLocation = quarantineLocation - } + public did: string, + public location: string, + public tmpLocation: string, + public quarantineLocation: string, + ) {} - static async create( + static creator( location: string, tmpLocation?: string, quarantineLocation?: string, - ): Promise { - const tmp = tmpLocation || path.join(os.tmpdir(), 'atproto/blobs') - const quarantine = quarantineLocation || path.join(location, 'quarantine') - await Promise.all([ - fs.mkdir(location, { recursive: true }), - fs.mkdir(tmp, { recursive: true }), - fs.mkdir(quarantine, { recursive: true }), - ]) - return new DiskBlobStore(location, tmp, quarantine) + ) { + return (did: string) => { + const tmp = tmpLocation || path.join(location, 'tempt') + const quarantine = quarantineLocation || path.join(location, 'quarantine') + return new DiskBlobStore(did, location, tmp, quarantine) + } + } + + private async ensureDir() { + await fs.mkdir(path.join(this.location, this.did), { recursive: true }) + } + + private async ensureTemp() { + await fs.mkdir(path.join(this.tmpLocation, this.did), { recursive: true }) + } + + private async ensureQuarantine() { + await fs.mkdir(path.join(this.quarantineLocation, this.did), { + recursive: true, + }) } private genKey() { @@ -44,15 +47,15 @@ export class DiskBlobStore implements BlobStore { } getTmpPath(key: string): string { - return path.join(this.tmpLocation, key) + return path.join(this.tmpLocation, this.did, key) } getStoredPath(cid: CID): string { - return path.join(this.location, cid.toString()) + return path.join(this.location, this.did, cid.toString()) } getQuarantinePath(cid: CID): string { - return path.join(this.quarantineLocation, cid.toString()) + return path.join(this.quarantineLocation, this.did, cid.toString()) } async hasTemp(key: string): Promise { @@ -64,12 +67,14 @@ export class DiskBlobStore implements BlobStore { } async putTemp(bytes: Uint8Array | stream.Readable): Promise { + await this.ensureTemp() const key = this.genKey() await fs.writeFile(this.getTmpPath(key), bytes) return key } async makePermanent(key: string, cid: CID): Promise { + await this.ensureDir() const tmpPath = this.getTmpPath(key) const storedPath = this.getStoredPath(cid) const alreadyHas = await this.hasStored(cid) @@ -88,25 +93,33 @@ export class DiskBlobStore implements BlobStore { cid: CID, bytes: Uint8Array | stream.Readable, ): Promise { + await this.ensureDir() await fs.writeFile(this.getStoredPath(cid), bytes) } async quarantine(cid: CID): Promise { - await fs.rename(this.getStoredPath(cid), this.getQuarantinePath(cid)) + await this.ensureQuarantine() + try { + await fs.rename(this.getStoredPath(cid), this.getQuarantinePath(cid)) + } catch (err) { + throw translateErr(err) + } } async unquarantine(cid: CID): Promise { - await fs.rename(this.getQuarantinePath(cid), this.getStoredPath(cid)) + await this.ensureDir() + try { + await fs.rename(this.getQuarantinePath(cid), this.getStoredPath(cid)) + } catch (err) { + throw translateErr(err) + } } async getBytes(cid: CID): Promise { try { return await fs.readFile(this.getStoredPath(cid)) } catch (err) { - if (isErrnoException(err) && err.code === 'ENOENT') { - throw new BlobNotFoundError() - } - throw err + throw translateErr(err) } } @@ -120,16 +133,23 @@ export class DiskBlobStore implements BlobStore { } async delete(cid: CID): Promise { - try { - await fs.rm(this.getStoredPath(cid)) - } catch (err) { - if (isErrnoException(err) && err.code === 'ENOENT') { - // if blob not found, then it's already been deleted & we can just return - return - } - throw err - } + await rmIfExists(this.getStoredPath(cid)) + } + + async deleteMany(cids: CID[]): Promise { + await Promise.all(cids.map((cid) => this.delete(cid))) + } + + async deleteAll(): Promise { + await rmIfExists(this.location, true) + } +} + +const translateErr = (err: unknown): BlobNotFoundError | unknown => { + if (isErrnoException(err) && err.code === 'ENOENT') { + return new BlobNotFoundError() } + return err } export default DiskBlobStore diff --git a/packages/pds/src/index.ts b/packages/pds/src/index.ts index 42544eba492..77f5948e8aa 100644 --- a/packages/pds/src/index.ts +++ b/packages/pds/src/index.ts @@ -19,7 +19,7 @@ import API from './api' import * as basicRoutes from './basic-routes' import * as wellKnown from './well-known' import * as error from './error' -import { dbLogger, loggerMiddleware, seqLogger } from './logger' +import { loggerMiddleware } from './logger' import { ServerConfig, ServerSecrets } from './config' import { createServer } from './lexicon' import { createHttpTerminator, HttpTerminator } from 'http-terminator' @@ -28,7 +28,7 @@ import compression from './util/compression' export * from './config' export { Database } from './db' -export { DiskBlobStore, MemoryBlobStore } from './storage' +export { DiskBlobStore } from './disk-blobstore' export { AppContext } from './context' export { httpLogger } from './logger' @@ -128,41 +128,7 @@ export class PDS { } async start(): Promise { - const { db, backgroundQueue } = this.ctx - if (db.cfg.dialect === 'pg') { - const { pool } = db.cfg - this.dbStatsInterval = setInterval(() => { - dbLogger.info( - { - idleCount: pool.idleCount, - totalCount: pool.totalCount, - waitingCount: pool.waitingCount, - }, - 'db pool stats', - ) - dbLogger.info( - { - runningCount: backgroundQueue.queue.pending, - waitingCount: backgroundQueue.queue.size, - }, - 'background queue stats', - ) - }, 10000) - } - this.sequencerStatsInterval = setInterval(async () => { - if (this.ctx.sequencerLeader?.isLeader) { - try { - const seq = await this.ctx.sequencerLeader.lastSeq() - seqLogger.info({ seq }, 'sequencer leader stats') - } catch (err) { - seqLogger.error({ err }, 'error getting last seq') - } - } - }, 500) - this.ctx.sequencerLeader?.run() await this.ctx.sequencer.start() - await this.ctx.db.startListeningToChannels() - await this.ctx.runtimeFlags.start() const server = this.app.listen(this.ctx.cfg.service.port) this.server = server this.server.keepAliveTimeout = 90000 @@ -172,11 +138,10 @@ export class PDS { } async destroy(): Promise { - await this.ctx.runtimeFlags.destroy() - await this.ctx.sequencerLeader?.destroy() + await this.ctx.sequencer.destroy() await this.terminator?.terminate() await this.ctx.backgroundQueue.destroy() - await this.ctx.db.close() + await this.ctx.accountManager.close() await this.ctx.redisScratch?.quit() clearInterval(this.dbStatsInterval) clearInterval(this.sequencerStatsInterval) diff --git a/packages/pds/src/lexicon/index.ts b/packages/pds/src/lexicon/index.ts index e4a075bee9f..c51998a66e6 100644 --- a/packages/pds/src/lexicon/index.ts +++ b/packages/pds/src/lexicon/index.ts @@ -9,6 +9,7 @@ import { StreamAuthVerifier, } from '@atproto/xrpc-server' import { schemas } from './lexicons' +import * as ComAtprotoAdminDeleteAccount from './types/com/atproto/admin/deleteAccount' import * as ComAtprotoAdminDisableAccountInvites from './types/com/atproto/admin/disableAccountInvites' import * as ComAtprotoAdminDisableInviteCodes from './types/com/atproto/admin/disableInviteCodes' import * as ComAtprotoAdminEmitModerationEvent from './types/com/atproto/admin/emitModerationEvent' @@ -73,6 +74,9 @@ import * as ComAtprotoSyncNotifyOfUpdate from './types/com/atproto/sync/notifyOf import * as ComAtprotoSyncRequestCrawl from './types/com/atproto/sync/requestCrawl' import * as ComAtprotoSyncSubscribeRepos from './types/com/atproto/sync/subscribeRepos' import * as ComAtprotoTempFetchLabels from './types/com/atproto/temp/fetchLabels' +import * as ComAtprotoTempImportRepo from './types/com/atproto/temp/importRepo' +import * as ComAtprotoTempPushBlob from './types/com/atproto/temp/pushBlob' +import * as ComAtprotoTempTransferAccount from './types/com/atproto/temp/transferAccount' import * as AppBskyActorGetPreferences from './types/app/bsky/actor/getPreferences' import * as AppBskyActorGetProfile from './types/app/bsky/actor/getProfile' import * as AppBskyActorGetProfiles from './types/app/bsky/actor/getProfiles' @@ -194,6 +198,17 @@ export class AdminNS { this._server = server } + deleteAccount( + cfg: ConfigOf< + AV, + ComAtprotoAdminDeleteAccount.Handler>, + ComAtprotoAdminDeleteAccount.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.admin.deleteAccount' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } + disableAccountInvites( cfg: ConfigOf< AV, @@ -953,6 +968,39 @@ export class TempNS { const nsid = 'com.atproto.temp.fetchLabels' // @ts-ignore return this._server.xrpc.method(nsid, cfg) } + + importRepo( + cfg: ConfigOf< + AV, + ComAtprotoTempImportRepo.Handler>, + ComAtprotoTempImportRepo.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.temp.importRepo' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } + + pushBlob( + cfg: ConfigOf< + AV, + ComAtprotoTempPushBlob.Handler>, + ComAtprotoTempPushBlob.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.temp.pushBlob' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } + + transferAccount( + cfg: ConfigOf< + AV, + ComAtprotoTempTransferAccount.Handler>, + ComAtprotoTempTransferAccount.HandlerReqCtx> + >, + ) { + const nsid = 'com.atproto.temp.transferAccount' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } } export class AppNS { @@ -1549,11 +1597,13 @@ type RouteRateLimitOpts = { calcKey?: (ctx: T) => string calcPoints?: (ctx: T) => number } +type HandlerOpts = { blobLimit?: number } type HandlerRateLimitOpts = SharedRateLimitOpts | RouteRateLimitOpts type ConfigOf = | Handler | { auth?: Auth + opts?: HandlerOpts rateLimit?: HandlerRateLimitOpts | HandlerRateLimitOpts[] handler: Handler } diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index a6377d85a9a..90176ef6486 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -820,6 +820,29 @@ export const schemaDict = { }, }, }, + ComAtprotoAdminDeleteAccount: { + lexicon: 1, + id: 'com.atproto.admin.deleteAccount', + defs: { + main: { + type: 'procedure', + description: 'Delete a user account as an administrator.', + input: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['did'], + properties: { + did: { + type: 'string', + format: 'did', + }, + }, + }, + }, + }, + }, + }, ComAtprotoAdminDisableAccountInvites: { lexicon: 1, id: 'com.atproto.admin.disableAccountInvites', @@ -3979,6 +4002,135 @@ export const schemaDict = { }, }, }, + ComAtprotoTempImportRepo: { + lexicon: 1, + id: 'com.atproto.temp.importRepo', + defs: { + main: { + type: 'procedure', + description: + "Gets the did's repo, optionally catching up from a specific revision.", + parameters: { + type: 'params', + required: ['did'], + properties: { + did: { + type: 'string', + format: 'did', + description: 'The DID of the repo.', + }, + }, + }, + input: { + encoding: 'application/vnd.ipld.car', + }, + output: { + encoding: 'text/plain', + }, + }, + }, + }, + ComAtprotoTempPushBlob: { + lexicon: 1, + id: 'com.atproto.temp.pushBlob', + defs: { + main: { + type: 'procedure', + description: + "Gets the did's repo, optionally catching up from a specific revision.", + parameters: { + type: 'params', + required: ['did'], + properties: { + did: { + type: 'string', + format: 'did', + description: 'The DID of the repo.', + }, + }, + }, + input: { + encoding: '*/*', + }, + }, + }, + }, + ComAtprotoTempTransferAccount: { + lexicon: 1, + id: 'com.atproto.temp.transferAccount', + defs: { + main: { + type: 'procedure', + description: 'Transfer an account.', + input: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['handle', 'did', 'plcOp'], + properties: { + handle: { + type: 'string', + format: 'handle', + }, + did: { + type: 'string', + format: 'did', + }, + plcOp: { + type: 'unknown', + }, + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['accessJwt', 'refreshJwt', 'handle', 'did'], + properties: { + accessJwt: { + type: 'string', + }, + refreshJwt: { + type: 'string', + }, + handle: { + type: 'string', + format: 'handle', + }, + did: { + type: 'string', + format: 'did', + }, + }, + }, + }, + errors: [ + { + name: 'InvalidHandle', + }, + { + name: 'InvalidPassword', + }, + { + name: 'InvalidInviteCode', + }, + { + name: 'HandleNotAvailable', + }, + { + name: 'UnsupportedDomain', + }, + { + name: 'UnresolvableDid', + }, + { + name: 'IncompatibleDidDoc', + }, + ], + }, + }, + }, AppBskyActorDefs: { lexicon: 1, id: 'app.bsky.actor.defs', @@ -7671,6 +7823,7 @@ export const schemas: LexiconDoc[] = Object.values(schemaDict) as LexiconDoc[] export const lexicons: Lexicons = new Lexicons(schemas) export const ids = { ComAtprotoAdminDefs: 'com.atproto.admin.defs', + ComAtprotoAdminDeleteAccount: 'com.atproto.admin.deleteAccount', ComAtprotoAdminDisableAccountInvites: 'com.atproto.admin.disableAccountInvites', ComAtprotoAdminDisableInviteCodes: 'com.atproto.admin.disableInviteCodes', @@ -7746,6 +7899,9 @@ export const ids = { ComAtprotoSyncRequestCrawl: 'com.atproto.sync.requestCrawl', ComAtprotoSyncSubscribeRepos: 'com.atproto.sync.subscribeRepos', ComAtprotoTempFetchLabels: 'com.atproto.temp.fetchLabels', + ComAtprotoTempImportRepo: 'com.atproto.temp.importRepo', + ComAtprotoTempPushBlob: 'com.atproto.temp.pushBlob', + ComAtprotoTempTransferAccount: 'com.atproto.temp.transferAccount', AppBskyActorDefs: 'app.bsky.actor.defs', AppBskyActorGetPreferences: 'app.bsky.actor.getPreferences', AppBskyActorGetProfile: 'app.bsky.actor.getProfile', diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/deleteAccount.ts b/packages/pds/src/lexicon/types/com/atproto/admin/deleteAccount.ts new file mode 100644 index 00000000000..13e68eb5c7d --- /dev/null +++ b/packages/pds/src/lexicon/types/com/atproto/admin/deleteAccount.ts @@ -0,0 +1,38 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' + +export interface QueryParams {} + +export interface InputSchema { + did: string + [k: string]: unknown +} + +export interface HandlerInput { + encoding: 'application/json' + body: InputSchema +} + +export interface HandlerError { + status: number + message?: string +} + +export type HandlerOutput = HandlerError | void +export type HandlerReqCtx = { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +} +export type Handler = ( + ctx: HandlerReqCtx, +) => Promise | HandlerOutput diff --git a/packages/pds/src/lexicon/types/com/atproto/temp/importRepo.ts b/packages/pds/src/lexicon/types/com/atproto/temp/importRepo.ts new file mode 100644 index 00000000000..d88361d9856 --- /dev/null +++ b/packages/pds/src/lexicon/types/com/atproto/temp/importRepo.ts @@ -0,0 +1,45 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import stream from 'stream' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' + +export interface QueryParams { + /** The DID of the repo. */ + did: string +} + +export type InputSchema = string | Uint8Array + +export interface HandlerInput { + encoding: 'application/vnd.ipld.car' + body: stream.Readable +} + +export interface HandlerSuccess { + encoding: 'text/plain' + body: Uint8Array | stream.Readable + headers?: { [key: string]: string } +} + +export interface HandlerError { + status: number + message?: string +} + +export type HandlerOutput = HandlerError | HandlerSuccess +export type HandlerReqCtx = { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +} +export type Handler = ( + ctx: HandlerReqCtx, +) => Promise | HandlerOutput diff --git a/packages/pds/src/lexicon/types/com/atproto/temp/pushBlob.ts b/packages/pds/src/lexicon/types/com/atproto/temp/pushBlob.ts new file mode 100644 index 00000000000..97e890dbb14 --- /dev/null +++ b/packages/pds/src/lexicon/types/com/atproto/temp/pushBlob.ts @@ -0,0 +1,39 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import stream from 'stream' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' + +export interface QueryParams { + /** The DID of the repo. */ + did: string +} + +export type InputSchema = string | Uint8Array + +export interface HandlerInput { + encoding: '*/*' + body: stream.Readable +} + +export interface HandlerError { + status: number + message?: string +} + +export type HandlerOutput = HandlerError | void +export type HandlerReqCtx = { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +} +export type Handler = ( + ctx: HandlerReqCtx, +) => Promise | HandlerOutput diff --git a/packages/pds/src/lexicon/types/com/atproto/temp/transferAccount.ts b/packages/pds/src/lexicon/types/com/atproto/temp/transferAccount.ts new file mode 100644 index 00000000000..86c1d750e07 --- /dev/null +++ b/packages/pds/src/lexicon/types/com/atproto/temp/transferAccount.ts @@ -0,0 +1,62 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' + +export interface QueryParams {} + +export interface InputSchema { + handle: string + did: string + plcOp: {} + [k: string]: unknown +} + +export interface OutputSchema { + accessJwt: string + refreshJwt: string + handle: string + did: string + [k: string]: unknown +} + +export interface HandlerInput { + encoding: 'application/json' + body: InputSchema +} + +export interface HandlerSuccess { + encoding: 'application/json' + body: OutputSchema + headers?: { [key: string]: string } +} + +export interface HandlerError { + status: number + message?: string + error?: + | 'InvalidHandle' + | 'InvalidPassword' + | 'InvalidInviteCode' + | 'HandleNotAvailable' + | 'UnsupportedDomain' + | 'UnresolvableDid' + | 'IncompatibleDidDoc' +} + +export type HandlerOutput = HandlerError | HandlerSuccess +export type HandlerReqCtx = { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +} +export type Handler = ( + ctx: HandlerReqCtx, +) => Promise | HandlerOutput diff --git a/packages/pds/src/logger.ts b/packages/pds/src/logger.ts index 339359db6ea..717e554d00b 100644 --- a/packages/pds/src/logger.ts +++ b/packages/pds/src/logger.ts @@ -1,15 +1,16 @@ import pino from 'pino' import pinoHttp from 'pino-http' import { subsystemLogger } from '@atproto/common' -import * as jwt from 'jsonwebtoken' +import * as jose from 'jose' import { parseBasicAuth } from './auth-verifier' export const dbLogger = subsystemLogger('pds:db') +export const didCacheLogger = subsystemLogger('pds:did-cache') export const readStickyLogger = subsystemLogger('pds:read-sticky') export const redisLogger = subsystemLogger('pds:redis') export const seqLogger = subsystemLogger('pds:sequencer') export const mailerLogger = subsystemLogger('pds:mailer') -export const labelerLogger = subsystemLogger('pds:labler') +export const labelerLogger = subsystemLogger('pds:labeler') export const crawlerLogger = subsystemLogger('pds:crawler') export const httpLogger = subsystemLogger('pds') @@ -28,7 +29,7 @@ export const loggerMiddleware = pinoHttp({ let auth: string | undefined = undefined if (authHeader.startsWith('Bearer ')) { const token = authHeader.slice('Bearer '.length) - const sub = jwt.decode(token)?.sub + const { sub } = jose.decodeJwt(token) if (sub) { auth = 'Bearer ' + sub } else { diff --git a/packages/pds/src/mailer/index.ts b/packages/pds/src/mailer/index.ts index 92ce8a88c83..0ce54ca4f17 100644 --- a/packages/pds/src/mailer/index.ts +++ b/packages/pds/src/mailer/index.ts @@ -35,7 +35,7 @@ export class ServerMailer { } async sendResetPassword( - params: { handle: string; token: string }, + params: { identifier: string; token: string }, mailOpts: Mail.Options, ) { return this.sendTemplate('resetPassword', params, { diff --git a/packages/pds/src/mailer/templates/reset-password.hbs b/packages/pds/src/mailer/templates/reset-password.hbs index 158298bfc4f..17cadf791a1 100644 --- a/packages/pds/src/mailer/templates/reset-password.hbs +++ b/packages/pds/src/mailer/templates/reset-password.hbs @@ -171,7 +171,7 @@ align="left" >We received a request to reset the password for the account: - {{handle}} + {{identifier}} | null + posts: RecordDescript[] +} + +export type RecordDescript = { + uri: AtUri + cid: CID + indexedAt: string + record: T +} + +export type ApiRes = { + headers: Headers + data: T +} + +export type MungeFn = ( + localViewer: LocalViewer, + original: T, + local: LocalRecords, + requester: string, +) => Promise + +export type HandlerResponse = { + encoding: 'application/json' + body: T + headers?: Record +} diff --git a/packages/pds/src/api/app/bsky/util/read-after-write.ts b/packages/pds/src/read-after-write/util.ts similarity index 67% rename from packages/pds/src/api/app/bsky/util/read-after-write.ts rename to packages/pds/src/read-after-write/util.ts index b834a91c1b7..db8770a32b6 100644 --- a/packages/pds/src/api/app/bsky/util/read-after-write.ts +++ b/packages/pds/src/read-after-write/util.ts @@ -1,25 +1,7 @@ import { Headers } from '@atproto/xrpc' -import { readStickyLogger as log } from '../../../../logger' -import { LocalRecords } from '../../../../services/local' -import AppContext from '../../../../context' - -export type ApiRes = { - headers: Headers - data: T -} - -export type MungeFn = ( - ctx: AppContext, - original: T, - local: LocalRecords, - requester: string, -) => Promise - -export type HandlerResponse = { - encoding: 'application/json' - body: T - headers?: Record -} +import { readStickyLogger as log } from '../logger' +import AppContext from '../context' +import { ApiRes, HandlerResponse, LocalRecords, MungeFn } from './types' export const getRepoRev = (headers: Headers): string | undefined => { return headers['atproto-repo-rev'] @@ -72,11 +54,11 @@ export const readAfterWriteInternal = async ( ): Promise<{ data: T; lag?: number }> => { const rev = getRepoRev(res.headers) if (!rev) return { data: res.data } - const localSrvc = ctx.services.local(ctx.db) - const local = await localSrvc.getRecordsSinceRev(requester, rev) - const data = await munge(ctx, res.data, local, requester) - return { - data, - lag: getLocalLag(local), - } + const keypair = await ctx.actorStore.keypair(requester) + return ctx.actorStore.read(requester, async (store) => { + const localViewer = ctx.localViewer(store, keypair) + const local = await localViewer.getRecordsSinceRev(rev) + const data = await munge(localViewer, res.data, local, requester) + return { data, lag: getLocalLag(local) } + }) } diff --git a/packages/pds/src/services/local/index.ts b/packages/pds/src/read-after-write/viewer.ts similarity index 60% rename from packages/pds/src/services/local/index.ts rename to packages/pds/src/read-after-write/viewer.ts index 0dfc2ca9355..a3da0e5eb88 100644 --- a/packages/pds/src/services/local/index.ts +++ b/packages/pds/src/read-after-write/viewer.ts @@ -1,73 +1,88 @@ import util from 'util' import { CID } from 'multiformats/cid' -import { AtUri } from '@atproto/syntax' +import { AtUri, INVALID_HANDLE } from '@atproto/syntax' import { cborToLexRecord } from '@atproto/repo' -import Database from '../../db' -import { Record as PostRecord } from '../../lexicon/types/app/bsky/feed/post' -import { Record as ProfileRecord } from '../../lexicon/types/app/bsky/actor/profile' -import { ids } from '../../lexicon/lexicons' +import { AtpAgent } from '@atproto/api' +import { Keypair } from '@atproto/crypto' +import { createServiceAuthHeaders } from '@atproto/xrpc-server' +import { Record as PostRecord } from '../lexicon/types/app/bsky/feed/post' +import { Record as ProfileRecord } from '../lexicon/types/app/bsky/actor/profile' +import { ids } from '../lexicon/lexicons' import { ProfileViewBasic, ProfileView, ProfileViewDetailed, -} from '../../lexicon/types/app/bsky/actor/defs' -import { FeedViewPost, PostView } from '../../lexicon/types/app/bsky/feed/defs' +} from '../lexicon/types/app/bsky/actor/defs' +import { FeedViewPost, PostView } from '../lexicon/types/app/bsky/feed/defs' import { Main as EmbedImages, isMain as isEmbedImages, -} from '../../lexicon/types/app/bsky/embed/images' +} from '../lexicon/types/app/bsky/embed/images' import { Main as EmbedExternal, isMain as isEmbedExternal, -} from '../../lexicon/types/app/bsky/embed/external' +} from '../lexicon/types/app/bsky/embed/external' import { Main as EmbedRecord, isMain as isEmbedRecord, View as EmbedRecordView, -} from '../../lexicon/types/app/bsky/embed/record' +} from '../lexicon/types/app/bsky/embed/record' import { Main as EmbedRecordWithMedia, isMain as isEmbedRecordWithMedia, -} from '../../lexicon/types/app/bsky/embed/recordWithMedia' -import { AtpAgent } from '@atproto/api' -import { Keypair } from '@atproto/crypto' -import { createServiceAuthHeaders } from '@atproto/xrpc-server' +} from '../lexicon/types/app/bsky/embed/recordWithMedia' +import { ActorStoreReader } from '../actor-store' +import { LocalRecords, RecordDescript } from './types' +import { AccountManager } from '../account-manager' type CommonSignedUris = 'avatar' | 'banner' | 'feed_thumbnail' | 'feed_fullsize' -export class LocalService { - constructor( - public db: Database, - public signingKey: Keypair, - public pdsHostname: string, - public appViewAgent?: AtpAgent, - public appviewDid?: string, - public appviewCdnUrlPattern?: string, - ) {} +export class LocalViewer { + did: string + actorStore: ActorStoreReader + actorKey: Keypair + accountManager: AccountManager + pdsHostname: string + appViewAgent?: AtpAgent + appviewDid?: string + appviewCdnUrlPattern?: string - static creator( - signingKey: Keypair, - pdsHostname: string, - appViewAgent?: AtpAgent, - appviewDid?: string, - appviewCdnUrlPattern?: string, - ) { - return (db: Database) => - new LocalService( - db, - signingKey, - pdsHostname, - appViewAgent, - appviewDid, - appviewCdnUrlPattern, - ) + constructor(params: { + actorStore: ActorStoreReader + actorKey: Keypair + accountManager: AccountManager + pdsHostname: string + appViewAgent?: AtpAgent + appviewDid?: string + appviewCdnUrlPattern?: string + }) { + this.did = params.actorStore.did + this.actorStore = params.actorStore + this.actorKey = params.actorKey + this.accountManager = params.accountManager + this.pdsHostname = params.pdsHostname + this.appViewAgent = params.appViewAgent + this.appviewDid = params.appviewDid + this.appviewCdnUrlPattern = params.appviewCdnUrlPattern + } + + static creator(params: { + accountManager: AccountManager + pdsHostname: string + appViewAgent?: AtpAgent + appviewDid?: string + appviewCdnUrlPattern?: string + }) { + return (actorStore: ActorStoreReader, actorKey: Keypair) => { + return new LocalViewer({ ...params, actorStore, actorKey }) + } } - getImageUrl(pattern: CommonSignedUris, did: string, cid: string) { + getImageUrl(pattern: CommonSignedUris, cid: string) { if (!this.appviewCdnUrlPattern) { - return `https://${this.pdsHostname}/xrpc/${ids.ComAtprotoSyncGetBlob}?did=${did}&cid=${cid}` + return `https://${this.pdsHostname}/xrpc/${ids.ComAtprotoSyncGetBlob}?did=${this.did}&cid=${cid}` } - return util.format(this.appviewCdnUrlPattern, pattern, did, cid) + return util.format(this.appviewCdnUrlPattern, pattern, this.did, cid) } async serviceAuthHeaders(did: string) { @@ -77,28 +92,36 @@ export class LocalService { return createServiceAuthHeaders({ iss: did, aud: this.appviewDid, - keypair: this.signingKey, + keypair: this.actorKey, }) } - async getRecordsSinceRev(did: string, rev: string): Promise { - const res = await this.db.db + async getRecordsSinceRev(rev: string): Promise { + const res = await this.actorStore.db.db .selectFrom('record') - .innerJoin('ipld_block', (join) => - join - .onRef('record.did', '=', 'ipld_block.creator') - .onRef('record.cid', '=', 'ipld_block.cid'), - ) + .innerJoin('repo_block', 'repo_block.cid', 'record.cid') .select([ - 'ipld_block.content', + 'repo_block.content', 'uri', - 'ipld_block.cid', + 'repo_block.cid', 'record.indexedAt', ]) - .where('did', '=', did) .where('record.repoRev', '>', rev) + .limit(10) .orderBy('record.repoRev', 'asc') .execute() + // sanity check to ensure that the clock received is not before _all_ local records (for instance in case of account migration) + if (res.length > 0) { + const sanityCheckRes = await this.actorStore.db.db + .selectFrom('record') + .selectAll() + .where('record.repoRev', '<=', rev) + .limit(1) + .executeTakeFirst() + if (!sanityCheckRes) { + return { profile: null, posts: [] } + } + } return res.reduce( (acc, cur) => { const descript = { @@ -121,30 +144,27 @@ export class LocalService { ) } - async getProfileBasic(did: string): Promise { - const res = await this.db.db - .selectFrom('did_handle') - .leftJoin('record', 'record.did', 'did_handle.did') - .leftJoin('ipld_block', (join) => - join - .onRef('record.did', '=', 'ipld_block.creator') - .onRef('record.cid', '=', 'ipld_block.cid'), - ) - .where('did_handle.did', '=', did) + async getProfileBasic(): Promise { + const profileQuery = this.actorStore.db.db + .selectFrom('record') + .leftJoin('repo_block', 'repo_block.cid', 'record.cid') .where('record.collection', '=', ids.AppBskyActorProfile) .where('record.rkey', '=', 'self') .selectAll() - .executeTakeFirst() - if (!res) return null - const record = res.content - ? (cborToLexRecord(res.content) as ProfileRecord) + const [profileRes, accountRes] = await Promise.all([ + profileQuery.executeTakeFirst(), + this.accountManager.getAccount(this.did), + ]) + if (!accountRes) return null + const record = profileRes?.content + ? (cborToLexRecord(profileRes.content) as ProfileRecord) : null return { - did, - handle: res.handle, + did: this.did, + handle: accountRes.handle ?? INVALID_HANDLE, displayName: record?.displayName, avatar: record?.avatar - ? this.getImageUrl('avatar', did, record.avatar.ref.toString()) + ? this.getImageUrl('avatar', record.avatar.ref.toString()) : undefined, } } @@ -178,7 +198,7 @@ export class LocalService { descript: RecordDescript, ): Promise { const { uri, cid, indexedAt, record } = descript - const author = await this.getProfileBasic(uri.hostname) + const author = await this.getProfileBasic() if (!author) return null const embed = record.embed ? await this.formatPostEmbed(author.did, record) @@ -197,9 +217,9 @@ export class LocalService { const embed = post.embed if (!embed) return null if (isEmbedImages(embed) || isEmbedExternal(embed)) { - return this.formatSimpleEmbed(did, embed) + return this.formatSimpleEmbed(embed) } else if (isEmbedRecord(embed)) { - return this.formatRecordEmbed(did, embed) + return this.formatRecordEmbed(embed) } else if (isEmbedRecordWithMedia(embed)) { return this.formatRecordWithMediaEmbed(did, embed) } else { @@ -207,19 +227,11 @@ export class LocalService { } } - async formatSimpleEmbed(did: string, embed: EmbedImages | EmbedExternal) { + async formatSimpleEmbed(embed: EmbedImages | EmbedExternal) { if (isEmbedImages(embed)) { const images = embed.images.map((img) => ({ - thumb: this.getImageUrl( - 'feed_thumbnail', - did, - img.image.ref.toString(), - ), - fullsize: this.getImageUrl( - 'feed_fullsize', - did, - img.image.ref.toString(), - ), + thumb: this.getImageUrl('feed_thumbnail', img.image.ref.toString()), + fullsize: this.getImageUrl('feed_fullsize', img.image.ref.toString()), aspectRatio: img.aspectRatio, alt: img.alt, })) @@ -236,18 +248,15 @@ export class LocalService { title, description, thumb: thumb - ? this.getImageUrl('feed_thumbnail', did, thumb.ref.toString()) + ? this.getImageUrl('feed_thumbnail', thumb.ref.toString()) : undefined, }, } } } - async formatRecordEmbed( - did: string, - embed: EmbedRecord, - ): Promise { - const view = await this.formatRecordEmbedInternal(did, embed) + async formatRecordEmbed(embed: EmbedRecord): Promise { + const view = await this.formatRecordEmbedInternal(embed) return { $type: 'app.bsky.embed.record#view', record: @@ -260,7 +269,7 @@ export class LocalService { } } - async formatRecordEmbedInternal(did: string, embed: EmbedRecord) { + async formatRecordEmbedInternal(embed: EmbedRecord) { if (!this.appViewAgent || !this.appviewDid) { return null } @@ -270,7 +279,7 @@ export class LocalService { { uris: [embed.record.uri], }, - await this.serviceAuthHeaders(did), + await this.serviceAuthHeaders(this.did), ) const post = res.data.posts[0] if (!post) return null @@ -289,7 +298,7 @@ export class LocalService { { feed: embed.record.uri, }, - await this.serviceAuthHeaders(did), + await this.serviceAuthHeaders(this.did), ) return { $type: 'app.bsaky.feed.defs#generatorView', @@ -300,7 +309,7 @@ export class LocalService { { list: embed.record.uri, }, - await this.serviceAuthHeaders(did), + await this.serviceAuthHeaders(this.did), ) return { $type: 'app.bsaky.graph.defs#listView', @@ -314,8 +323,8 @@ export class LocalService { if (!isEmbedImages(embed.media) && !isEmbedExternal(embed.media)) { return null } - const media = this.formatSimpleEmbed(did, embed.media) - const record = await this.formatRecordEmbed(did, embed.record) + const media = this.formatSimpleEmbed(embed.media) + const record = await this.formatRecordEmbed(embed.record) return { $type: 'app.bsky.embed.recordWithMedia#view', record, @@ -331,7 +340,7 @@ export class LocalService { ...view, displayName: record.displayName, avatar: record.avatar - ? this.getImageUrl('avatar', view.did, record.avatar.ref.toString()) + ? this.getImageUrl('avatar', record.avatar.ref.toString()) : undefined, } } @@ -350,20 +359,8 @@ export class LocalService { return { ...this.updateProfileView(view, record), banner: record.banner - ? this.getImageUrl('banner', view.did, record.banner.ref.toString()) + ? this.getImageUrl('banner', record.banner.ref.toString()) : undefined, } } } - -export type LocalRecords = { - profile: RecordDescript | null - posts: RecordDescript[] -} - -export type RecordDescript = { - uri: AtUri - cid: CID - indexedAt: string - record: T -} diff --git a/packages/pds/src/runtime-flags.ts b/packages/pds/src/runtime-flags.ts deleted file mode 100644 index b4c3437d1f9..00000000000 --- a/packages/pds/src/runtime-flags.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { BailableWait, bailableWait } from '@atproto/common' -import Database from './db' -import { dbLogger as log } from './logger' - -export type FlagName = '' - -export class RuntimeFlags { - destroyed = false - private flags = new Map() - private pollWait: BailableWait | undefined = undefined - - constructor(public db: Database) {} - - async start() { - await this.refresh() - this.poll() - } - - async destroy() { - this.destroyed = true - this.pollWait?.bail() - await this.pollWait?.wait() - } - - get(flag: FlagName) { - return this.flags.get(flag) || null - } - - async refresh() { - const flags = await this.db.db - .selectFrom('runtime_flag') - .selectAll() - .execute() - this.flags = new Map() - for (const flag of flags) { - this.flags.set(flag.name, flag.value) - } - } - - async poll() { - try { - if (this.destroyed) return - await this.refresh() - } catch (err) { - log.error({ err }, 'runtime flags failed to refresh') - } - this.pollWait = bailableWait(5000) - await this.pollWait.wait() - this.poll() - } -} diff --git a/packages/pds/src/sequencer/db/index.ts b/packages/pds/src/sequencer/db/index.ts new file mode 100644 index 00000000000..132e46f5edd --- /dev/null +++ b/packages/pds/src/sequencer/db/index.ts @@ -0,0 +1,21 @@ +import { Database, Migrator } from '../../db' +import { SequencerDbSchema } from './schema' +import migrations from './migrations' + +export * from './schema' + +export type SequencerDb = Database + +export const getDb = ( + location: string, + disableWalAutoCheckpoint = false, +): SequencerDb => { + const pragmas: Record = disableWalAutoCheckpoint + ? { wal_autocheckpoint: '0' } + : {} + return Database.sqlite(location, pragmas) +} + +export const getMigrator = (db: Database) => { + return new Migrator(db.db, migrations) +} diff --git a/packages/pds/src/sequencer/db/migrations/001-init.ts b/packages/pds/src/sequencer/db/migrations/001-init.ts new file mode 100644 index 00000000000..6f5c9c09ded --- /dev/null +++ b/packages/pds/src/sequencer/db/migrations/001-init.ts @@ -0,0 +1,35 @@ +import { Kysely } from 'kysely' + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('repo_seq') + .addColumn('seq', 'integer', (col) => col.autoIncrement().primaryKey()) + .addColumn('did', 'varchar', (col) => col.notNull()) + .addColumn('eventType', 'varchar', (col) => col.notNull()) + .addColumn('event', 'blob', (col) => col.notNull()) + .addColumn('invalidated', 'int2', (col) => col.notNull().defaultTo(0)) + .addColumn('sequencedAt', 'varchar', (col) => col.notNull()) + .execute() + // for filtering seqs based on did + await db.schema + .createIndex('repo_seq_did_idx') + .on('repo_seq') + .column('did') + .execute() + // for filtering seqs based on event type + await db.schema + .createIndex('repo_seq_event_type_idx') + .on('repo_seq') + .column('eventType') + .execute() + // for entering into the seq stream at a particular time + await db.schema + .createIndex('repo_seq_sequenced_at_index') + .on('repo_seq') + .column('sequencedAt') + .execute() +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable('repo_seq').execute() +} diff --git a/packages/pds/src/sequencer/db/migrations/index.ts b/packages/pds/src/sequencer/db/migrations/index.ts new file mode 100644 index 00000000000..4b694f0f0f4 --- /dev/null +++ b/packages/pds/src/sequencer/db/migrations/index.ts @@ -0,0 +1,5 @@ +import * as init from './001-init' + +export default { + '001': init, +} diff --git a/packages/pds/src/sequencer/db/schema.ts b/packages/pds/src/sequencer/db/schema.ts new file mode 100644 index 00000000000..c479c40d884 --- /dev/null +++ b/packages/pds/src/sequencer/db/schema.ts @@ -0,0 +1,24 @@ +import { Generated, GeneratedAlways, Insertable, Selectable } from 'kysely' + +export type RepoSeqEventType = + | 'append' + | 'rebase' + | 'handle' + | 'migrate' + | 'tombstone' + +export interface RepoSeq { + seq: GeneratedAlways + did: string + eventType: RepoSeqEventType + event: Uint8Array + invalidated: Generated<0 | 1> + sequencedAt: string +} + +export type RepoSeqInsert = Insertable +export type RepoSeqEntry = Selectable + +export type SequencerDbSchema = { + repo_seq: RepoSeq +} diff --git a/packages/pds/src/sequencer/events.ts b/packages/pds/src/sequencer/events.ts index eb7bbee5b04..4ba9ab4d06d 100644 --- a/packages/pds/src/sequencer/events.ts +++ b/packages/pds/src/sequencer/events.ts @@ -1,4 +1,3 @@ -import Database from '../db' import { z } from 'zod' import { cborEncode, schema } from '@atproto/common' import { @@ -10,33 +9,7 @@ import { } from '@atproto/repo' import { PreparedWrite } from '../repo' import { CID } from 'multiformats/cid' -import { EventType, RepoSeqInsert } from '../db/tables/repo-seq' - -export const sequenceEvt = async (dbTxn: Database, evt: RepoSeqInsert) => { - dbTxn.assertTransaction() - await dbTxn.notify('new_repo_event') - if (evt.eventType === 'rebase') { - await invalidatePrevRepoOps(dbTxn, evt.did) - } else if (evt.eventType === 'handle') { - await invalidatePrevHandleOps(dbTxn, evt.did) - } - - const res = await dbTxn.db - .insertInto('repo_seq') - .values(evt) - .returning('id') - .executeTakeFirst() - - // since sqlite is serializable, sequence right after insert instead of relying on sequencer-leader - if (res && dbTxn.dialect === 'sqlite') { - await dbTxn.db - .updateTable('repo_seq') - .set({ seq: res.id }) - .where('id', '=', res.id) - .execute() - await dbTxn.notify('outgoing_repo_seq') - } -} +import { RepoSeqInsert } from './db' export const formatSeqCommit = async ( did: string, @@ -122,29 +95,6 @@ export const formatSeqTombstone = async ( } } -export const invalidatePrevSeqEvts = async ( - db: Database, - did: string, - eventTypes: EventType[], -) => { - if (eventTypes.length < 1) return - await db.db - .updateTable('repo_seq') - .where('did', '=', did) - .where('eventType', 'in', eventTypes) - .where('invalidated', '=', 0) - .set({ invalidated: 1 }) - .execute() -} - -export const invalidatePrevRepoOps = async (db: Database, did: string) => { - return invalidatePrevSeqEvts(db, did, ['append', 'rebase']) -} - -export const invalidatePrevHandleOps = async (db: Database, did: string) => { - return invalidatePrevSeqEvts(db, did, ['handle']) -} - export const commitEvtOp = z.object({ action: z.union([ z.literal('create'), diff --git a/packages/pds/src/sequencer/index.ts b/packages/pds/src/sequencer/index.ts index c61c3c8b4b1..5e5e9a6eccd 100644 --- a/packages/pds/src/sequencer/index.ts +++ b/packages/pds/src/sequencer/index.ts @@ -1,4 +1,3 @@ export * from './sequencer' -export * from './sequencer-leader' export * from './outbox' export * from './events' diff --git a/packages/pds/src/sequencer/sequencer-leader.ts b/packages/pds/src/sequencer/sequencer-leader.ts deleted file mode 100644 index 34afbacea13..00000000000 --- a/packages/pds/src/sequencer/sequencer-leader.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { sql } from 'kysely' -import { DisconnectError } from '@atproto/xrpc-server' -import { jitter, wait } from '@atproto/common' -import { Leader } from '../db/leader' -import { seqLogger as log } from '../logger' -import Database from '../db' -import { REPO_SEQ_SEQUENCE } from '../db/tables/repo-seq' -import { countAll } from '../db/util' - -export const SEQUENCER_LEADER_ID = 1100 - -export class SequencerLeader { - leader: Leader - - destroyed = false - polling = false - queued = false - - constructor(public db: Database, lockId = SEQUENCER_LEADER_ID) { - this.leader = new Leader(lockId, this.db) - } - - get isLeader() { - return !!this.leader.session - } - - async run() { - if (this.db.dialect === 'sqlite') return - - while (!this.destroyed) { - try { - const { ran } = await this.leader.run(async ({ signal }) => { - const seqListener = () => { - if (this.polling) { - this.queued = true - } else { - this.polling = true - this.pollDb() - } - } - if (signal.aborted) { - return - } - this.db.channels.new_repo_event.addListener('message', seqListener) - await new Promise((resolve, reject) => { - signal.addEventListener('abort', () => { - this.db.channels.new_repo_event.removeListener( - 'message', - seqListener, - ) - const err = signal.reason - if (!err || err instanceof DisconnectError) { - resolve() - } else { - reject(err) - } - }) - }) - }) - if (ran && !this.destroyed) { - throw new Error( - 'Sequencer leader completed, but should be persistent', - ) - } - } catch (err) { - log.error({ err }, 'sequence leader errored') - } finally { - if (!this.destroyed) { - await wait(1000 + jitter(500)) - } - } - } - } - - async pollDb() { - if (this.destroyed) { - this.polling = false - this.queued = false - return - } - - try { - await this.sequenceOutgoing() - } catch (err) { - log.error({ err }, 'sequencer leader failed to sequence batch') - } finally { - // check if we should continue polling - if (this.queued === false) { - this.polling = false - } else { - this.queued = false - this.pollDb() - } - } - } - - async sequenceOutgoing() { - await this.db.db - .updateTable('repo_seq') - .from((qb) => - qb - .selectFrom('repo_seq') - .select([ - 'id as update_id', - sql`nextval(${sql.literal(REPO_SEQ_SEQUENCE)})`.as( - 'update_seq', - ), - ]) - .where('seq', 'is', null) - .orderBy('id', 'asc') - .as('update'), - ) - .set({ seq: sql`update_seq::bigint` }) - .whereRef('id', '=', 'update_id') - .execute() - - await this.db.notify('outgoing_repo_seq') - } - - async getUnsequencedCount() { - const res = await this.db.db - .selectFrom('repo_seq') - .where('seq', 'is', null) - .select(countAll.as('count')) - .executeTakeFirst() - return res?.count ?? 0 - } - - async isCaughtUp(): Promise { - if (this.db.dialect === 'sqlite') return true - const count = await this.getUnsequencedCount() - return count === 0 - } - - async lastSeq(): Promise { - const res = await this.db.db - .selectFrom('repo_seq') - .select('seq') - .where('seq', 'is not', null) - .orderBy('seq', 'desc') - .limit(1) - .executeTakeFirst() - return res?.seq ?? 0 - } - - destroy() { - this.destroyed = true - this.leader.destroy(new DisconnectError()) - } -} diff --git a/packages/pds/src/sequencer/sequencer.ts b/packages/pds/src/sequencer/sequencer.ts index 23624800a41..7ea23db13f0 100644 --- a/packages/pds/src/sequencer/sequencer.ts +++ b/packages/pds/src/sequencer/sequencer.ts @@ -1,53 +1,80 @@ import EventEmitter from 'events' import TypedEmitter from 'typed-emitter' -import Database from '../db' import { seqLogger as log } from '../logger' -import { RepoSeqEntry } from '../db/tables/repo-seq' -import { cborDecode } from '@atproto/common' -import { CommitEvt, HandleEvt, SeqEvt, TombstoneEvt } from './events' +import { SECOND, cborDecode, wait } from '@atproto/common' +import { CommitData } from '@atproto/repo' +import { + CommitEvt, + HandleEvt, + SeqEvt, + TombstoneEvt, + formatSeqCommit, + formatSeqHandleUpdate, + formatSeqTombstone, +} from './events' +import { + SequencerDb, + getMigrator, + RepoSeqEntry, + RepoSeqInsert, + getDb, +} from './db' +import { PreparedWrite } from '../repo' +import { Crawlers } from '../crawlers' export * from './events' export class Sequencer extends (EventEmitter as new () => SequencerEmitter) { - polling = false - queued = false - - constructor(public db: Database, public lastSeen = 0) { + db: SequencerDb + destroyed = false + pollPromise: Promise | null = null + triesWithNoResults = 0 + + constructor( + dbLocation: string, + public crawlers: Crawlers, + public lastSeen = 0, + disableWalAutoCheckpoint = false, + ) { super() // note: this does not err when surpassed, just prints a warning to stderr this.setMaxListeners(100) + this.db = getDb(dbLocation, disableWalAutoCheckpoint) } async start() { + await this.db.ensureWal() + const migrator = getMigrator(this.db) + await migrator.migrateToLatestOrThrow() const curr = await this.curr() - if (curr) { - this.lastSeen = curr.seq ?? 0 + this.lastSeen = curr ?? 0 + if (this.pollPromise === null) { + this.pollPromise = this.pollDb() } - this.db.channels.outgoing_repo_seq.addListener('message', () => { - if (!this.polling) { - this.pollDb() - } else { - this.queued = true // poll again once current poll completes - } - }) } - async curr(): Promise { + async destroy() { + this.destroyed = true + if (this.pollPromise) { + await this.pollPromise + } + this.emit('close') + } + + async curr(): Promise { const got = await this.db.db .selectFrom('repo_seq') .selectAll() - .where('seq', 'is not', null) .orderBy('seq', 'desc') .limit(1) .executeTakeFirst() - return got || null + return got?.seq ?? null } async next(cursor: number): Promise { const got = await this.db.db .selectFrom('repo_seq') .selectAll() - .where('seq', 'is not', null) .where('seq', '>', cursor) .limit(1) .orderBy('seq', 'asc') @@ -59,7 +86,6 @@ export class Sequencer extends (EventEmitter as new () => SequencerEmitter) { const got = await this.db.db .selectFrom('repo_seq') .selectAll() - .where('seq', 'is not', null) .where('sequencedAt', '>=', time) .orderBy('sequencedAt', 'asc') .limit(1) @@ -79,7 +105,6 @@ export class Sequencer extends (EventEmitter as new () => SequencerEmitter) { .selectFrom('repo_seq') .selectAll() .orderBy('seq', 'asc') - .where('seq', 'is not', null) .where('invalidated', '=', 0) if (earliestSeq !== undefined) { seqQb = seqQb.where('seq', '>', earliestSeq) @@ -133,35 +158,77 @@ export class Sequencer extends (EventEmitter as new () => SequencerEmitter) { return seqEvts } - async pollDb() { + private async pollDb(): Promise { + if (this.destroyed) return + // if already polling, do not start another poll try { - this.polling = true const evts = await this.requestSeqRange({ earliestSeq: this.lastSeen, limit: 1000, }) if (evts.length > 0) { - this.queued = true // should poll again immediately + this.triesWithNoResults = 0 this.emit('events', evts) this.lastSeen = evts.at(-1)?.seq ?? this.lastSeen + } else { + await this.exponentialBackoff() } + this.pollPromise = this.pollDb() } catch (err) { log.error({ err, lastSeen: this.lastSeen }, 'sequencer failed to poll db') - } finally { - this.polling = false - if (this.queued) { - // if queued, poll again - this.queued = false - this.pollDb() - } + await this.exponentialBackoff() + this.pollPromise = this.pollDb() } } + + // when no results, exponential backoff on pulling, with a max of a second wait + private async exponentialBackoff(): Promise { + this.triesWithNoResults++ + const waitTime = Math.min(Math.pow(2, this.triesWithNoResults), SECOND) + await wait(waitTime) + } + + async sequenceEvt(evt: RepoSeqInsert) { + await this.db.executeWithRetry( + this.db.db.insertInto('repo_seq').values(evt), + ) + this.crawlers.notifyOfUpdate() + } + + async sequenceCommit( + did: string, + commitData: CommitData, + writes: PreparedWrite[], + ) { + const evt = await formatSeqCommit(did, commitData, writes) + await this.sequenceEvt(evt) + } + + async sequenceHandleUpdate(did: string, handle: string) { + const evt = await formatSeqHandleUpdate(did, handle) + await this.sequenceEvt(evt) + } + + async sequenceTombstone(did: string) { + const evt = await formatSeqTombstone(did) + await this.sequenceEvt(evt) + } + + async deleteAllForUser(did: string) { + await this.db.executeWithRetry( + this.db.db + .deleteFrom('repo_seq') + .where('did', '=', did) + .where('eventType', '!=', 'tombstone'), + ) + } } type SeqRow = RepoSeqEntry type SequencerEvents = { events: (evts: SeqEvt[]) => void + close: () => void } export type SequencerEmitter = TypedEmitter diff --git a/packages/pds/src/services/account/index.ts b/packages/pds/src/services/account/index.ts deleted file mode 100644 index 2bcf651c2a2..00000000000 --- a/packages/pds/src/services/account/index.ts +++ /dev/null @@ -1,673 +0,0 @@ -import { sql } from 'kysely' -import { randomStr } from '@atproto/crypto' -import { InvalidRequestError } from '@atproto/xrpc-server' -import { MINUTE, lessThanAgoMs } from '@atproto/common' -import { dbLogger as log } from '../../logger' -import Database from '../../db' -import * as scrypt from '../../db/scrypt' -import { UserAccountEntry } from '../../db/tables/user-account' -import { DidHandle } from '../../db/tables/did-handle' -import { RepoRoot } from '../../db/tables/repo-root' -import { - countAll, - isErrUniqueViolation, - notSoftDeletedClause, -} from '../../db/util' -import { paginate, TimeCidKeyset } from '../../db/pagination' -import * as sequencer from '../../sequencer' -import { AppPassword } from '../../lexicon/types/com/atproto/server/createAppPassword' -import { EmailTokenPurpose } from '../../db/tables/email-token' -import { getRandomToken } from '../../api/com/atproto/server/util' -import { AccountView } from '../../lexicon/types/com/atproto/admin/defs' -import { INVALID_HANDLE } from '@atproto/syntax' - -export class AccountService { - constructor(public db: Database) {} - - static creator() { - return (db: Database) => new AccountService(db) - } - - async getAccount( - handleOrDid: string, - includeSoftDeleted = false, - ): Promise<(UserAccountEntry & DidHandle & RepoRoot) | null> { - const { ref } = this.db.db.dynamic - const result = await this.db.db - .selectFrom('user_account') - .innerJoin('did_handle', 'did_handle.did', 'user_account.did') - .innerJoin('repo_root', 'repo_root.did', 'did_handle.did') - .if(!includeSoftDeleted, (qb) => - qb.where(notSoftDeletedClause(ref('repo_root'))), - ) - .where((qb) => { - if (handleOrDid.startsWith('did:')) { - return qb.where('did_handle.did', '=', handleOrDid) - } else { - // lower() is a little hack to avoid using the handle trgm index here, which is slow. not sure why it was preferring - // the handle trgm index over the handle unique index. in any case, we end-up using did_handle_handle_lower_idx instead, which is fast. - return qb.where( - sql`lower(${ref('did_handle.handle')})`, - '=', - handleOrDid, - ) - } - }) - .selectAll('user_account') - .selectAll('did_handle') - .selectAll('repo_root') - .executeTakeFirst() - return result || null - } - - // Repo exists and is not taken-down - async isRepoAvailable(did: string) { - const found = await this.db.db - .selectFrom('repo_root') - .where('did', '=', did) - .where('takedownRef', 'is', null) - .select('did') - .executeTakeFirst() - return found !== undefined - } - - async getAccountByEmail( - email: string, - includeSoftDeleted = false, - ): Promise<(UserAccountEntry & DidHandle & RepoRoot) | null> { - const { ref } = this.db.db.dynamic - const found = await this.db.db - .selectFrom('user_account') - .innerJoin('did_handle', 'did_handle.did', 'user_account.did') - .innerJoin('repo_root', 'repo_root.did', 'did_handle.did') - .if(!includeSoftDeleted, (qb) => - qb.where(notSoftDeletedClause(ref('repo_root'))), - ) - .where('email', '=', email.toLowerCase()) - .selectAll('user_account') - .selectAll('did_handle') - .selectAll('repo_root') - .executeTakeFirst() - return found || null - } - - async getDidForActor( - handleOrDid: string, - includeSoftDeleted = false, - ): Promise { - if (handleOrDid.startsWith('did:')) { - if (includeSoftDeleted) { - return handleOrDid - } - const available = await this.isRepoAvailable(handleOrDid) - return available ? handleOrDid : null - } - const { ref } = this.db.db.dynamic - const found = await this.db.db - .selectFrom('did_handle') - .innerJoin('repo_root', 'repo_root.did', 'did_handle.did') - .if(!includeSoftDeleted, (qb) => - qb.where(notSoftDeletedClause(ref('repo_root'))), - ) - .where('handle', '=', handleOrDid) - .select('did_handle.did') - .executeTakeFirst() - return found ? found.did : null - } - - async registerUser(opts: { - email: string - handle: string - did: string - passwordScrypt: string - }) { - this.db.assertTransaction() - const { email, handle, did, passwordScrypt } = opts - log.debug({ handle, email }, 'registering user') - const registerUserAccnt = this.db.db - .insertInto('user_account') - .values({ - email: email.toLowerCase(), - did, - passwordScrypt, - createdAt: new Date().toISOString(), - }) - .onConflict((oc) => oc.doNothing()) - .returning('did') - .executeTakeFirst() - const registerDidHandle = this.db.db - .insertInto('did_handle') - .values({ did, handle }) - .onConflict((oc) => oc.doNothing()) - .returning('handle') - .executeTakeFirst() - - const [res1, res2] = await Promise.all([ - registerUserAccnt, - registerDidHandle, - ]) - if (!res1 || !res2) { - throw new UserAlreadyExistsError() - } - log.info({ handle, email, did }, 'registered user') - } - - // @NOTE should always be paired with a sequenceHandle(). - // the token output from this method should be passed to sequenceHandle(). - async updateHandle( - did: string, - handle: string, - ): Promise { - const res = await this.db.db - .updateTable('did_handle') - .set({ handle }) - .where('did', '=', did) - .whereNotExists( - // @NOTE see also condition in isHandleAvailable() - this.db.db - .selectFrom('did_handle') - .where('handle', '=', handle) - .selectAll(), - ) - .executeTakeFirst() - if (res.numUpdatedRows < 1) { - throw new UserAlreadyExistsError() - } - return { did, handle } - } - - async sequenceHandle(tok: HandleSequenceToken) { - this.db.assertTransaction() - const seqEvt = await sequencer.formatSeqHandleUpdate(tok.did, tok.handle) - await sequencer.sequenceEvt(this.db, seqEvt) - } - - async getHandleDid(handle: string): Promise { - // @NOTE see also condition in updateHandle() - const found = await this.db.db - .selectFrom('did_handle') - .where('handle', '=', handle) - .selectAll() - .executeTakeFirst() - return found?.did ?? null - } - - async updateEmail(did: string, email: string) { - try { - await this.db.db - .updateTable('user_account') - .set({ email: email.toLowerCase(), emailConfirmedAt: null }) - .where('did', '=', did) - .executeTakeFirst() - } catch (err) { - if (isErrUniqueViolation(err)) { - throw new UserAlreadyExistsError() - } else { - throw err - } - } - } - - async updateUserPassword(did: string, password: string) { - const passwordScrypt = await scrypt.genSaltAndHash(password) - await this.db.db - .updateTable('user_account') - .set({ passwordScrypt }) - .where('did', '=', did) - .execute() - } - - async createAppPassword(did: string, name: string): Promise { - // create an app password with format: - // 1234-abcd-5678-efgh - const str = randomStr(16, 'base32').slice(0, 16) - const chunks = [ - str.slice(0, 4), - str.slice(4, 8), - str.slice(8, 12), - str.slice(12, 16), - ] - const password = chunks.join('-') - const passwordScrypt = await scrypt.hashAppPassword(did, password) - const got = await this.db.db - .insertInto('app_password') - .values({ - did, - name, - passwordScrypt, - createdAt: new Date().toISOString(), - }) - .returningAll() - .executeTakeFirst() - if (!got) { - throw new InvalidRequestError('could not create app-specific password') - } - return { - name, - password, - createdAt: got.createdAt, - } - } - - async deleteAppPassword(did: string, name: string) { - await this.db.db - .deleteFrom('app_password') - .where('did', '=', did) - .where('name', '=', name) - .execute() - } - - async verifyAccountPassword(did: string, password: string): Promise { - const found = await this.db.db - .selectFrom('user_account') - .selectAll() - .where('did', '=', did) - .executeTakeFirst() - return found ? await scrypt.verify(password, found.passwordScrypt) : false - } - - async verifyAppPassword( - did: string, - password: string, - ): Promise { - const passwordScrypt = await scrypt.hashAppPassword(did, password) - const found = await this.db.db - .selectFrom('app_password') - .selectAll() - .where('did', '=', did) - .where('passwordScrypt', '=', passwordScrypt) - .executeTakeFirst() - return found?.name ?? null - } - - async listAppPasswords( - did: string, - ): Promise<{ name: string; createdAt: string }[]> { - return this.db.db - .selectFrom('app_password') - .select(['name', 'createdAt']) - .where('did', '=', did) - .execute() - } - - async search(opts: { - query: string - limit: number - cursor?: string - includeSoftDeleted?: boolean - }): Promise<(RepoRoot & DidHandle)[]> { - const { query, limit, cursor, includeSoftDeleted } = opts - const { ref } = this.db.db.dynamic - - const builder = this.db.db - .selectFrom('did_handle') - .innerJoin('repo_root', 'repo_root.did', 'did_handle.did') - .innerJoin('user_account', 'user_account.did', 'did_handle.did') - .if(!includeSoftDeleted, (qb) => - qb.where(notSoftDeletedClause(ref('repo_root'))), - ) - .where((qb) => { - // sqlite doesn't support "ilike", but performs "like" case-insensitively - const likeOp = this.db.dialect === 'pg' ? 'ilike' : 'like' - if (query.includes('@')) { - return qb.where('user_account.email', likeOp, `%${query}%`) - } - if (query.startsWith('did:')) { - return qb.where('did_handle.did', '=', query) - } - return qb.where('did_handle.handle', likeOp, `${query}%`) - }) - .selectAll(['did_handle', 'repo_root']) - - const keyset = new ListKeyset( - ref('repo_root.indexedAt'), - ref('did_handle.handle'), - ) - - return await paginate(builder, { - limit, - cursor, - keyset, - }).execute() - } - - async list(opts: { - limit: number - cursor?: string - includeSoftDeleted?: boolean - invitedBy?: string - }): Promise<(RepoRoot & DidHandle)[]> { - const { limit, cursor, includeSoftDeleted, invitedBy } = opts - const { ref } = this.db.db.dynamic - - let builder = this.db.db - .selectFrom('repo_root') - .innerJoin('did_handle', 'did_handle.did', 'repo_root.did') - .if(!includeSoftDeleted, (qb) => - qb.where(notSoftDeletedClause(ref('repo_root'))), - ) - .selectAll('did_handle') - .selectAll('repo_root') - - if (invitedBy) { - builder = builder - .innerJoin( - 'invite_code_use as code_use', - 'code_use.usedBy', - 'did_handle.did', - ) - .innerJoin('invite_code', 'invite_code.code', 'code_use.code') - .where('invite_code.forUser', '=', invitedBy) - } - - const keyset = new ListKeyset(ref('indexedAt'), ref('handle')) - - return await paginate(builder, { - limit, - cursor, - keyset, - }).execute() - } - - async deleteAccount(did: string): Promise { - // Not done in transaction because it would be too long, prone to contention. - // Also, this can safely be run multiple times if it fails. - await this.db.db - .deleteFrom('refresh_token') - .where('did', '=', did) - .execute() - await this.db.db - .deleteFrom('user_account') - .where('user_account.did', '=', did) - .execute() - await this.db.db - .deleteFrom('did_handle') - .where('did_handle.did', '=', did) - .execute() - const seqEvt = await sequencer.formatSeqTombstone(did) - await this.db.transaction(async (txn) => { - await sequencer.sequenceEvt(txn, seqEvt) - }) - } - - async adminView(did: string): Promise { - const accountQb = this.db.db - .selectFrom('did_handle') - .innerJoin('user_account', 'user_account.did', 'did_handle.did') - .where('did_handle.did', '=', did) - .select([ - 'did_handle.did', - 'did_handle.handle', - 'user_account.email', - 'user_account.emailConfirmedAt', - 'user_account.invitesDisabled', - 'user_account.inviteNote', - 'user_account.createdAt as indexedAt', - ]) - - const [account, invites, invitedBy] = await Promise.all([ - accountQb.executeTakeFirst(), - this.getAccountInviteCodes(did), - this.getInvitedByForAccounts([did]), - ]) - - if (!account) return null - - return { - ...account, - handle: account?.handle ?? INVALID_HANDLE, - invitesDisabled: account.invitesDisabled === 1, - inviteNote: account.inviteNote ?? undefined, - emailConfirmedAt: account.emailConfirmedAt ?? undefined, - invites, - invitedBy: invitedBy[did], - } - } - - selectInviteCodesQb() { - const ref = this.db.db.dynamic.ref - const builder = this.db.db - .selectFrom('invite_code') - .select([ - 'invite_code.code as code', - 'invite_code.availableUses as available', - 'invite_code.disabled as disabled', - 'invite_code.forUser as forAccount', - 'invite_code.createdBy as createdBy', - 'invite_code.createdAt as createdAt', - this.db.db - .selectFrom('invite_code_use') - .select(countAll.as('count')) - .whereRef('invite_code_use.code', '=', ref('invite_code.code')) - .as('uses'), - ]) - return this.db.db.selectFrom(builder.as('codes')).selectAll() - } - - async getInviteCodesUses( - codes: string[], - ): Promise> { - const uses: Record = {} - if (codes.length > 0) { - const usesRes = await this.db.db - .selectFrom('invite_code_use') - .where('code', 'in', codes) - .selectAll() - .execute() - for (const use of usesRes) { - const { code, usedBy, usedAt } = use - uses[code] ??= [] - uses[code].push({ usedBy, usedAt }) - } - } - return uses - } - - async getAccountInviteCodes(did: string): Promise { - const res = await this.selectInviteCodesQb() - .where('forAccount', '=', did) - .execute() - const codes = res.map((row) => row.code) - const uses = await this.getInviteCodesUses(codes) - return res.map((row) => ({ - ...row, - uses: uses[row.code] ?? [], - disabled: row.disabled === 1, - })) - } - - async getInvitedByForAccounts( - dids: string[], - ): Promise> { - if (dids.length < 1) return {} - const codeDetailsRes = await this.selectInviteCodesQb() - .where('code', 'in', (qb) => - qb - .selectFrom('invite_code_use') - .where('usedBy', 'in', dids) - .select('code') - .distinct(), - ) - .execute() - const uses = await this.getInviteCodesUses( - codeDetailsRes.map((row) => row.code), - ) - const codeDetails = codeDetailsRes.map((row) => ({ - ...row, - uses: uses[row.code] ?? [], - disabled: row.disabled === 1, - })) - return codeDetails.reduce((acc, cur) => { - for (const use of cur.uses) { - acc[use.usedBy] = cur - } - return acc - }, {} as Record) - } - - async createEmailToken( - did: string, - purpose: EmailTokenPurpose, - ): Promise { - const token = getRandomToken().toUpperCase() - await this.db.db - .insertInto('email_token') - .values({ purpose, did, token, requestedAt: new Date() }) - .onConflict((oc) => - oc - .columns(['purpose', 'did']) - .doUpdateSet({ token, requestedAt: new Date() }), - ) - .execute() - return token - } - - async deleteEmailToken(did: string, purpose: EmailTokenPurpose) { - await this.db.db - .deleteFrom('email_token') - .where('did', '=', did) - .where('purpose', '=', purpose) - .executeTakeFirst() - } - - async assertValidToken( - did: string, - purpose: EmailTokenPurpose, - token: string, - expirationLen = 15 * MINUTE, - ) { - const res = await this.db.db - .selectFrom('email_token') - .selectAll() - .where('purpose', '=', purpose) - .where('did', '=', did) - .where('token', '=', token.toUpperCase()) - .executeTakeFirst() - if (!res) { - throw new InvalidRequestError('Token is invalid', 'InvalidToken') - } - const expired = !lessThanAgoMs(res.requestedAt, expirationLen) - if (expired) { - throw new InvalidRequestError('Token is expired', 'ExpiredToken') - } - } - - async assertValidTokenAndFindDid( - purpose: EmailTokenPurpose, - token: string, - expirationLen = 15 * MINUTE, - ): Promise { - const res = await this.db.db - .selectFrom('email_token') - .selectAll() - .where('purpose', '=', purpose) - .where('token', '=', token.toUpperCase()) - .executeTakeFirst() - if (!res) { - throw new InvalidRequestError('Token is invalid', 'InvalidToken') - } - const expired = !lessThanAgoMs(res.requestedAt, expirationLen) - if (expired) { - throw new InvalidRequestError('Token is expired', 'ExpiredToken') - } - return res.did - } - - async getPreferences( - did: string, - namespace?: string, - ): Promise { - const prefsRes = await this.db.db - .selectFrom('user_pref') - .where('did', '=', did) - .orderBy('id') - .selectAll() - .execute() - return prefsRes - .filter((pref) => !namespace || matchNamespace(namespace, pref.name)) - .map((pref) => JSON.parse(pref.valueJson)) - } - - async putPreferences( - did: string, - values: UserPreference[], - namespace: string, - ): Promise { - this.db.assertTransaction() - if (!values.every((value) => matchNamespace(namespace, value.$type))) { - throw new InvalidRequestError( - `Some preferences are not in the ${namespace} namespace`, - ) - } - // short-held row lock to prevent races - if (this.db.dialect === 'pg') { - await this.db.db - .selectFrom('user_account') - .selectAll() - .forUpdate() - .where('did', '=', did) - .executeTakeFirst() - } - // get all current prefs for user and prep new pref rows - const allPrefs = await this.db.db - .selectFrom('user_pref') - .where('did', '=', did) - .select(['id', 'name']) - .execute() - const putPrefs = values.map((value) => { - return { - did, - name: value.$type, - valueJson: JSON.stringify(value), - } - }) - const allPrefIdsInNamespace = allPrefs - .filter((pref) => matchNamespace(namespace, pref.name)) - .map((pref) => pref.id) - // replace all prefs in given namespace - if (allPrefIdsInNamespace.length) { - await this.db.db - .deleteFrom('user_pref') - .where('did', '=', did) - .where('id', 'in', allPrefIdsInNamespace) - .execute() - } - if (putPrefs.length) { - await this.db.db.insertInto('user_pref').values(putPrefs).execute() - } - } -} - -export type UserPreference = Record & { $type: string } - -export type CodeDetail = { - code: string - available: number - disabled: boolean - forAccount: string - createdBy: string - createdAt: string - uses: CodeUse[] -} - -type CodeUse = { - usedBy: string - usedAt: string -} - -export class UserAlreadyExistsError extends Error {} - -export class ListKeyset extends TimeCidKeyset<{ - indexedAt: string - handle: string // handles are treated identically to cids in TimeCidKeyset -}> { - labelResult(result: { indexedAt: string; handle: string }) { - return { primary: result.indexedAt, secondary: result.handle } - } -} - -const matchNamespace = (namespace: string, fullname: string) => { - return fullname === namespace || fullname.startsWith(`${namespace}.`) -} - -export type HandleSequenceToken = { did: string; handle: string } diff --git a/packages/pds/src/services/auth.ts b/packages/pds/src/services/auth.ts deleted file mode 100644 index 9e21cb9a629..00000000000 --- a/packages/pds/src/services/auth.ts +++ /dev/null @@ -1,175 +0,0 @@ -import * as jwt from 'jsonwebtoken' -import * as ui8 from 'uint8arrays' -import * as crypto from '@atproto/crypto' -import { HOUR } from '@atproto/common' -import Database from '../db' -import { AuthScope } from '../auth-verifier' - -const REFRESH_GRACE_MS = 2 * HOUR - -export type AuthToken = { - scope: AuthScope - sub: string - exp: number -} - -export type RefreshToken = AuthToken & { jti: string } - -export class AuthService { - constructor(public db: Database, private _secret: string) {} - - static creator(jwtSecret: string) { - return (db: Database) => new AuthService(db, jwtSecret) - } - - createAccessToken(opts: { - did: string - scope?: AuthScope - expiresIn?: string | number - }) { - const { did, scope = AuthScope.Access, expiresIn = '120mins' } = opts - const payload = { - scope, - sub: did, - } - return { - payload: payload as AuthToken, // exp set by sign() - jwt: jwt.sign(payload, this._secret, { - expiresIn: expiresIn, - mutatePayload: true, - }), - } - } - - createRefreshToken(opts: { - did: string - jti?: string - expiresIn?: string | number - }) { - const { did, jti = getRefreshTokenId(), expiresIn = '90days' } = opts - const payload = { - scope: AuthScope.Refresh, - sub: did, - jti, - } - return { - payload: payload as RefreshToken, // exp set by sign() - jwt: jwt.sign(payload, this._secret, { - expiresIn: expiresIn, - mutatePayload: true, - }), - } - } - - async createSession(did: string, appPasswordName: string | null) { - const access = this.createAccessToken({ - did, - scope: appPasswordName === null ? AuthScope.Access : AuthScope.AppPass, - }) - const refresh = this.createRefreshToken({ did }) - await this.storeRefreshToken(refresh.payload, appPasswordName) - return { access, refresh } - } - - async storeRefreshToken( - payload: RefreshToken, - appPasswordName: string | null, - ) { - return this.db.db - .insertInto('refresh_token') - .values({ - id: payload.jti, - did: payload.sub, - appPasswordName, - expiresAt: new Date(payload.exp * 1000).toISOString(), - }) - .onConflict((oc) => oc.doNothing()) // E.g. when re-granting during a refresh grace period - .executeTakeFirst() - } - - async rotateRefreshToken(id: string) { - this.db.assertTransaction() - const token = await this.db.db - .selectFrom('refresh_token') - .if(this.db.dialect !== 'sqlite', (qb) => qb.forUpdate()) - .where('id', '=', id) - .selectAll() - .executeTakeFirst() - if (!token) return null - - // take the chance to tidy all of a user's expired tokens - const now = new Date() - await this.db.db - .deleteFrom('refresh_token') - .where('did', '=', token.did) - .where('expiresAt', '<=', now.toISOString()) - .returningAll() - .executeTakeFirst() - - // Shorten the refresh token lifespan down from its - // original expiration time to its revocation grace period. - const prevExpiresAt = new Date(token.expiresAt) - const graceExpiresAt = new Date(now.getTime() + REFRESH_GRACE_MS) - - const expiresAt = - graceExpiresAt < prevExpiresAt ? graceExpiresAt : prevExpiresAt - - if (expiresAt <= now) { - return null - } - - // Determine the next refresh token id: upon refresh token - // reuse you always receive a refresh token with the same id. - const nextId = token.nextId ?? getRefreshTokenId() - - // Update token w/ possibly-updated expiration time - // and next id, and tidy all of user's expired tokens. - await this.db.db - .updateTable('refresh_token') - .where('id', '=', id) - .set({ expiresAt: expiresAt.toISOString(), nextId }) - .executeTakeFirst() - - const refresh = this.createRefreshToken({ - did: token.did, - jti: nextId, - }) - const access = this.createAccessToken({ - did: token.did, - scope: - token.appPasswordName === null ? AuthScope.Access : AuthScope.AppPass, - }) - await this.storeRefreshToken(refresh.payload, token.appPasswordName) - - return { access, refresh } - } - - async revokeRefreshToken(id: string) { - const { numDeletedRows } = await this.db.db - .deleteFrom('refresh_token') - .where('id', '=', id) - .executeTakeFirst() - return numDeletedRows > 0 - } - - async revokeRefreshTokensByDid(did: string) { - const { numDeletedRows } = await this.db.db - .deleteFrom('refresh_token') - .where('did', '=', did) - .executeTakeFirst() - return numDeletedRows > 0 - } - - async revokeAppPasswordRefreshToken(did: string, appPassName: string) { - const { numDeletedRows } = await this.db.db - .deleteFrom('refresh_token') - .where('did', '=', did) - .where('appPasswordName', '=', appPassName) - .executeTakeFirst() - return numDeletedRows > 0 - } -} - -const getRefreshTokenId = () => { - return ui8.toString(crypto.randomBytes(32), 'base64') -} diff --git a/packages/pds/src/services/index.ts b/packages/pds/src/services/index.ts deleted file mode 100644 index 5289e30e7e8..00000000000 --- a/packages/pds/src/services/index.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { AtpAgent } from '@atproto/api' -import * as crypto from '@atproto/crypto' -import { BlobStore } from '@atproto/repo' -import Database from '../db' -import { AccountService } from './account' -import { AuthService } from './auth' -import { RecordService } from './record' -import { RepoService } from './repo' -import { ModerationService } from './moderation' -import { BackgroundQueue } from '../background' -import { Crawlers } from '../crawlers' -import { LocalService } from './local' - -export function createServices(resources: { - repoSigningKey: crypto.Keypair - blobstore: BlobStore - pdsHostname: string - jwtSecret: string - appViewAgent?: AtpAgent - appViewDid?: string - appViewCdnUrlPattern?: string - backgroundQueue: BackgroundQueue - crawlers: Crawlers -}): Services { - const { - repoSigningKey, - blobstore, - pdsHostname, - jwtSecret, - appViewAgent, - appViewDid, - appViewCdnUrlPattern, - backgroundQueue, - crawlers, - } = resources - return { - account: AccountService.creator(), - auth: AuthService.creator(jwtSecret), - record: RecordService.creator(), - repo: RepoService.creator( - repoSigningKey, - blobstore, - backgroundQueue, - crawlers, - ), - local: LocalService.creator( - repoSigningKey, - pdsHostname, - appViewAgent, - appViewDid, - appViewCdnUrlPattern, - ), - moderation: ModerationService.creator(blobstore), - } -} - -export type Services = { - account: FromDb - auth: FromDb - record: FromDb - repo: FromDb - local: FromDb - moderation: FromDb -} - -type FromDb = (db: Database) => T diff --git a/packages/pds/src/services/moderation/index.ts b/packages/pds/src/services/moderation/index.ts deleted file mode 100644 index 240a95004d2..00000000000 --- a/packages/pds/src/services/moderation/index.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { CID } from 'multiformats/cid' -import { BlobStore } from '@atproto/repo' -import { AtUri } from '@atproto/syntax' -import Database from '../../db' -import { - RepoBlobRef, - RepoRef, - StatusAttr, -} from '../../lexicon/types/com/atproto/admin/defs' -import { Main as StrongRef } from '../../lexicon/types/com/atproto/repo/strongRef' - -export class ModerationService { - constructor(public db: Database, public blobstore: BlobStore) {} - - static creator(blobstore: BlobStore) { - return (db: Database) => new ModerationService(db, blobstore) - } - - async getRepoTakedownState( - did: string, - ): Promise | null> { - const res = await this.db.db - .selectFrom('repo_root') - .select('takedownRef') - .where('did', '=', did) - .executeTakeFirst() - if (!res) return null - const state = takedownRefToStatus(res.takedownRef ?? null) - return { - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: did, - }, - takedown: state, - } - } - - async getRecordTakedownState( - uri: AtUri, - ): Promise | null> { - const res = await this.db.db - .selectFrom('record') - .select(['takedownRef', 'cid']) - .where('uri', '=', uri.toString()) - .executeTakeFirst() - if (!res) return null - const state = takedownRefToStatus(res.takedownRef ?? null) - return { - subject: { - $type: 'com.atproto.repo.strongRef', - uri: uri.toString(), - cid: res.cid, - }, - takedown: state, - } - } - - async getBlobTakedownState( - did: string, - cid: CID, - ): Promise | null> { - const res = await this.db.db - .selectFrom('repo_blob') - .select('takedownRef') - .where('did', '=', did) - .where('cid', '=', cid.toString()) - .executeTakeFirst() - if (!res) return null - const state = takedownRefToStatus(res.takedownRef ?? null) - return { - subject: { - $type: 'com.atproto.admin.defs#repoBlobRef', - did: did, - cid: cid.toString(), - }, - takedown: state, - } - } - - async updateRepoTakedownState(did: string, takedown: StatusAttr) { - const takedownRef = statusTotakedownRef(takedown) - await this.db.db - .updateTable('repo_root') - .set({ takedownRef }) - .where('did', '=', did) - .execute() - } - - async updateRecordTakedownState(uri: AtUri, takedown: StatusAttr) { - const takedownRef = statusTotakedownRef(takedown) - await this.db.db - .updateTable('record') - .set({ takedownRef }) - .where('uri', '=', uri.toString()) - .execute() - } - - async updateBlobTakedownState(did: string, blob: CID, takedown: StatusAttr) { - const takedownRef = statusTotakedownRef(takedown) - await this.db.db - .updateTable('repo_blob') - .set({ takedownRef }) - .where('did', '=', did) - .where('cid', '=', blob.toString()) - .execute() - if (takedown.applied) { - await this.blobstore.quarantine(blob) - } else { - await this.blobstore.unquarantine(blob) - } - } -} - -type StatusResponse = { - subject: T - takedown: StatusAttr -} - -const takedownRefToStatus = (id: string | null): StatusAttr => { - return id === null ? { applied: false } : { applied: true, ref: id } -} - -const statusTotakedownRef = (state: StatusAttr): string | null => { - return state.applied ? state.ref ?? new Date().toISOString() : null -} diff --git a/packages/pds/src/services/repo/index.ts b/packages/pds/src/services/repo/index.ts deleted file mode 100644 index 406635b736d..00000000000 --- a/packages/pds/src/services/repo/index.ts +++ /dev/null @@ -1,305 +0,0 @@ -import { CID } from 'multiformats/cid' -import * as crypto from '@atproto/crypto' -import { BlobStore, CommitData, Repo, WriteOpAction } from '@atproto/repo' -import { InvalidRequestError } from '@atproto/xrpc-server' -import { AtUri } from '@atproto/syntax' -import Database from '../../db' -import SqlRepoStorage from '../../sql-repo-storage' -import { - BadCommitSwapError, - BadRecordSwapError, - PreparedCreate, - PreparedWrite, -} from '../../repo/types' -import { RepoBlobs } from './blobs' -import { createWriteToOp, writeToOp } from '../../repo' -import { RecordService } from '../record' -import * as sequencer from '../../sequencer' -import { wait } from '@atproto/common' -import { BackgroundQueue } from '../../background' -import { Crawlers } from '../../crawlers' - -export class RepoService { - blobs: RepoBlobs - - constructor( - public db: Database, - public repoSigningKey: crypto.Keypair, - public blobstore: BlobStore, - public backgroundQueue: BackgroundQueue, - public crawlers: Crawlers, - ) { - this.blobs = new RepoBlobs(db, blobstore, backgroundQueue) - } - - static creator( - keypair: crypto.Keypair, - blobstore: BlobStore, - backgroundQueue: BackgroundQueue, - crawlers: Crawlers, - ) { - return (db: Database) => - new RepoService(db, keypair, blobstore, backgroundQueue, crawlers) - } - - services = { - record: RecordService.creator(), - } - - private async serviceTx( - fn: (srvc: RepoService) => Promise, - ): Promise { - this.db.assertNotTransaction() - return this.db.transaction((dbTxn) => { - const srvc = new RepoService( - dbTxn, - this.repoSigningKey, - this.blobstore, - this.backgroundQueue, - this.crawlers, - ) - return fn(srvc) - }) - } - - async createRepo(did: string, writes: PreparedCreate[], now: string) { - this.db.assertTransaction() - const storage = new SqlRepoStorage(this.db, did, now) - const writeOps = writes.map(createWriteToOp) - const commit = await Repo.formatInitCommit( - storage, - did, - this.repoSigningKey, - writeOps, - ) - await Promise.all([ - storage.applyCommit(commit), - this.indexWrites(writes, now), - this.blobs.processWriteBlobs(did, commit.rev, writes), - ]) - await this.afterWriteProcessing(did, commit, writes) - } - - async processCommit( - did: string, - writes: PreparedWrite[], - commitData: CommitData, - now: string, - ) { - this.db.assertTransaction() - const storage = new SqlRepoStorage(this.db, did, now) - const obtained = await storage.lockRepo() - if (!obtained) { - throw new ConcurrentWriteError() - } - await Promise.all([ - // persist the commit to repo storage - storage.applyCommit(commitData), - // & send to indexing - this.indexWrites(writes, now, commitData.rev), - // process blobs - this.blobs.processWriteBlobs(did, commitData.rev, writes), - // do any other processing needed after write - ]) - await this.afterWriteProcessing(did, commitData, writes) - } - - async processWrites( - toWrite: { did: string; writes: PreparedWrite[]; swapCommitCid?: CID }, - times: number, - timeout = 100, - prevStorage?: SqlRepoStorage, - ) { - this.db.assertNotTransaction() - const { did, writes, swapCommitCid } = toWrite - // we may have some useful cached blocks in the storage, so re-use the previous instance - const storage = prevStorage ?? new SqlRepoStorage(this.db, did) - try { - const commit = await this.formatCommit( - storage, - did, - writes, - swapCommitCid, - ) - await this.serviceTx(async (srvcTx) => - srvcTx.processCommit(did, writes, commit, new Date().toISOString()), - ) - } catch (err) { - if (err instanceof ConcurrentWriteError) { - if (times <= 1) { - throw err - } - await wait(timeout) - return this.processWrites(toWrite, times - 1, timeout, storage) - } else { - throw err - } - } - } - - async formatCommit( - storage: SqlRepoStorage, - did: string, - writes: PreparedWrite[], - swapCommit?: CID, - ): Promise { - // this is not in a txn, so this won't actually hold the lock, - // we just check if it is currently held by another txn - const available = await storage.lockAvailable() - if (!available) { - throw new ConcurrentWriteError() - } - const currRoot = await storage.getRootDetailed() - if (!currRoot) { - throw new InvalidRequestError( - `${did} is not a registered repo on this server`, - ) - } - if (swapCommit && !currRoot.cid.equals(swapCommit)) { - throw new BadCommitSwapError(currRoot.cid) - } - // cache last commit since there's likely overlap - await storage.cacheRev(currRoot.rev) - const recordTxn = this.services.record(this.db) - const newRecordCids: CID[] = [] - const delAndUpdateUris: AtUri[] = [] - for (const write of writes) { - const { action, uri, swapCid } = write - if (action !== WriteOpAction.Delete) { - newRecordCids.push(write.cid) - } - if (action !== WriteOpAction.Create) { - delAndUpdateUris.push(uri) - } - if (swapCid === undefined) { - continue - } - const record = await recordTxn.getRecord(uri, null, true) - const currRecord = record && CID.parse(record.cid) - if (action === WriteOpAction.Create && swapCid !== null) { - throw new BadRecordSwapError(currRecord) // There should be no current record for a create - } - if (action === WriteOpAction.Update && swapCid === null) { - throw new BadRecordSwapError(currRecord) // There should be a current record for an update - } - if (action === WriteOpAction.Delete && swapCid === null) { - throw new BadRecordSwapError(currRecord) // There should be a current record for a delete - } - if ((currRecord || swapCid) && !currRecord?.equals(swapCid)) { - throw new BadRecordSwapError(currRecord) - } - } - - let commit: CommitData - try { - const repo = await Repo.load(storage, currRoot.cid) - const writeOps = writes.map(writeToOp) - commit = await repo.formatCommit(writeOps, this.repoSigningKey) - } catch (err) { - // if an error occurs, check if it is attributable to a concurrent write - const curr = await storage.getRoot() - if (!currRoot.cid.equals(curr)) { - throw new ConcurrentWriteError() - } else { - throw err - } - } - - // find blocks that would be deleted but are referenced by another record - const dupeRecordCids = await this.getDuplicateRecordCids( - did, - commit.removedCids.toList(), - delAndUpdateUris, - ) - for (const cid of dupeRecordCids) { - commit.removedCids.delete(cid) - } - - // find blocks that are relevant to ops but not included in diff - // (for instance a record that was moved but cid stayed the same) - const newRecordBlocks = commit.newBlocks.getMany(newRecordCids) - if (newRecordBlocks.missing.length > 0) { - const missingBlocks = await storage.getBlocks(newRecordBlocks.missing) - commit.newBlocks.addMap(missingBlocks.blocks) - } - return commit - } - - async indexWrites(writes: PreparedWrite[], now: string, rev?: string) { - this.db.assertTransaction() - const recordTxn = this.services.record(this.db) - await Promise.all( - writes.map(async (write) => { - if ( - write.action === WriteOpAction.Create || - write.action === WriteOpAction.Update - ) { - await recordTxn.indexRecord( - write.uri, - write.cid, - write.record, - write.action, - rev, - now, - ) - } else if (write.action === WriteOpAction.Delete) { - await recordTxn.deleteRecord(write.uri) - } - }), - ) - } - - async getDuplicateRecordCids( - did: string, - cids: CID[], - touchedUris: AtUri[], - ): Promise { - if (touchedUris.length === 0 || cids.length === 0) { - return [] - } - const cidStrs = cids.map((c) => c.toString()) - const uriStrs = touchedUris.map((u) => u.toString()) - const res = await this.db.db - .selectFrom('record') - .where('did', '=', did) - .where('cid', 'in', cidStrs) - .where('uri', 'not in', uriStrs) - .select('cid') - .execute() - return res.map((row) => CID.parse(row.cid)) - } - - async afterWriteProcessing( - did: string, - commitData: CommitData, - writes: PreparedWrite[], - ) { - this.db.onCommit(() => { - this.backgroundQueue.add(async () => { - await this.crawlers.notifyOfUpdate() - }) - }) - - const seqEvt = await sequencer.formatSeqCommit(did, commitData, writes) - await sequencer.sequenceEvt(this.db, seqEvt) - } - - async deleteRepo(did: string) { - // Not done in transaction because it would be too long, prone to contention. - // Also, this can safely be run multiple times if it fails. - // delete all blocks from this did & no other did - await this.db.db.deleteFrom('repo_root').where('did', '=', did).execute() - await this.db.db.deleteFrom('repo_seq').where('did', '=', did).execute() - await this.db.db - .deleteFrom('ipld_block') - .where('creator', '=', did) - .execute() - await this.blobs.deleteForUser(did) - } -} - -export class ConcurrentWriteError extends Error { - constructor() { - super('too many concurrent writes') - } -} diff --git a/packages/pds/src/sql-repo-storage.ts b/packages/pds/src/sql-repo-storage.ts deleted file mode 100644 index 13301ae300f..00000000000 --- a/packages/pds/src/sql-repo-storage.ts +++ /dev/null @@ -1,286 +0,0 @@ -import { - CommitData, - RepoStorage, - BlockMap, - CidSet, - ReadableBlockstore, - writeCarStream, -} from '@atproto/repo' -import { chunkArray } from '@atproto/common' -import { CID } from 'multiformats/cid' -import Database from './db' -import { IpldBlock } from './db/tables/ipld-block' -import { ConcurrentWriteError } from './services/repo' -import { sql } from 'kysely' - -export class SqlRepoStorage extends ReadableBlockstore implements RepoStorage { - cache: BlockMap = new BlockMap() - - constructor( - public db: Database, - public did: string, - public timestamp?: string, - ) { - super() - } - - async lockRepo(): Promise { - if (this.db.dialect === 'sqlite') return true - return this.db.takeTxAdvisoryLock(this.did) - } - - async lockAvailable(): Promise { - if (this.db.dialect === 'sqlite') return true - return this.db.checkTxAdvisoryLock(this.did) - } - - async getRoot(): Promise { - const res = await this.db.db - .selectFrom('repo_root') - .selectAll() - .where('did', '=', this.did) - .executeTakeFirst() - if (!res) return null - return CID.parse(res.root) - } - - async getRootDetailed(): Promise<{ cid: CID; rev: string } | null> { - const res = await this.db.db - .selectFrom('repo_root') - .selectAll() - .where('did', '=', this.did) - .executeTakeFirst() - if (!res) return null - return { - cid: CID.parse(res.root), - rev: res.rev ?? '', // @TODO add not-null constraint to rev - } - } - - // proactively cache all blocks from a particular commit (to prevent multiple roundtrips) - async cacheRev(rev: string): Promise { - const res = await this.db.db - .selectFrom('ipld_block') - .where('creator', '=', this.did) - .where('repoRev', '=', rev) - .select(['ipld_block.cid', 'ipld_block.content']) - .limit(15) - .execute() - for (const row of res) { - this.cache.set(CID.parse(row.cid), row.content) - } - } - - async getBytes(cid: CID): Promise { - const cached = this.cache.get(cid) - if (cached) return cached - const found = await this.db.db - .selectFrom('ipld_block') - .where('ipld_block.creator', '=', this.did) - .where('ipld_block.cid', '=', cid.toString()) - .select('content') - .executeTakeFirst() - if (!found) return null - this.cache.set(cid, found.content) - return found.content - } - - async has(cid: CID): Promise { - const got = await this.getBytes(cid) - return !!got - } - - async getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }> { - const cached = this.cache.getMany(cids) - if (cached.missing.length < 1) return cached - const missing = new CidSet(cached.missing) - const missingStr = cached.missing.map((c) => c.toString()) - const blocks = new BlockMap() - await Promise.all( - chunkArray(missingStr, 500).map(async (batch) => { - const res = await this.db.db - .selectFrom('ipld_block') - .where('ipld_block.creator', '=', this.did) - .where('ipld_block.cid', 'in', batch) - .select(['ipld_block.cid as cid', 'ipld_block.content as content']) - .execute() - for (const row of res) { - const cid = CID.parse(row.cid) - blocks.set(cid, row.content) - missing.delete(cid) - } - }), - ) - this.cache.addMap(blocks) - blocks.addMap(cached.blocks) - return { blocks, missing: missing.toList() } - } - - async putBlock(cid: CID, block: Uint8Array, rev: string): Promise { - this.db.assertTransaction() - await this.db.db - .insertInto('ipld_block') - .values({ - cid: cid.toString(), - creator: this.did, - repoRev: rev, - size: block.length, - content: block, - }) - .onConflict((oc) => oc.doNothing()) - .execute() - this.cache.set(cid, block) - } - - async putMany(toPut: BlockMap, rev: string): Promise { - this.db.assertTransaction() - const blocks: IpldBlock[] = [] - toPut.forEach((bytes, cid) => { - blocks.push({ - cid: cid.toString(), - creator: this.did, - repoRev: rev, - size: bytes.length, - content: bytes, - }) - this.cache.addMap(toPut) - }) - await Promise.all( - chunkArray(blocks, 500).map((batch) => - this.db.db - .insertInto('ipld_block') - .values(batch) - .onConflict((oc) => oc.doNothing()) - .execute(), - ), - ) - } - - async deleteMany(cids: CID[]) { - if (cids.length < 1) return - const cidStrs = cids.map((c) => c.toString()) - await this.db.db - .deleteFrom('ipld_block') - .where('creator', '=', this.did) - .where('cid', 'in', cidStrs) - .execute() - } - - async applyCommit(commit: CommitData) { - await Promise.all([ - this.updateRoot(commit.cid, commit.prev ?? undefined), - this.putMany(commit.newBlocks, commit.rev), - this.deleteMany(commit.removedCids.toList()), - ]) - } - - async updateRoot(cid: CID, ensureSwap?: CID): Promise { - if (ensureSwap) { - const res = await this.db.db - .updateTable('repo_root') - .set({ - root: cid.toString(), - indexedAt: this.getTimestamp(), - }) - .where('did', '=', this.did) - .where('root', '=', ensureSwap.toString()) - .executeTakeFirst() - if (res.numUpdatedRows < 1) { - throw new ConcurrentWriteError() - } - } else { - await this.db.db - .insertInto('repo_root') - .values({ - did: this.did, - root: cid.toString(), - indexedAt: this.getTimestamp(), - }) - .onConflict((oc) => - oc.column('did').doUpdateSet({ - root: cid.toString(), - indexedAt: this.getTimestamp(), - }), - ) - .execute() - } - } - - async getCarStream(since?: string) { - const root = await this.getRoot() - if (!root) { - throw new RepoRootNotFoundError() - } - return writeCarStream(root, async (car) => { - let cursor: RevCursor | undefined = undefined - const writeRows = async ( - rows: { cid: string; content: Uint8Array }[], - ) => { - for (const row of rows) { - await car.put({ - cid: CID.parse(row.cid), - bytes: row.content, - }) - } - } - // allow us to write to car while fetching the next page - let writePromise: Promise = Promise.resolve() - do { - const res = await this.getBlockRange(since, cursor) - await writePromise - writePromise = writeRows(res) - const lastRow = res.at(-1) - if (lastRow && lastRow.repoRev) { - cursor = { - cid: CID.parse(lastRow.cid), - rev: lastRow.repoRev, - } - } else { - cursor = undefined - } - } while (cursor) - // ensure we flush the last page of blocks - await writePromise - }) - } - - async getBlockRange(since?: string, cursor?: RevCursor) { - const { ref } = this.db.db.dynamic - let builder = this.db.db - .selectFrom('ipld_block') - .where('creator', '=', this.did) - .select(['cid', 'repoRev', 'content']) - .orderBy('repoRev', 'desc') - .orderBy('cid', 'desc') - .limit(500) - if (cursor) { - // use this syntax to ensure we hit the index - builder = builder.where( - sql`((${ref('repoRev')}, ${ref('cid')}) < (${ - cursor.rev - }, ${cursor.cid.toString()}))`, - ) - } - if (since) { - builder = builder.where('repoRev', '>', since) - } - return builder.execute() - } - - getTimestamp(): string { - return this.timestamp || new Date().toISOString() - } - - async destroy(): Promise { - throw new Error('Destruction of SQL repo storage not allowed at runtime') - } -} - -type RevCursor = { - cid: CID - rev: string -} - -export default SqlRepoStorage - -export class RepoRootNotFoundError extends Error {} diff --git a/packages/pds/src/storage/index.ts b/packages/pds/src/storage/index.ts deleted file mode 100644 index 6fe2e4cce34..00000000000 --- a/packages/pds/src/storage/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './disk-blobstore' -export * from './memory-blobstore' diff --git a/packages/pds/src/storage/memory-blobstore.ts b/packages/pds/src/storage/memory-blobstore.ts deleted file mode 100644 index 6173c016c2a..00000000000 --- a/packages/pds/src/storage/memory-blobstore.ts +++ /dev/null @@ -1,96 +0,0 @@ -import stream from 'stream' -import { CID } from 'multiformats/cid' -import { BlobNotFoundError, BlobStore } from '@atproto/repo' -import { randomStr } from '@atproto/crypto' -import { bytesToStream, streamToBuffer } from '@atproto/common' - -export class MemoryBlobStore implements BlobStore { - temp: Map = new Map() - blocks: Map = new Map() - quarantined: Map = new Map() - - constructor() {} - - private genKey() { - return randomStr(32, 'base32') - } - - async hasTemp(key: string): Promise { - return this.temp.has(key) - } - - async hasStored(cid: CID): Promise { - return this.blocks.has(cid.toString()) - } - - async putTemp(bytes: Uint8Array | stream.Readable): Promise { - const key = this.genKey() - let byteArray: Uint8Array - if (ArrayBuffer.isView(bytes)) { - byteArray = bytes - } else { - byteArray = await streamToBuffer(bytes) - } - this.temp.set(key, byteArray) - return key - } - - async makePermanent(key: string, cid: CID): Promise { - const value = this.temp.get(key) - if (!value) { - throw new BlobNotFoundError() - } - this.blocks.set(cid.toString(), value) - this.temp.delete(key) - } - - async putPermanent( - cid: CID, - bytes: Uint8Array | stream.Readable, - ): Promise { - let byteArray: Uint8Array - if (ArrayBuffer.isView(bytes)) { - byteArray = bytes - } else { - byteArray = await streamToBuffer(bytes) - } - this.blocks.set(cid.toString(), byteArray) - } - - async quarantine(cid: CID): Promise { - const cidStr = cid.toString() - const bytes = this.blocks.get(cidStr) - if (bytes) { - this.blocks.delete(cidStr) - this.quarantined.set(cidStr, bytes) - } - } - - async unquarantine(cid: CID): Promise { - const cidStr = cid.toString() - const bytes = this.quarantined.get(cidStr) - if (bytes) { - this.quarantined.delete(cidStr) - this.blocks.set(cidStr, bytes) - } - } - - async getBytes(cid: CID): Promise { - const value = this.blocks.get(cid.toString()) - if (!value) { - throw new BlobNotFoundError() - } - return value - } - - async getStream(cid: CID): Promise { - const bytes = await this.getBytes(cid) - return bytesToStream(bytes) - } - - async delete(cid: CID): Promise { - this.blocks.delete(cid.toString()) - } -} - -export default MemoryBlobStore diff --git a/packages/pds/src/well-known.ts b/packages/pds/src/well-known.ts index cc19434e42f..d1c6169cac9 100644 --- a/packages/pds/src/well-known.ts +++ b/packages/pds/src/well-known.ts @@ -14,7 +14,7 @@ export const createRouter = (ctx: AppContext): express.Router => { } let did: string | undefined try { - const user = await ctx.services.account(ctx.db).getAccount(handle, true) + const user = await ctx.accountManager.getAccount(handle, true) did = user?.did } catch (err) { return res.status(500).send('Internal Server Error') diff --git a/packages/pds/tests/account-deletion.test.ts b/packages/pds/tests/account-deletion.test.ts index c9adbaf0c5e..cfcd5bf4c76 100644 --- a/packages/pds/tests/account-deletion.test.ts +++ b/packages/pds/tests/account-deletion.test.ts @@ -4,27 +4,28 @@ import { Selectable } from 'kysely' import Mail from 'nodemailer/lib/mailer' import AtpAgent from '@atproto/api' import basicSeed from './seeds/basic' -import { Database } from '../src' import { ServerMailer } from '../src/mailer' -import { BlobNotFoundError, BlobStore } from '@atproto/repo' -import { RepoRoot } from '../src/db/tables/repo-root' -import { UserAccount } from '../src/db/tables/user-account' -import { IpldBlock } from '../src/db/tables/ipld-block' -import { RepoBlob } from '../src/db/tables/repo-blob' -import { Blob } from '../src/db/tables/blob' -import { Record } from '../src/db/tables/record' -import { RepoSeq } from '../src/db/tables/repo-seq' +import { BlobNotFoundError } from '@atproto/repo' +import { + RepoRoot, + Account, + AppPassword, + EmailToken, + RefreshToken, +} from '../src/account-manager/db' +import { fileExists } from '@atproto/common' +import { AppContext } from '../src' +import { RepoSeq } from '../src/sequencer/db' describe('account deletion', () => { let network: TestNetworkNoAppView let agent: AtpAgent let sc: SeedClient + let ctx: AppContext let mailer: ServerMailer - let db: Database let initialDbContents: DbContents let updatedDbContents: DbContents - let blobstore: BlobStore const mailCatcher = new EventEmitter() let _origSendMail @@ -35,9 +36,8 @@ describe('account deletion', () => { network = await TestNetworkNoAppView.create({ dbPostgresSchema: 'account_deletion', }) - mailer = network.pds.ctx.mailer - db = network.pds.ctx.db - blobstore = network.pds.ctx.blobstore + ctx = network.pds.ctx + mailer = ctx.mailer agent = new AtpAgent({ service: network.pds.url }) sc = network.getSeedClient() await basicSeed(sc) @@ -51,7 +51,7 @@ describe('account deletion', () => { return result } - initialDbContents = await getDbContents(db) + initialDbContents = await getDbContents(ctx) }) afterAll(async () => { @@ -104,7 +104,7 @@ describe('account deletion', () => { }) it('deletes account with a valid token & password', async () => { - // Perform account deletion, including when there's an existing takedown on the account + // Perform account deletion, including when the account is already "taken down" await agent.api.com.atproto.admin.updateSubjectStatus( { subject: { @@ -135,49 +135,53 @@ describe('account deletion', () => { }) it('no longer store the user account or repo', async () => { - updatedDbContents = await getDbContents(db) - expect(updatedDbContents.roots).toEqual( - initialDbContents.roots.filter((row) => row.did !== carol.did), + updatedDbContents = await getDbContents(ctx) + expect(updatedDbContents.repoRoots).toEqual( + initialDbContents.repoRoots.filter((row) => row.did !== carol.did), ) - expect(updatedDbContents.users).toEqual( - initialDbContents.users.filter((row) => row.did !== carol.did), - ) - expect(updatedDbContents.blocks).toEqual( - initialDbContents.blocks.filter((row) => row.creator !== carol.did), + expect(updatedDbContents.userAccounts).toEqual( + initialDbContents.userAccounts.filter((row) => row.did !== carol.did), ) // check all seqs for this did are gone, except for the tombstone expect( - updatedDbContents.seqs.filter((row) => row.eventType !== 'tombstone'), - ).toEqual(initialDbContents.seqs.filter((row) => row.did !== carol.did)) + updatedDbContents.repoSeqs.filter((row) => row.eventType !== 'tombstone'), + ).toEqual(initialDbContents.repoSeqs.filter((row) => row.did !== carol.did)) // check we do have a tombstone for this did expect( - updatedDbContents.seqs.filter( + updatedDbContents.repoSeqs.filter( (row) => row.did === carol.did && row.eventType === 'tombstone', ).length, ).toEqual(1) - - expect(updatedDbContents.records).toEqual( - initialDbContents.records.filter((row) => row.did !== carol.did), + expect(updatedDbContents.appPasswords).toEqual( + initialDbContents.appPasswords.filter((row) => row.did !== carol.did), + ) + expect(updatedDbContents.emailTokens).toEqual( + initialDbContents.emailTokens.filter((row) => row.did !== carol.did), + ) + expect(updatedDbContents.refreshTokens).toEqual( + initialDbContents.refreshTokens.filter((row) => row.did !== carol.did), ) }) + it('deletes the users actor store', async () => { + const carolLoc = await network.pds.ctx.actorStore.getLocation(carol.did) + const dbExists = await fileExists(carolLoc.dbLocation) + expect(dbExists).toBe(false) + const walExists = await fileExists(`${carolLoc.dbLocation}-wal`) + expect(walExists).toBe(false) + const shmExists = await fileExists(`${carolLoc.dbLocation}-shm`) + expect(shmExists).toBe(false) + }) + it('deletes relevant blobs', async () => { const imgs = sc.posts[carol.did][0].images - // carols first blob is used by other accounts const first = imgs[0].image.ref - // carols second blob is used by only her const second = imgs[1].image.ref - const got = await blobstore.getBytes(first) - expect(got).toBeDefined() - const attempt = blobstore.getBytes(second) - await expect(attempt).rejects.toThrow(BlobNotFoundError) - - expect(updatedDbContents.repoBlobs).toEqual( - initialDbContents.repoBlobs.filter((row) => row.did !== carol.did), - ) - expect(updatedDbContents.blobs).toEqual( - initialDbContents.blobs.filter((row) => row.creator !== carol.did), - ) + const blobstore = network.pds.ctx.blobstore(carol.did) + const attempt1 = blobstore.getBytes(first) + await expect(attempt1).rejects.toThrow(BlobNotFoundError) + const attempt2 = blobstore.getBytes(second) + await expect(attempt2).rejects.toThrow(BlobNotFoundError) }) it('can delete an empty user', async () => { @@ -203,47 +207,91 @@ describe('account deletion', () => { password: eve.password, }) }) + + it('can be performed by an administrator.', async () => { + const ferris = await sc.createAccount('ferris', { + handle: 'ferris.test', + email: 'ferris@test.com', + password: 'ferris-test', + }) + + const tryUnauthed = agent.api.com.atproto.admin.deleteAccount({ + did: ferris.did, + }) + await expect(tryUnauthed).rejects.toThrow('Authentication Required') + + const tryAsModerator = agent.api.com.atproto.admin.deleteAccount( + { did: ferris.did }, + { + headers: network.pds.adminAuthHeaders('moderator'), + encoding: 'application/json', + }, + ) + await expect(tryAsModerator).rejects.toThrow( + 'Must be an admin to delete an account', + ) + + const { data: acct } = await agent.api.com.atproto.admin.getAccountInfo( + { did: ferris.did }, + { headers: network.pds.adminAuthHeaders('admin') }, + ) + expect(acct.did).toBe(ferris.did) + + await agent.api.com.atproto.admin.deleteAccount( + { did: ferris.did }, + { + headers: network.pds.adminAuthHeaders('admin'), + encoding: 'application/json', + }, + ) + + const tryGetAccountInfo = agent.api.com.atproto.admin.getAccountInfo( + { did: ferris.did }, + { headers: network.pds.adminAuthHeaders('admin') }, + ) + await expect(tryGetAccountInfo).rejects.toThrow('Account not found') + }) }) type DbContents = { - roots: RepoRoot[] - users: Selectable[] - blocks: IpldBlock[] - seqs: Selectable[] - records: Record[] - repoBlobs: RepoBlob[] - blobs: Blob[] + repoRoots: RepoRoot[] + userAccounts: Selectable[] + appPasswords: AppPassword[] + emailTokens: EmailToken[] + refreshTokens: RefreshToken[] + repoSeqs: Selectable[] } -const getDbContents = async (db: Database): Promise => { - const [roots, users, blocks, seqs, records, repoBlobs, blobs] = - await Promise.all([ - db.db.selectFrom('repo_root').orderBy('did').selectAll().execute(), - db.db.selectFrom('user_account').orderBy('did').selectAll().execute(), - db.db - .selectFrom('ipld_block') - .orderBy('creator') - .orderBy('cid') - .selectAll() - .execute(), - db.db.selectFrom('repo_seq').orderBy('id').selectAll().execute(), - db.db.selectFrom('record').orderBy('uri').selectAll().execute(), - db.db - .selectFrom('repo_blob') - .orderBy('did') - .orderBy('cid') - .selectAll() - .execute(), - db.db.selectFrom('blob').orderBy('cid').selectAll().execute(), - ]) +const getDbContents = async (ctx: AppContext): Promise => { + const { sequencer, accountManager } = ctx + const db = accountManager.db + const [ + repoRoots, + userAccounts, + appPasswords, + emailTokens, + refreshTokens, + repoSeqs, + ] = await Promise.all([ + db.db.selectFrom('repo_root').orderBy('did').selectAll().execute(), + db.db.selectFrom('account').orderBy('did').selectAll().execute(), + db.db + .selectFrom('app_password') + .orderBy('did') + .orderBy('name') + .selectAll() + .execute(), + db.db.selectFrom('email_token').orderBy('token').selectAll().execute(), + db.db.selectFrom('refresh_token').orderBy('id').selectAll().execute(), + sequencer.db.db.selectFrom('repo_seq').orderBy('seq').selectAll().execute(), + ]) return { - roots, - users, - blocks, - seqs, - records, - repoBlobs, - blobs, + repoRoots, + userAccounts, + appPasswords, + emailTokens, + refreshTokens, + repoSeqs, } } diff --git a/packages/pds/tests/account.test.ts b/packages/pds/tests/account.test.ts index f157380a1c1..70ca5abf741 100644 --- a/packages/pds/tests/account.test.ts +++ b/packages/pds/tests/account.test.ts @@ -4,7 +4,7 @@ import { IdResolver } from '@atproto/identity' import * as crypto from '@atproto/crypto' import { TestNetworkNoAppView } from '@atproto/dev-env' import Mail from 'nodemailer/lib/mailer' -import { AppContext, Database } from '../src' +import { AppContext } from '../src' import { ServerMailer } from '../src/mailer' const email = 'alice@test.com' @@ -16,10 +16,8 @@ const minsToMs = 60 * 1000 describe('account', () => { let network: TestNetworkNoAppView let ctx: AppContext - let repoSigningKey: string let agent: AtpAgent let mailer: ServerMailer - let db: Database let idResolver: IdResolver const mailCatcher = new EventEmitter() let _origSendMail @@ -33,9 +31,7 @@ describe('account', () => { }, }) mailer = network.pds.ctx.mailer - db = network.pds.ctx.db ctx = network.pds.ctx - repoSigningKey = network.pds.ctx.repoSigningKey.did() idResolver = network.pds.ctx.idResolver agent = network.pds.getClient() @@ -114,10 +110,11 @@ describe('account', () => { it('generates a properly formatted PLC DID', async () => { const didData = await idResolver.did.resolveAtprotoData(did) + const signingKey = await network.pds.ctx.actorStore.keypair(did) expect(didData.did).toBe(did) expect(didData.handle).toBe(handle) - expect(didData.signingKey).toBe(repoSigningKey) + expect(didData.signingKey).toBe(signingKey.did()) expect(didData.pds).toBe(network.pds.url) }) @@ -139,99 +136,100 @@ describe('account', () => { ]) }) - it('allows a user to bring their own DID', async () => { - const userKey = await crypto.Secp256k1Keypair.create() - const handle = 'byo-did.test' - const did = await ctx.plcClient.createDid({ - signingKey: ctx.repoSigningKey.did(), - handle, - rotationKeys: [ - userKey.did(), - ctx.cfg.identity.recoveryDidKey ?? '', - ctx.plcRotationKey.did(), - ], - pds: network.pds.url, - signer: userKey, - }) - - const res = await agent.api.com.atproto.server.createAccount({ - email: 'byo-did@test.com', - handle, - did, - password: 'byo-did-pass', - }) - - expect(res.data.handle).toEqual(handle) - expect(res.data.did).toEqual(did) - }) - - it('requires that the did a user brought be correctly set up for the server', async () => { - const userKey = await crypto.Secp256k1Keypair.create() - const baseDidInfo = { - signingKey: ctx.repoSigningKey.did(), - handle: 'byo-did.test', - rotationKeys: [ - userKey.did(), - ctx.cfg.identity.recoveryDidKey ?? '', - ctx.plcRotationKey.did(), - ], - pds: ctx.cfg.service.publicUrl, - signer: userKey, - } - const baseAccntInfo = { - email: 'byo-did@test.com', - handle: 'byo-did.test', - password: 'byo-did-pass', - } - - const did1 = await ctx.plcClient.createDid({ - ...baseDidInfo, - handle: 'different-handle.test', - }) - const attempt1 = agent.api.com.atproto.server.createAccount({ - ...baseAccntInfo, - did: did1, - }) - await expect(attempt1).rejects.toThrow( - 'provided handle does not match DID document handle', - ) - - const did2 = await ctx.plcClient.createDid({ - ...baseDidInfo, - pds: 'https://other-pds.com', - }) - const attempt2 = agent.api.com.atproto.server.createAccount({ - ...baseAccntInfo, - did: did2, - }) - await expect(attempt2).rejects.toThrow( - 'DID document pds endpoint does not match service endpoint', - ) - - const did3 = await ctx.plcClient.createDid({ - ...baseDidInfo, - rotationKeys: [userKey.did()], - }) - const attempt3 = agent.api.com.atproto.server.createAccount({ - ...baseAccntInfo, - did: did3, - }) - await expect(attempt3).rejects.toThrow( - 'PLC DID does not include service rotation key', - ) - - const did4 = await ctx.plcClient.createDid({ - ...baseDidInfo, - signingKey: userKey.did(), - }) - const attempt4 = agent.api.com.atproto.server.createAccount({ - ...baseAccntInfo, - did: did4, - }) - await expect(attempt4).rejects.toThrow( - 'DID document signing key does not match service signing key', - ) - }) + // @NOTE currently disabled until we allow a user to resver a keypair before migration + // it('allows a user to bring their own DID', async () => { + // const userKey = await crypto.Secp256k1Keypair.create() + // const handle = 'byo-did.test' + // const did = await ctx.plcClient.createDid({ + // signingKey: ctx.repoSigningKey.did(), + // handle, + // rotationKeys: [ + // userKey.did(), + // ctx.cfg.identity.recoveryDidKey ?? '', + // ctx.plcRotationKey.did(), + // ], + // pds: network.pds.url, + // signer: userKey, + // }) + + // const res = await agent.api.com.atproto.server.createAccount({ + // email: 'byo-did@test.com', + // handle, + // did, + // password: 'byo-did-pass', + // }) + + // expect(res.data.handle).toEqual(handle) + // expect(res.data.did).toEqual(did) + // }) + + // it('requires that the did a user brought be correctly set up for the server', async () => { + // const userKey = await crypto.Secp256k1Keypair.create() + // const baseDidInfo = { + // signingKey: ctx.repoSigningKey.did(), + // handle: 'byo-did.test', + // rotationKeys: [ + // userKey.did(), + // ctx.cfg.identity.recoveryDidKey ?? '', + // ctx.plcRotationKey.did(), + // ], + // pds: ctx.cfg.service.publicUrl, + // signer: userKey, + // } + // const baseAccntInfo = { + // email: 'byo-did@test.com', + // handle: 'byo-did.test', + // password: 'byo-did-pass', + // } + + // const did1 = await ctx.plcClient.createDid({ + // ...baseDidInfo, + // handle: 'different-handle.test', + // }) + // const attempt1 = agent.api.com.atproto.server.createAccount({ + // ...baseAccntInfo, + // did: did1, + // }) + // await expect(attempt1).rejects.toThrow( + // 'provided handle does not match DID document handle', + // ) + + // const did2 = await ctx.plcClient.createDid({ + // ...baseDidInfo, + // pds: 'https://other-pds.com', + // }) + // const attempt2 = agent.api.com.atproto.server.createAccount({ + // ...baseAccntInfo, + // did: did2, + // }) + // await expect(attempt2).rejects.toThrow( + // 'DID document pds endpoint does not match service endpoint', + // ) + + // const did3 = await ctx.plcClient.createDid({ + // ...baseDidInfo, + // rotationKeys: [userKey.did()], + // }) + // const attempt3 = agent.api.com.atproto.server.createAccount({ + // ...baseAccntInfo, + // did: did3, + // }) + // await expect(attempt3).rejects.toThrow( + // 'PLC DID does not include service rotation key', + // ) + + // const did4 = await ctx.plcClient.createDid({ + // ...baseDidInfo, + // signingKey: userKey.did(), + // }) + // const attempt4 = agent.api.com.atproto.server.createAccount({ + // ...baseAccntInfo, + // did: did4, + // }) + // await expect(attempt4).rejects.toThrow( + // 'DID document signing key does not match service signing key', + // ) + // }) it('allows administrative email updates', async () => { await agent.api.com.atproto.admin.updateAccountEmail( @@ -245,7 +243,7 @@ describe('account', () => { }, ) - const accnt = await ctx.services.account(ctx.db).getAccount(handle) + const accnt = await ctx.accountManager.getAccount(handle) expect(accnt?.email).toBe('alice-new@test.com') await agent.api.com.atproto.admin.updateAccountEmail( @@ -259,7 +257,7 @@ describe('account', () => { }, ) - const accnt2 = await ctx.services.account(ctx.db).getAccount(handle) + const accnt2 = await ctx.accountManager.getAccount(handle) expect(accnt2?.email).toBe(email) }) @@ -525,12 +523,12 @@ describe('account', () => { it('allows only unexpired password reset tokens', async () => { await agent.api.com.atproto.server.requestPasswordReset({ email }) - const res = await db.db + const res = await ctx.accountManager.db.db .updateTable('email_token') .where('purpose', '=', 'reset_password') .where('did', '=', did) .set({ - requestedAt: new Date(Date.now() - 16 * minsToMs), + requestedAt: new Date(Date.now() - 16 * minsToMs).toISOString(), }) .returning(['token']) .executeTakeFirst() diff --git a/packages/pds/tests/app-passwords.test.ts b/packages/pds/tests/app-passwords.test.ts index 0feb0a53712..d228896a6ca 100644 --- a/packages/pds/tests/app-passwords.test.ts +++ b/packages/pds/tests/app-passwords.test.ts @@ -1,6 +1,6 @@ import { TestNetworkNoAppView } from '@atproto/dev-env' import AtpAgent from '@atproto/api' -import * as jwt from 'jsonwebtoken' +import * as jose from 'jose' describe('app_passwords', () => { let network: TestNetworkNoAppView @@ -44,9 +44,7 @@ describe('app_passwords', () => { }) it('creates an access token for an app with a restricted scope', () => { - const decoded = jwt.decode(appAgent.session?.accessJwt ?? '', { - json: true, - }) + const decoded = jose.decodeJwt(appAgent.session?.accessJwt ?? '') expect(decoded?.scope).toEqual('com.atproto.appPass') }) diff --git a/packages/pds/tests/auth.test.ts b/packages/pds/tests/auth.test.ts index 650b2d1e9a7..d8d29942ccd 100644 --- a/packages/pds/tests/auth.test.ts +++ b/packages/pds/tests/auth.test.ts @@ -1,8 +1,9 @@ -import * as jwt from 'jsonwebtoken' +import * as jose from 'jose' import AtpAgent from '@atproto/api' import { TestNetworkNoAppView, SeedClient } from '@atproto/dev-env' import * as CreateSession from '@atproto/api/src/client/types/com/atproto/server/createSession' import * as RefreshSession from '@atproto/api/src/client/types/com/atproto/server/refreshSession' +import { createRefreshToken } from '../src/account-manager/helpers/auth' describe('auth', () => { let network: TestNetworkNoAppView @@ -41,12 +42,10 @@ describe('auth', () => { headers: SeedClient.getHeaders(jwt), }) } - const refreshSession = async (jwt) => { + const refreshSession = async (jwt: string) => { const { data } = await agent.api.com.atproto.server.refreshSession( undefined, - { - headers: SeedClient.getHeaders(jwt), - }, + { headers: SeedClient.getHeaders(jwt) }, ) return data } @@ -149,6 +148,31 @@ describe('auth', () => { ) }) + it('handles racing refreshes', async () => { + const email = 'dan@test.com' + const account = await createAccount({ + handle: 'dan.test', + password: 'password', + email, + }) + const tokenIdPromises: Promise[] = [] + const doRefresh = async () => { + const res = await refreshSession(account.refreshJwt) + const decoded = jose.decodeJwt(res.refreshJwt) + if (!decoded?.jti) { + throw new Error('undefined jti on refresh token') + } + return decoded.jti + } + for (let i = 0; i < 10; i++) { + tokenIdPromises.push(doRefresh()) + } + const tokenIds = await Promise.all(tokenIdPromises) + for (let i = 0; i < 10; i++) { + expect(tokenIds[i]).toEqual(tokenIds[0]) + } + }) + it('refresh token provides new token with same id on multiple uses during grace period.', async () => { const account = await createAccount({ handle: 'eve.test', @@ -158,9 +182,9 @@ describe('auth', () => { const refresh1 = await refreshSession(account.refreshJwt) const refresh2 = await refreshSession(account.refreshJwt) - const token0 = jwt.decode(account.refreshJwt, { json: true }) - const token1 = jwt.decode(refresh1.refreshJwt, { json: true }) - const token2 = jwt.decode(refresh2.refreshJwt, { json: true }) + const token0 = jose.decodeJwt(account.refreshJwt) + const token1 = jose.decodeJwt(refresh1.refreshJwt) + const token2 = jose.decodeJwt(refresh2.refreshJwt) expect(typeof token1?.jti).toEqual('string') expect(token1?.jti).toEqual(token2?.jti) @@ -169,14 +193,14 @@ describe('auth', () => { }) it('refresh token is revoked after grace period completes.', async () => { - const { db } = network.pds.ctx + const { db } = network.pds.ctx.accountManager const account = await createAccount({ handle: 'evan.test', email: 'evan@test.com', password: 'password', }) await refreshSession(account.refreshJwt) - const token = jwt.decode(account.refreshJwt, { json: true }) + const token = jose.decodeJwt(account.refreshJwt) // Update expiration (i.e. grace period) to end immediately const refreshUpdated = await db.db @@ -222,18 +246,20 @@ describe('auth', () => { }) it('expired refresh token cannot be used to refresh a session.', async () => { - const { services, db } = network.pds.ctx const account = await createAccount({ handle: 'holga.test', email: 'holga@test.com', password: 'password', }) - const refresh = services - .auth(db) - .createRefreshToken({ did: account.did, expiresIn: -1 }) - const refreshExpired = refreshSession(refresh.jwt) + const refreshJwt = await createRefreshToken({ + did: account.did, + jwtKey: network.pds.jwtSecretKey(), + serviceDid: network.pds.ctx.cfg.service.did, + expiresIn: -1, + }) + const refreshExpired = refreshSession(refreshJwt) await expect(refreshExpired).rejects.toThrow('Token has expired') - await deleteSession(refresh.jwt) // No problem revoking an expired token + await deleteSession(refreshJwt) // No problem revoking an expired token }) it('actor takedown disallows fresh session.', async () => { diff --git a/packages/pds/tests/blob-deletes.test.ts b/packages/pds/tests/blob-deletes.test.ts index bf7f36c256c..019f6dec92f 100644 --- a/packages/pds/tests/blob-deletes.test.ts +++ b/packages/pds/tests/blob-deletes.test.ts @@ -1,16 +1,14 @@ import { TestNetworkNoAppView, SeedClient } from '@atproto/dev-env' import AtpAgent, { BlobRef } from '@atproto/api' -import { Database } from '../src' -import DiskBlobStore from '../src/storage/disk-blobstore' import { ids } from '../src/lexicon/lexicons' +import { AppContext } from '../src' describe('blob deletes', () => { let network: TestNetworkNoAppView let agent: AtpAgent let sc: SeedClient - let blobstore: DiskBlobStore - let db: Database + let ctx: AppContext let alice: string let bob: string @@ -19,8 +17,7 @@ describe('blob deletes', () => { network = await TestNetworkNoAppView.create({ dbPostgresSchema: 'blob_deletes', }) - blobstore = network.pds.ctx.blobstore as DiskBlobStore - db = network.pds.ctx.db + ctx = network.pds.ctx agent = network.pds.getClient() sc = network.getSeedClient() await sc.createAccount('alice', { @@ -42,11 +39,9 @@ describe('blob deletes', () => { }) const getDbBlobsForDid = (did: string) => { - return db.db - .selectFrom('blob') - .selectAll() - .where('creator', '=', did) - .execute() + return ctx.actorStore.read(did, (store) => + store.db.db.selectFrom('blob').selectAll().execute(), + ) } it('deletes blob when record is deleted', async () => { @@ -62,7 +57,7 @@ describe('blob deletes', () => { const dbBlobs = await getDbBlobsForDid(alice) expect(dbBlobs.length).toBe(0) - const hasImg = await blobstore.hasStored(img.image.ref) + const hasImg = await ctx.blobstore(alice).hasStored(img.image.ref) expect(hasImg).toBeFalsy() }) @@ -85,10 +80,10 @@ describe('blob deletes', () => { expect(dbBlobs.length).toBe(1) expect(dbBlobs[0].cid).toEqual(img2.image.ref.toString()) - const hasImg = await blobstore.hasStored(img.image.ref) + const hasImg = await ctx.blobstore(alice).hasStored(img.image.ref) expect(hasImg).toBeFalsy() - const hasImg2 = await blobstore.hasStored(img2.image.ref) + const hasImg2 = await ctx.blobstore(alice).hasStored(img2.image.ref) expect(hasImg2).toBeTruthy() // reset @@ -113,10 +108,10 @@ describe('blob deletes', () => { const dbBlobs = await getDbBlobsForDid(alice) expect(dbBlobs.length).toBe(2) - const hasImg = await blobstore.hasStored(img.image.ref) + const hasImg = await ctx.blobstore(alice).hasStored(img.image.ref) expect(hasImg).toBeTruthy() - const hasImg2 = await blobstore.hasStored(img2.image.ref) + const hasImg2 = await ctx.blobstore(alice).hasStored(img2.image.ref) expect(hasImg2).toBeTruthy() await updateProfile(sc, alice) }) @@ -164,11 +159,11 @@ describe('blob deletes', () => { const dbBlobs = await getDbBlobsForDid(alice) expect(dbBlobs.length).toBe(1) - const hasImg = await blobstore.hasStored(img.image.ref) + const hasImg = await ctx.blobstore(alice).hasStored(img.image.ref) expect(hasImg).toBeTruthy() }) - it('does not delete blob from blob store if another user is using it', async () => { + it('does delete blob from user blob store if another user is using it', async () => { const imgAlice = await sc.uploadFile( alice, 'tests/sample-img/key-landscape-small.jpg', @@ -182,9 +177,10 @@ describe('blob deletes', () => { const postAlice = await sc.post(alice, 'post', undefined, [imgAlice]) await sc.post(bob, 'post', undefined, [imgBob]) await sc.deletePost(alice, postAlice.ref.uri) + await network.processAll() - const hasImg = await blobstore.hasStored(imgBob.image.ref) - expect(hasImg).toBeTruthy() + const hasImg = await ctx.blobstore(alice).hasStored(imgAlice.image.ref) + expect(hasImg).toBeFalsy() }) }) diff --git a/packages/pds/tests/crud.test.ts b/packages/pds/tests/crud.test.ts index 5a7189b3707..19470e38394 100644 --- a/packages/pds/tests/crud.test.ts +++ b/packages/pds/tests/crud.test.ts @@ -179,9 +179,9 @@ describe('crud operations', () => { }) const uploaded = uploadedRes.data.blob // Expect blobstore not to have image yet - await expect(ctx.blobstore.getBytes(uploaded.ref)).rejects.toThrow( - BlobNotFoundError, - ) + await expect( + ctx.blobstore(alice.did).getBytes(uploaded.ref), + ).rejects.toThrow(BlobNotFoundError) // Associate image with post, image should be placed in blobstore const res = await aliceAgent.api.app.bsky.feed.post.create( { repo: alice.did }, @@ -205,7 +205,7 @@ describe('crud operations', () => { expect(images.length).toEqual(1) expect(uploaded.ref.equals(images[0].image.ref)).toBeTruthy() // Ensure that the uploaded image is now in the blobstore, i.e. doesn't throw BlobNotFoundError - await ctx.blobstore.getBytes(uploaded.ref) + await ctx.blobstore(alice.did).getBytes(uploaded.ref) // Cleanup await aliceAgent.api.app.bsky.feed.post.delete({ rkey: postUri.rkey, @@ -458,6 +458,29 @@ describe('crud operations', () => { }) }) + it('does not produce commit on no-op update', async () => { + const { repo } = bobAgent.api.com.atproto + const rootRes1 = await bobAgent.api.com.atproto.sync.getLatestCommit({ + did: bob.did, + }) + const { data: put } = await repo.putRecord({ + ...profilePath, + repo: bob.did, + record: { + displayName: 'Robert', + description: 'Dog lover', + }, + }) + expect(put.uri).toEqual(`at://${bob.did}/${ids.AppBskyActorProfile}/self`) + + const rootRes2 = await bobAgent.api.com.atproto.sync.getLatestCommit({ + did: bob.did, + }) + + expect(rootRes2.data.cid).toEqual(rootRes1.data.cid) + expect(rootRes2.data.rev).toEqual(rootRes1.data.rev) + }) + it('temporarily only allows updates to profile', async () => { const { repo } = bobAgent.api.com.atproto const put = await repo.putRecord({ diff --git a/packages/pds/tests/db-notify.test.ts b/packages/pds/tests/db-notify.test.ts deleted file mode 100644 index cc711e6e549..00000000000 --- a/packages/pds/tests/db-notify.test.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { allComplete, createDeferrables, wait } from '@atproto/common' -import { Database } from '../src' - -describe('db notify', () => { - let dbOne: Database - let dbTwo: Database - - beforeAll(async () => { - if (process.env.DB_POSTGRES_URL) { - dbOne = Database.postgres({ - url: process.env.DB_POSTGRES_URL, - schema: 'db_notify', - }) - dbTwo = Database.postgres({ - url: process.env.DB_POSTGRES_URL, - schema: 'db_notify', - }) - await dbOne.startListeningToChannels() - await dbTwo.startListeningToChannels() - } else { - // in the sqlite case, we just use two references to the same db - dbOne = Database.memory() - dbTwo = dbOne - } - }) - - afterAll(async () => { - await dbOne.close() - await dbTwo.close() - }) - - it('notifies', async () => { - const sendCount = 5 - const deferrables = createDeferrables(sendCount) - let receivedCount = 0 - dbOne.channels.new_repo_event.addListener('message', () => { - deferrables[receivedCount]?.resolve() - receivedCount++ - }) - - for (let i = 0; i < sendCount; i++) { - dbTwo.notify('new_repo_event') - } - - await allComplete(deferrables) - expect(receivedCount).toBe(sendCount) - }) - - it('can notifies multiple listeners', async () => { - const sendCount = 5 - const deferrables = createDeferrables(sendCount * 2) - let receivedOne = 0 - let receivedTwo = 0 - dbOne.channels.new_repo_event.addListener('message', () => { - deferrables[receivedOne]?.resolve() - receivedOne++ - }) - dbOne.channels.new_repo_event.addListener('message', () => { - deferrables[receivedTwo + sendCount]?.resolve() - receivedTwo++ - }) - - for (let i = 0; i < sendCount; i++) { - await dbTwo.notify('new_repo_event') - } - - await allComplete(deferrables) - expect(receivedOne).toBe(sendCount) - expect(receivedTwo).toBe(sendCount) - }) - - it('notifies on multiple channels', async () => { - const sendCountOne = 5 - const sendCountTwo = 5 - const deferrablesOne = createDeferrables(sendCountOne) - const deferrablesTwo = createDeferrables(sendCountTwo) - let receivedCountOne = 0 - let receivedCountTwo = 0 - dbOne.channels.new_repo_event.addListener('message', () => { - deferrablesOne[receivedCountOne]?.resolve() - receivedCountOne++ - }) - dbOne.channels.outgoing_repo_seq.addListener('message', () => { - deferrablesTwo[receivedCountTwo]?.resolve() - receivedCountTwo++ - }) - - for (let i = 0; i < sendCountOne; i++) { - dbTwo.notify('new_repo_event') - } - for (let i = 0; i < sendCountTwo; i++) { - dbTwo.notify('outgoing_repo_seq') - } - - await allComplete(deferrablesOne) - await allComplete(deferrablesTwo) - expect(receivedCountOne).toBe(sendCountOne) - expect(receivedCountTwo).toBe(sendCountTwo) - }) - - it('bundles within txs', async () => { - const sendCount = 5 - let receivedCount = 0 - dbOne.channels.new_repo_event.addListener('message', () => { - receivedCount++ - }) - - await dbTwo.transaction(async (dbTx) => { - for (let i = 0; i < sendCount; i++) { - await dbTx.notify('new_repo_event') - } - }) - - await wait(200) - expect(receivedCount).toBe(1) - }) - - it('does not send on failed tx', async () => { - let received = false - dbOne.channels.new_repo_event.addListener('message', () => { - received = true - }) - - const fakeErr = new Error('test') - try { - await dbTwo.transaction(async (dbTx) => { - await dbTx.notify('new_repo_event') - throw fakeErr - }) - } catch (err) { - if (err !== fakeErr) { - throw err - } - } - await wait(200) - expect(received).toBeFalsy() - }) -}) diff --git a/packages/pds/tests/db.test.ts b/packages/pds/tests/db.test.ts index d5a87595ac8..c5b10078882 100644 --- a/packages/pds/tests/db.test.ts +++ b/packages/pds/tests/db.test.ts @@ -1,391 +1,164 @@ -import { sql } from 'kysely' -import { once } from 'events' import { TestNetworkNoAppView } from '@atproto/dev-env' -import { createDeferrable, wait } from '@atproto/common' -import { Database } from '../src' -import { Leader, appMigration } from '../src/db/leader' +import { AccountDb } from '../src/account-manager/db' describe('db', () => { let network: TestNetworkNoAppView - let db: Database + let db: AccountDb beforeAll(async () => { network = await TestNetworkNoAppView.create({ dbPostgresSchema: 'db', }) - db = network.pds.ctx.db + db = network.pds.ctx.accountManager.db }) afterAll(async () => { await network.close() }) - describe('transaction()', () => { - it('commits changes', async () => { - const result = await db.transaction(async (dbTxn) => { - return await dbTxn.db - .insertInto('repo_root') - .values({ - did: 'x', - root: 'x', - rev: 'x', - indexedAt: 'bad-date', - }) - .returning('did') - .executeTakeFirst() - }) - - if (!result) { - return expect(result).toBeTruthy() - } - - expect(result.did).toEqual('x') - - const row = await db.db - .selectFrom('repo_root') - .selectAll() - .where('did', '=', 'x') + it('commits changes', async () => { + const result = await db.transaction(async (dbTxn) => { + return await dbTxn.db + .insertInto('repo_root') + .values({ + did: 'x', + cid: 'x', + rev: 'x', + indexedAt: 'bad-date', + }) + .returning('did') .executeTakeFirst() - - expect(row).toEqual({ - did: 'x', - root: 'x', - rev: 'x', - indexedAt: 'bad-date', - takedownRef: null, - }) }) - it('rolls-back changes on failure', async () => { - const promise = db.transaction(async (dbTxn) => { - await dbTxn.db - .insertInto('repo_root') - .values({ - did: 'y', - root: 'y', - indexedAt: 'bad-date', - }) - .returning('did') - .executeTakeFirst() + if (!result) { + return expect(result).toBeTruthy() + } - throw new Error('Oops!') - }) - - await expect(promise).rejects.toThrow('Oops!') + expect(result.did).toEqual('x') - const row = await db.db - .selectFrom('repo_root') - .selectAll() - .where('did', '=', 'y') - .executeTakeFirst() + const row = await db.db + .selectFrom('repo_root') + .selectAll() + .where('did', '=', 'x') + .executeTakeFirst() - expect(row).toBeUndefined() + expect(row).toEqual({ + did: 'x', + cid: 'x', + rev: 'x', + indexedAt: 'bad-date', }) + }) - it('indicates isTransaction', async () => { - expect(db.isTransaction).toEqual(false) - - await db.transaction(async (dbTxn) => { - expect(db.isTransaction).toEqual(false) - expect(dbTxn.isTransaction).toEqual(true) - }) - - expect(db.isTransaction).toEqual(false) - }) - - it('asserts transaction', async () => { - expect(() => db.assertTransaction()).toThrow('Transaction required') - - await db.transaction(async (dbTxn) => { - expect(() => dbTxn.assertTransaction()).not.toThrow() - }) - }) - - it('does not allow leaky transactions', async () => { - let leakedTx: Database | undefined - - const tx = db.transaction(async (dbTxn) => { - leakedTx = dbTxn - await dbTxn.db - .insertInto('repo_root') - .values({ root: 'a', did: 'a', indexedAt: 'bad-date' }) - .execute() - throw new Error('test tx failed') - }) - await expect(tx).rejects.toThrow('test tx failed') - - const attempt = leakedTx?.db + it('rolls-back changes on failure', async () => { + const promise = db.transaction(async (dbTxn) => { + await dbTxn.db .insertInto('repo_root') - .values({ root: 'b', did: 'b', indexedAt: 'bad-date' }) - .execute() - await expect(attempt).rejects.toThrow('tx already failed') - - const res = await db.db - .selectFrom('repo_root') - .selectAll() - .where('did', 'in', ['a', 'b']) - .execute() - - expect(res.length).toBe(0) - }) - - it('ensures all inflight queries are rolled back', async () => { - let promise: Promise | undefined = undefined - const names: string[] = [] - try { - await db.transaction(async (dbTxn) => { - const queries: Promise[] = [] - for (let i = 0; i < 20; i++) { - const name = `user${i}` - const query = dbTxn.db - .insertInto('repo_root') - .values({ - root: name, - did: name, - indexedAt: 'bad-date', - }) - .execute() - names.push(name) - queries.push(query) - } - promise = Promise.allSettled(queries) - throw new Error() + .values({ + did: 'y', + cid: 'y', + rev: 'y', + indexedAt: 'bad-date', }) - } catch (err) { - expect(err).toBeDefined() - } - if (promise) { - await promise - } + .returning('did') + .executeTakeFirst() - const res = await db.db - .selectFrom('repo_root') - .selectAll() - .where('did', 'in', names) - .execute() - expect(res.length).toBe(0) + throw new Error('Oops!') }) - }) - describe('transaction advisory locks', () => { - it('allows locks in txs to run sequentially', async () => { - if (db.dialect !== 'pg') return - for (let i = 0; i < 100; i++) { - await db.transaction(async (dbTxn) => { - const locked = await dbTxn.takeTxAdvisoryLock('asfd') - expect(locked).toBe(true) - }) - } - }) + await expect(promise).rejects.toThrow('Oops!') - it('locks block between txns', async () => { - if (db.dialect !== 'pg') return - const deferable = createDeferrable() - const tx1 = db.transaction(async (dbTxn) => { - const locked = await dbTxn.takeTxAdvisoryLock('asdf') - expect(locked).toBe(true) - await deferable.complete - }) - // give it just a second to ensure it gets the lock - await wait(10) - const tx2 = db.transaction(async (dbTxn) => { - const locked = await dbTxn.takeTxAdvisoryLock('asdf') - expect(locked).toBe(false) - deferable.resolve() - await tx1 - const locked2 = await dbTxn.takeTxAdvisoryLock('asdf') - expect(locked2).toBe(true) - }) - await tx2 - }) + const row = await db.db + .selectFrom('repo_root') + .selectAll() + .where('did', '=', 'y') + .executeTakeFirst() + + expect(row).toBeUndefined() }) - describe('Leader', () => { - it('allows leaders to run sequentially.', async () => { - const task = async () => { - await wait(25) - return 'complete' - } - const leader1 = new Leader(777, db) - const leader2 = new Leader(777, db) - const leader3 = new Leader(777, db) - const result1 = await leader1.run(task) - await wait(5) // Short grace period for pg to close session - const result2 = await leader2.run(task) - await wait(5) - const result3 = await leader3.run(task) - await wait(5) - const result4 = await leader3.run(task) - expect([result1, result2, result3, result4]).toEqual([ - { ran: true, result: 'complete' }, - { ran: true, result: 'complete' }, - { ran: true, result: 'complete' }, - { ran: true, result: 'complete' }, - ]) - }) + it('indicates isTransaction', async () => { + expect(db.isTransaction).toEqual(false) - it('only allows one leader at a time.', async () => { - await wait(5) - const task = async () => { - await wait(25) - return 'complete' - } - const results = await Promise.all([ - new Leader(777, db).run(task), - new Leader(777, db).run(task), - new Leader(777, db).run(task), - ]) - const byRan = (a, b) => Number(a.ran) - Number(b.ran) - expect(results.sort(byRan)).toEqual([ - { ran: false }, - { ran: false }, - { ran: true, result: 'complete' }, - ]) + await db.transaction(async (dbTxn) => { + expect(db.isTransaction).toEqual(false) + expect(dbTxn.isTransaction).toEqual(true) }) - it('leaders with different ids do not conflict.', async () => { - await wait(5) - const task = async () => { - await wait(25) - return 'complete' - } - const results = await Promise.all([ - new Leader(777, db).run(task), - new Leader(778, db).run(task), - new Leader(779, db).run(task), - ]) - expect(results).toEqual([ - { ran: true, result: 'complete' }, - { ran: true, result: 'complete' }, - { ran: true, result: 'complete' }, - ]) - }) + expect(db.isTransaction).toEqual(false) + }) + + it('asserts transaction', async () => { + expect(() => db.assertTransaction()).toThrow('Transaction required') - it('supports abort.', async () => { - const task = async (ctx: { signal: AbortSignal }) => { - wait(10).then(abort) - return await Promise.race([ - wait(50), - once(ctx.signal, 'abort').then(() => ctx.signal.reason), - ]) - } - const leader = new Leader(777, db) - const abort = () => { - leader.session?.abortController.abort(new Error('Oops!')) - } - const result = await leader.run(task) - expect(result).toEqual({ ran: true, result: new Error('Oops!') }) + await db.transaction(async (dbTxn) => { + expect(() => dbTxn.assertTransaction()).not.toThrow() }) }) - describe('appMigration()', () => { - it('fails once together', async () => { - if (db.cfg.dialect !== 'pg') return // postgres-only + it('does not allow leaky transactions', async () => { + let leakedTx: AccountDb | undefined - await db.db.deleteFrom('did_handle').execute() - await db.db - .insertInto('did_handle') - .values([ - { - did: 'did:plc:1', - handle: 'user1', - }, - { - did: 'did:plc:2', - handle: 'user2', - }, - ]) + const tx = db.transaction(async (dbTxn) => { + leakedTx = dbTxn + await dbTxn.db + .insertInto('repo_root') + .values({ cid: 'a', did: 'a', rev: 'a', indexedAt: 'bad-date' }) .execute() - - let runCount = 0 - const migration = async (tx: Database) => { - const nthRun = runCount++ - await wait(100) - await tx.db.deleteFrom('did_handle').execute() - await wait(100) - if (nthRun === 0) throw new Error('Intentional failure') - } - - const results = await Promise.allSettled([ - appMigration(db, 'migration-fail', migration), - appMigration(db, 'migration-fail', migration), - appMigration(db, 'migration-fail', migration), - appMigration(db, 'migration-fail', migration), - appMigration(db, 'migration-fail', migration), - appMigration(db, 'migration-fail', migration), - appMigration(db, 'migration-fail', migration), - appMigration(db, 'migration-fail', migration), - ]) - - const errMessages = results - .map((res) => res['reason']?.['message'] ?? null) - .sort() - - expect(runCount).toEqual(1) - expect(errMessages).toEqual([ - 'Intentional failure', - 'Migration previously failed', - 'Migration previously failed', - 'Migration previously failed', - 'Migration previously failed', - 'Migration previously failed', - 'Migration previously failed', - 'Migration previously failed', - ]) - - const after = await db.db - .selectFrom('did_handle') - .select(sql`count(*)`.as('count')) - .executeTakeFirstOrThrow() - expect(after.count).toEqual(2) + throw new Error('test tx failed') }) + await expect(tx).rejects.toThrow('test tx failed') - it('succeeds once together', async () => { - if (db.cfg.dialect !== 'pg') return // postgres-only + const attempt = leakedTx?.db + .insertInto('repo_root') + .values({ cid: 'b', did: 'b', rev: 'b', indexedAt: 'bad-date' }) + .execute() + await expect(attempt).rejects.toThrow('tx already failed') - await db.db.deleteFrom('did_handle').execute() + const res = await db.db + .selectFrom('repo_root') + .selectAll() + .where('did', 'in', ['a', 'b']) + .execute() - let runCount = 0 - const migration = async (tx: Database) => { - const nthRun = runCount++ - await wait(100) - await tx.db - .insertInto('did_handle') - .values({ - did: `did:plc:${nthRun}`, - handle: `user${nthRun}`, - }) - .execute() - await wait(100) - } - - const results = await Promise.allSettled([ - appMigration(db, 'migration-succeed', migration), - appMigration(db, 'migration-succeed', migration), - appMigration(db, 'migration-succeed', migration), - appMigration(db, 'migration-succeed', migration), - appMigration(db, 'migration-succeed', migration), - appMigration(db, 'migration-succeed', migration), - appMigration(db, 'migration-succeed', migration), - appMigration(db, 'migration-succeed', migration), - ]) - - const statuses = results.map((res) => res.status) - - expect(runCount).toEqual(1) - expect(statuses).toEqual([ - 'fulfilled', - 'fulfilled', - 'fulfilled', - 'fulfilled', - 'fulfilled', - 'fulfilled', - 'fulfilled', - 'fulfilled', - ]) + expect(res.length).toBe(0) + }) - const after = await db.db.selectFrom('did_handle').select('did').execute() - expect(after).toEqual([{ did: 'did:plc:0' }]) - }) + it('ensures all inflight queries are rolled back', async () => { + let promise: Promise | undefined = undefined + const names: string[] = [] + try { + await db.transaction(async (dbTxn) => { + const queries: Promise[] = [] + for (let i = 0; i < 20; i++) { + const name = `user${i}` + const query = dbTxn.db + .insertInto('repo_root') + .values({ + cid: name, + did: name, + rev: name, + indexedAt: 'bad-date', + }) + .execute() + names.push(name) + queries.push(query) + } + promise = Promise.allSettled(queries) + throw new Error() + }) + } catch (err) { + expect(err).toBeDefined() + } + if (promise) { + await promise + } + + const res = await db.db + .selectFrom('repo_root') + .selectAll() + .where('did', 'in', names) + .execute() + expect(res.length).toBe(0) }) }) diff --git a/packages/pds/tests/entryway.test.ts b/packages/pds/tests/entryway.test.ts new file mode 100644 index 00000000000..8d2c03dc9b3 --- /dev/null +++ b/packages/pds/tests/entryway.test.ts @@ -0,0 +1,172 @@ +import * as os from 'node:os' +import * as path from 'node:path' +import AtpAgent from '@atproto/api' +import { Secp256k1Keypair, randomStr } from '@atproto/crypto' +import { SeedClient, TestPds, TestPlc, mockResolvers } from '@atproto/dev-env' +import * as pdsEntryway from '@atproto/pds-entryway' +import * as ui8 from 'uint8arrays' +import getPort from 'get-port' + +describe('entryway', () => { + let plc: TestPlc + let pds: TestPds + let entryway: pdsEntryway.PDS + let pdsAgent: AtpAgent + let entrywayAgent: AtpAgent + let alice: string + let accessToken: string + + beforeAll(async () => { + const jwtSigningKey = await Secp256k1Keypair.create({ exportable: true }) + const plcRotationKey = await Secp256k1Keypair.create({ exportable: true }) + const entrywayPort = await getPort() + plc = await TestPlc.create({}) + pds = await TestPds.create({ + entrywayUrl: `http://localhost:${entrywayPort}`, + entrywayDid: 'did:example:entryway', + entrywayJwtVerifyKeyK256PublicKeyHex: getPublicHex(jwtSigningKey), + entrywayPlcRotationKey: plcRotationKey.did(), + adminPassword: 'admin-pass', + serviceHandleDomains: [], + didPlcUrl: plc.url, + serviceDid: 'did:example:pds', + inviteRequired: false, + }) + entryway = await createEntryway({ + dbPostgresSchema: 'entryway', + port: entrywayPort, + adminPassword: 'admin-pass', + jwtSigningKeyK256PrivateKeyHex: await getPrivateHex(jwtSigningKey), + plcRotationKeyK256PrivateKeyHex: await getPrivateHex(plcRotationKey), + inviteRequired: false, + serviceDid: 'did:example:entryway', + didPlcUrl: plc.url, + }) + mockResolvers(pds.ctx.idResolver, pds) + mockResolvers(entryway.ctx.idResolver, pds) + await entryway.ctx.db.db + .insertInto('pds') + .values({ + did: pds.ctx.cfg.service.did, + host: new URL(pds.ctx.cfg.service.publicUrl).host, + weight: 1, + }) + .execute() + pdsAgent = pds.getClient() + entrywayAgent = new AtpAgent({ + service: entryway.ctx.cfg.service.publicUrl, + }) + }) + + afterAll(async () => { + await plc.close() + await entryway.destroy() + await pds.close() + }) + + it('creates account.', async () => { + const res = await entrywayAgent.api.com.atproto.server.createAccount({ + email: 'alice@test.com', + handle: 'alice.test', + password: 'test123', + }) + alice = res.data.did + accessToken = res.data.accessJwt + + const account = await pds.ctx.accountManager.getAccount(alice) + expect(account?.did).toEqual(alice) + expect(account?.handle).toEqual('alice.test') + }) + + it('auths with both services.', async () => { + const entrywaySession = + await entrywayAgent.api.com.atproto.server.getSession(undefined, { + headers: SeedClient.getHeaders(accessToken), + }) + const pdsSession = await pdsAgent.api.com.atproto.server.getSession( + undefined, + { headers: SeedClient.getHeaders(accessToken) }, + ) + expect(entrywaySession.data).toEqual(pdsSession.data) + }) + + it('updates handle from pds.', async () => { + await pdsAgent.api.com.atproto.identity.updateHandle( + { handle: 'alice2.test' }, + { + headers: SeedClient.getHeaders(accessToken), + encoding: 'application/json', + }, + ) + const doc = await pds.ctx.idResolver.did.resolve(alice) + const handleToDid = await pds.ctx.idResolver.handle.resolve('alice2.test') + const accountFromPds = await pds.ctx.accountManager.getAccount(alice) + const accountFromEntryway = await entryway.ctx.services + .account(entryway.ctx.db) + .getAccount(alice) + expect(doc?.alsoKnownAs).toEqual(['at://alice2.test']) + expect(handleToDid).toEqual(alice) + expect(accountFromPds?.handle).toEqual('alice2.test') + expect(accountFromEntryway?.handle).toEqual('alice2.test') + }) + + it('updates handle from entryway.', async () => { + await entrywayAgent.api.com.atproto.identity.updateHandle( + { handle: 'alice3.test' }, + { + headers: SeedClient.getHeaders(accessToken), + encoding: 'application/json', + }, + ) + const doc = await entryway.ctx.idResolver.did.resolve(alice) + const handleToDid = await entryway.ctx.idResolver.handle.resolve( + 'alice3.test', + ) + const accountFromPds = await pds.ctx.accountManager.getAccount(alice) + const accountFromEntryway = await entryway.ctx.services + .account(entryway.ctx.db) + .getAccount(alice) + expect(doc?.alsoKnownAs).toEqual(['at://alice3.test']) + expect(handleToDid).toEqual(alice) + expect(accountFromPds?.handle).toEqual('alice3.test') + expect(accountFromEntryway?.handle).toEqual('alice3.test') + }) +}) + +const createEntryway = async ( + config: pdsEntryway.ServerEnvironment & { + adminPassword: string + jwtSigningKeyK256PrivateKeyHex: string + plcRotationKeyK256PrivateKeyHex: string + }, +) => { + const signingKey = await Secp256k1Keypair.create({ exportable: true }) + const recoveryKey = await Secp256k1Keypair.create({ exportable: true }) + const env: pdsEntryway.ServerEnvironment = { + isEntryway: true, + recoveryDidKey: recoveryKey.did(), + serviceHandleDomains: ['.test'], + dbPostgresUrl: process.env.DB_POSTGRES_URL, + blobstoreDiskLocation: path.join(os.tmpdir(), randomStr(8, 'base32')), + bskyAppViewUrl: 'https://appview.invalid', + bskyAppViewDid: 'did:example:invalid', + bskyAppViewCdnUrlPattern: 'http://cdn.appview.com/%s/%s/%s', + jwtSecret: randomStr(8, 'base32'), + repoSigningKeyK256PrivateKeyHex: await getPrivateHex(signingKey), + ...config, + } + const cfg = pdsEntryway.envToCfg(env) + const secrets = pdsEntryway.envToSecrets(env) + const server = await pdsEntryway.PDS.create(cfg, secrets) + await server.ctx.db.migrateToLatestOrThrow() + await server.start() + return server +} + +const getPublicHex = (key: Secp256k1Keypair) => { + return key.publicKeyStr('hex') +} + +const getPrivateHex = async (key: Secp256k1Keypair) => { + return ui8.toString(await key.export(), 'hex') +} diff --git a/packages/pds/tests/file-uploads.test.ts b/packages/pds/tests/file-uploads.test.ts index 07b4c6ebb55..fd4c7ad1a17 100644 --- a/packages/pds/tests/file-uploads.test.ts +++ b/packages/pds/tests/file-uploads.test.ts @@ -1,70 +1,50 @@ import fs from 'fs/promises' import { gzipSync } from 'zlib' import AtpAgent from '@atproto/api' -import { Database } from '../src' -import DiskBlobStore from '../src/storage/disk-blobstore' +import { AppContext } from '../src' +import DiskBlobStore from '../src/disk-blobstore' import * as uint8arrays from 'uint8arrays' import { randomBytes } from '@atproto/crypto' import { BlobRef } from '@atproto/lexicon' -import { ids } from '../src/lexicon/lexicons' -import { TestNetworkNoAppView } from '@atproto/dev-env' - -const alice = { - email: 'alice@test.com', - handle: 'alice.test', - did: '', - password: 'alice-pass', -} -const bob = { - email: 'bob@test.com', - handle: 'bob.test', - did: '', - password: 'bob-pass', -} +import { SeedClient, TestNetworkNoAppView } from '@atproto/dev-env' +import { users } from './seeds/users' +import { ActorDb } from '../src/actor-store/db' describe('file uploads', () => { let network: TestNetworkNoAppView - let aliceAgent: AtpAgent - let bobAgent: AtpAgent - let blobstore: DiskBlobStore - let db: Database + let ctx: AppContext + let aliceDb: ActorDb + let alice: string + let bob: string + let agent: AtpAgent + let sc: SeedClient beforeAll(async () => { network = await TestNetworkNoAppView.create({ dbPostgresSchema: 'file_uploads', }) - blobstore = network.pds.ctx.blobstore as DiskBlobStore - db = network.pds.ctx.db - aliceAgent = network.pds.getClient() - bobAgent = network.pds.getClient() + ctx = network.pds.ctx + agent = network.pds.getClient() + sc = network.getSeedClient() + await sc.createAccount('alice', users.alice) + await sc.createAccount('bob', users.bob) + alice = sc.dids.alice + bob = sc.dids.bob + aliceDb = await network.pds.ctx.actorStore.openDb(alice) }) afterAll(async () => { + aliceDb.close() await network.close() }) - it('registers users', async () => { - const res = await aliceAgent.createAccount({ - email: alice.email, - handle: alice.handle, - password: alice.password, - }) - alice.did = res.data.did - const res2 = await bobAgent.createAccount({ - email: bob.email, - handle: bob.handle, - password: bob.password, - }) - bob.did = res2.data.did - }) - let smallBlob: BlobRef let smallFile: Uint8Array it('handles client abort', async () => { const abortController = new AbortController() - const _putTemp = network.pds.ctx.blobstore.putTemp - network.pds.ctx.blobstore.putTemp = function (...args) { + const _putTemp = DiskBlobStore.prototype.putTemp + DiskBlobStore.prototype.putTemp = function (...args) { // Abort just as processing blob in packages/pds/src/services/repo/blobs.ts process.nextTick(() => abortController.abort()) return _putTemp.call(this, ...args) @@ -76,26 +56,27 @@ describe('file uploads', () => { body: Buffer.alloc(5000000), // Enough bytes to get some chunking going on signal: abortController.signal, headers: { + ...sc.getHeaders(alice), 'content-type': 'image/jpeg', - authorization: `Bearer ${aliceAgent.session?.accessJwt}`, }, }, ) await expect(response).rejects.toThrow('operation was aborted') // Cleanup - network.pds.ctx.blobstore.putTemp = _putTemp + DiskBlobStore.prototype.putTemp = _putTemp // This test would fail from an uncaught exception: this grace period gives time for that to surface await new Promise((res) => setTimeout(res, 10)) }) it('uploads files', async () => { smallFile = await fs.readFile('tests/sample-img/key-portrait-small.jpg') - const res = await aliceAgent.api.com.atproto.repo.uploadBlob(smallFile, { + const res = await agent.api.com.atproto.repo.uploadBlob(smallFile, { + headers: sc.getHeaders(alice), encoding: 'image/jpeg', }) smallBlob = res.data.blob - const found = await db.db + const found = await aliceDb.db .selectFrom('blob') .selectAll() .where('cid', '=', smallBlob.ref.toString()) @@ -106,32 +87,30 @@ describe('file uploads', () => { expect(found?.tempKey).toBeDefined() expect(found?.width).toBe(87) expect(found?.height).toBe(150) - expect(await blobstore.hasTemp(found?.tempKey as string)).toBeTruthy() + const hasKey = await ctx.blobstore(alice).hasTemp(found?.tempKey as string) + expect(hasKey).toBeTruthy() }) it('can reference the file', async () => { - await updateProfile(aliceAgent, { - displayName: 'Alice', - avatar: smallBlob, - }) + await sc.updateProfile(alice, { displayName: 'Alice', avatar: smallBlob }) }) it('after being referenced, the file is moved to permanent storage', async () => { - const found = await db.db + const found = await aliceDb.db .selectFrom('blob') .selectAll() .where('cid', '=', smallBlob.ref.toString()) .executeTakeFirst() - expect(found?.tempKey).toBeNull() - expect(await blobstore.hasStored(smallBlob.ref)).toBeTruthy() - const storedBytes = await blobstore.getBytes(smallBlob.ref) + const hasStored = ctx.blobstore(alice).hasStored(smallBlob.ref) + expect(hasStored).toBeTruthy() + const storedBytes = await ctx.blobstore(alice).getBytes(smallBlob.ref) expect(uint8arrays.equals(smallFile, storedBytes)).toBeTruthy() }) it('can fetch the file after being referenced', async () => { - const { headers, data } = await aliceAgent.api.com.atproto.sync.getBlob({ - did: alice.did, + const { headers, data } = await agent.api.com.atproto.sync.getBlob({ + did: alice, cid: smallBlob.ref.toString(), }) expect(headers['content-type']).toEqual('image/jpeg') @@ -147,12 +126,13 @@ describe('file uploads', () => { it('does not allow referencing a file that is outside blob constraints', async () => { largeFile = await fs.readFile('tests/sample-img/hd-key.jpg') - const res = await aliceAgent.api.com.atproto.repo.uploadBlob(largeFile, { + const res = await agent.api.com.atproto.repo.uploadBlob(largeFile, { + headers: sc.getHeaders(alice), encoding: 'image/jpeg', }) largeBlob = res.data.blob - const profilePromise = updateProfile(aliceAgent, { + const profilePromise = sc.updateProfile(alice, { avatar: largeBlob, }) @@ -160,57 +140,62 @@ describe('file uploads', () => { }) it('does not make a blob permanent if referencing failed', async () => { - const found = await db.db + const found = await aliceDb.db .selectFrom('blob') .selectAll() .where('cid', '=', largeBlob.ref.toString()) .executeTakeFirst() expect(found?.tempKey).toBeDefined() - expect(await blobstore.hasTemp(found?.tempKey as string)).toBeTruthy() - expect(await blobstore.hasStored(largeBlob.ref)).toBeFalsy() + const hasTemp = await ctx.blobstore(alice).hasTemp(found?.tempKey as string) + expect(hasTemp).toBeTruthy() + const hasStored = await ctx.blobstore(alice).hasStored(largeBlob.ref) + expect(hasStored).toBeFalsy() }) it('permits duplicate uploads of the same file', async () => { const file = await fs.readFile('tests/sample-img/key-landscape-small.jpg') - const { data: uploadA } = await aliceAgent.api.com.atproto.repo.uploadBlob( + const { data: uploadA } = await agent.api.com.atproto.repo.uploadBlob( file, { + headers: sc.getHeaders(alice), encoding: 'image/jpeg', } as any, ) - const { data: uploadB } = await bobAgent.api.com.atproto.repo.uploadBlob( + const { data: uploadB } = await agent.api.com.atproto.repo.uploadBlob( file, { + headers: sc.getHeaders(bob), encoding: 'image/jpeg', } as any, ) expect(uploadA).toEqual(uploadB) - await updateProfile(aliceAgent, { + await sc.updateProfile(alice, { displayName: 'Alice', avatar: uploadA.blob, }) - const profileA = await aliceAgent.api.app.bsky.actor.profile.get({ - repo: alice.did, + const profileA = await agent.api.app.bsky.actor.profile.get({ + repo: alice, rkey: 'self', }) expect((profileA.value as any).avatar.cid).toEqual(uploadA.cid) - await updateProfile(bobAgent, { + await sc.updateProfile(bob, { displayName: 'Bob', avatar: uploadB.blob, }) - const profileB = await bobAgent.api.app.bsky.actor.profile.get({ - repo: bob.did, + const profileB = await agent.api.app.bsky.actor.profile.get({ + repo: bob, rkey: 'self', }) expect((profileB.value as any).avatar.cid).toEqual(uploadA.cid) const { data: uploadAfterPermanent } = - await aliceAgent.api.com.atproto.repo.uploadBlob(file, { + await agent.api.com.atproto.repo.uploadBlob(file, { + headers: sc.getHeaders(alice), encoding: 'image/jpeg', } as any) expect(uploadAfterPermanent).toEqual(uploadA) - const blob = await db.db + const blob = await aliceDb.db .selectFrom('blob') .selectAll() .where('cid', '=', uploadAfterPermanent.blob.ref.toString()) @@ -219,11 +204,12 @@ describe('file uploads', () => { }) it('supports compression during upload', async () => { - const { data: uploaded } = await aliceAgent.api.com.atproto.repo.uploadBlob( + const { data: uploaded } = await agent.api.com.atproto.repo.uploadBlob( gzipSync(smallFile), { encoding: 'image/jpeg', headers: { + ...sc.getHeaders(alice), 'content-encoding': 'gzip', }, } as any, @@ -233,11 +219,12 @@ describe('file uploads', () => { it('corrects a bad mimetype', async () => { const file = await fs.readFile('tests/sample-img/key-landscape-large.jpg') - const res = await aliceAgent.api.com.atproto.repo.uploadBlob(file, { + const res = await agent.api.com.atproto.repo.uploadBlob(file, { + headers: sc.getHeaders(alice), encoding: 'video/mp4', } as any) - const found = await db.db + const found = await aliceDb.db .selectFrom('blob') .selectAll() .where('cid', '=', res.data.blob.ref.toString()) @@ -250,11 +237,12 @@ describe('file uploads', () => { it('handles pngs', async () => { const file = await fs.readFile('tests/sample-img/at.png') - const res = await aliceAgent.api.com.atproto.repo.uploadBlob(file, { + const res = await agent.api.com.atproto.repo.uploadBlob(file, { + headers: sc.getHeaders(alice), encoding: 'image/png', }) - const found = await db.db + const found = await aliceDb.db .selectFrom('blob') .selectAll() .where('cid', '=', res.data.blob.ref.toString()) @@ -267,11 +255,12 @@ describe('file uploads', () => { it('handles unknown mimetypes', async () => { const file = await randomBytes(20000) - const res = await aliceAgent.api.com.atproto.repo.uploadBlob(file, { + const res = await agent.api.com.atproto.repo.uploadBlob(file, { + headers: sc.getHeaders(alice), encoding: 'test/fake', } as any) - const found = await db.db + const found = await aliceDb.db .selectFrom('blob') .selectAll() .where('cid', '=', res.data.blob.ref.toString()) @@ -280,12 +269,3 @@ describe('file uploads', () => { expect(found?.mimeType).toBe('test/fake') }) }) - -async function updateProfile(agent: AtpAgent, record: Record) { - return await agent.api.com.atproto.repo.putRecord({ - repo: agent.session?.did ?? '', - collection: ids.AppBskyActorProfile, - rkey: 'self', - record, - }) -} diff --git a/packages/pds/tests/handles.test.ts b/packages/pds/tests/handles.test.ts index 7c6833bdb78..3d6e5ecb41f 100644 --- a/packages/pds/tests/handles.test.ts +++ b/packages/pds/tests/handles.test.ts @@ -49,12 +49,8 @@ describe('handles', () => { }) const getHandleFromDb = async (did: string): Promise => { - const res = await ctx.db.db - .selectFrom('did_handle') - .selectAll() - .where('did', '=', did) - .executeTakeFirst() - return res?.handle + const res = await ctx.accountManager.getAccount(did) + return res?.handle ?? undefined } it('resolves handles', async () => { diff --git a/packages/pds/tests/invite-codes.test.ts b/packages/pds/tests/invite-codes.test.ts index e48e1b46fc7..90d16aad00b 100644 --- a/packages/pds/tests/invite-codes.test.ts +++ b/packages/pds/tests/invite-codes.test.ts @@ -126,8 +126,8 @@ describe('account', () => { // next, pretend account was made 2 days in the past const twoDaysAgo = new Date(Date.now() - 2 * DAY).toISOString() - await ctx.db.db - .updateTable('user_account') + await ctx.accountManager.db.db + .updateTable('actor') .set({ createdAt: twoDaysAgo }) .where('did', '=', account.did) .execute() @@ -150,8 +150,8 @@ describe('account', () => { // again, pretend account was made 2 days ago const twoDaysAgo = new Date(Date.now() - 2 * DAY).toISOString() - await ctx.db.db - .updateTable('user_account') + await ctx.accountManager.db.db + .updateTable('actor') .set({ createdAt: twoDaysAgo }) .where('did', '=', account.did) .execute() @@ -180,8 +180,8 @@ describe('account', () => { // first, pretend account was made 2 days ago & get those two codes const twoDaysAgo = new Date(Date.now() - 2 * DAY).toISOString() - await ctx.db.db - .updateTable('user_account') + await ctx.accountManager.db.db + .updateTable('actor') .set({ createdAt: twoDaysAgo }) .where('did', '=', account.did) .execute() @@ -192,8 +192,8 @@ describe('account', () => { // then pretend account was made ever so slightly over 10 days ago const tenDaysAgo = new Date(Date.now() - 10.01 * DAY).toISOString() - await ctx.db.db - .updateTable('user_account') + await ctx.accountManager.db.db + .updateTable('actor') .set({ createdAt: tenDaysAgo }) .where('did', '=', account.did) .execute() @@ -213,11 +213,14 @@ describe('account', () => { code: code, availableUses: 1, disabled: 0 as const, - forUser: account.did, + forAccount: account.did, createdBy: account.did, createdAt: new Date(Date.now() - 5 * DAY).toISOString(), })) - await ctx.db.db.insertInto('invite_code').values(inviteRows).execute() + await ctx.accountManager.db.db + .insertInto('invite_code') + .values(inviteRows) + .execute() const res3 = await account.agent.api.com.atproto.server.getAccountInviteCodes({ includeUsed: false, @@ -225,7 +228,7 @@ describe('account', () => { expect(res3.data.codes.length).toBe(10) // no we use the codes which should still not allow them to generate anymore - await ctx.db.db + await ctx.accountManager.db.db .insertInto('invite_code_use') .values( inviteRows.map((row) => ({ @@ -295,18 +298,18 @@ describe('account', () => { }, ) expect(res.data.codes.length).toBe(3) - const fromDb = await ctx.db.db + const fromDb = await ctx.accountManager.db.db .selectFrom('invite_code') .selectAll() - .where('forUser', 'in', accounts) + .where('forAccount', 'in', accounts) .execute() expect(fromDb.length).toBe(6) const dbCodesByUser = {} for (const row of fromDb) { expect(row.disabled).toBe(0) expect(row.availableUses).toBe(2) - dbCodesByUser[row.forUser] ??= [] - dbCodesByUser[row.forUser].push(row.code) + dbCodesByUser[row.forAccount] ??= [] + dbCodesByUser[row.forAccount].push(row.code) } for (const { account, codes } of res.data.codes) { expect(codes.length).toBe(2) diff --git a/packages/pds/tests/invites-admin.test.ts b/packages/pds/tests/invites-admin.test.ts index d971b75285c..16b1f6df5e4 100644 --- a/packages/pds/tests/invites-admin.test.ts +++ b/packages/pds/tests/invites-admin.test.ts @@ -203,9 +203,8 @@ describe('pds admin invite views', () => { }) it('disables an account from getting additional invite codes', async () => { - const reasonForDisabling = 'User is selling invites' await agent.api.com.atproto.admin.disableAccountInvites( - { account: carol, note: reasonForDisabling }, + { account: carol }, { encoding: 'application/json', headers: network.pds.adminAuthHeaders() }, ) @@ -214,7 +213,6 @@ describe('pds admin invite views', () => { { headers: network.pds.adminAuthHeaders() }, ) expect(repoRes.data.invitesDisabled).toBe(true) - expect(repoRes.data.inviteNote).toBe(reasonForDisabling) const invRes = await agent.api.com.atproto.server.getAccountInviteCodes( {}, @@ -224,10 +222,8 @@ describe('pds admin invite views', () => { }) it('allows setting reason when enabling and disabling invite codes', async () => { - const reasonForEnabling = 'User is confirmed they will play nice' - const reasonForDisabling = 'User is selling invites' await agent.api.com.atproto.admin.enableAccountInvites( - { account: carol, note: reasonForEnabling }, + { account: carol }, { encoding: 'application/json', headers: network.pds.adminAuthHeaders() }, ) @@ -236,10 +232,9 @@ describe('pds admin invite views', () => { { headers: network.pds.adminAuthHeaders() }, ) expect(afterEnable.data.invitesDisabled).toBe(false) - expect(afterEnable.data.inviteNote).toBe(reasonForEnabling) await agent.api.com.atproto.admin.disableAccountInvites( - { account: carol, note: reasonForDisabling }, + { account: carol }, { encoding: 'application/json', headers: network.pds.adminAuthHeaders() }, ) @@ -248,13 +243,12 @@ describe('pds admin invite views', () => { { headers: network.pds.adminAuthHeaders() }, ) expect(afterDisable.data.invitesDisabled).toBe(true) - expect(afterDisable.data.inviteNote).toBe(reasonForDisabling) }) it('creates codes in the background but disables them', async () => { - const res = await network.pds.ctx.db.db + const res = await network.pds.ctx.accountManager.db.db .selectFrom('invite_code') - .where('forUser', '=', carol) + .where('forAccount', '=', carol) .selectAll() .execute() expect(res.length).toBe(5) diff --git a/packages/pds/tests/moderation.test.ts b/packages/pds/tests/moderation.test.ts index ee68bb7aab5..562795af581 100644 --- a/packages/pds/tests/moderation.test.ts +++ b/packages/pds/tests/moderation.test.ts @@ -212,22 +212,24 @@ describe('moderation', () => { }) it('removes blob from the store', async () => { - const tryGetBytes = network.pds.ctx.blobstore.getBytes(blobRef.image.ref) + const tryGetBytes = network.pds.ctx + .blobstore(blobSubject.did) + .getBytes(blobRef.image.ref) await expect(tryGetBytes).rejects.toThrow(BlobNotFoundError) }) it('prevents blob from being referenced again.', async () => { const uploaded = await sc.uploadFile( - sc.dids.alice, + sc.dids.carol, 'tests/sample-img/key-alt.jpg', 'image/jpeg', ) expect(uploaded.image.ref.equals(blobRef.image.ref)).toBeTruthy() - const referenceBlob = sc.post(sc.dids.alice, 'pic', [], [blobRef]) + const referenceBlob = sc.post(sc.dids.carol, 'pic', [], [blobRef]) await expect(referenceBlob).rejects.toThrow('Could not find blob:') }) - it('prevents image blob from being served, even when cached.', async () => { + it('prevents image blob from being served.', async () => { const attempt = agent.api.com.atproto.sync.getBlob({ did: sc.dids.carol, cid: blobRef.image.ref.toString(), @@ -248,7 +250,7 @@ describe('moderation', () => { ) // Can post and reference blob - const post = await sc.post(sc.dids.alice, 'pic', [], [blobRef]) + const post = await sc.post(sc.dids.carol, 'pic', [], [blobRef]) expect(post.images[0].image.ref.equals(blobRef.image.ref)).toBeTruthy() // Can fetch through image server @@ -259,6 +261,63 @@ describe('moderation', () => { expect(res.data.byteLength).toBeGreaterThan(9000) }) + + it('prevents blobs of takendown accounts from being served.', async () => { + await agent.api.com.atproto.admin.updateSubjectStatus( + { + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.carol, + }, + takedown: { applied: true }, + }, + { + encoding: 'application/json', + headers: network.pds.adminAuthHeaders(), + }, + ) + const blobParams = { + did: sc.dids.carol, + cid: blobRef.image.ref.toString(), + } + // public, disallow + const attempt1 = agent.api.com.atproto.sync.getBlob(blobParams) + await expect(attempt1).rejects.toThrow('Blob not found') + // logged-in, disallow + const attempt2 = agent.api.com.atproto.sync.getBlob(blobParams, { + headers: sc.getHeaders(sc.dids.bob), + }) + await expect(attempt2).rejects.toThrow('Blob not found') + // non-admin role, disallow + const attempt3 = agent.api.com.atproto.sync.getBlob(blobParams, { + headers: network.pds.adminAuthHeaders('moderator'), + }) + await expect(attempt3).rejects.toThrow('Blob not found') + // logged-in as account, allow + const res1 = await agent.api.com.atproto.sync.getBlob(blobParams, { + headers: sc.getHeaders(sc.dids.carol), + }) + expect(res1.data.byteLength).toBeGreaterThan(9000) + // admin role, allow + const res2 = await agent.api.com.atproto.sync.getBlob(blobParams, { + headers: network.pds.adminAuthHeaders('admin'), + }) + expect(res2.data.byteLength).toBeGreaterThan(9000) + // revert takedown + await agent.api.com.atproto.admin.updateSubjectStatus( + { + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.carol, + }, + takedown: { applied: false }, + }, + { + encoding: 'application/json', + headers: network.pds.adminAuthHeaders(), + }, + ) + }) }) describe('auth', () => { diff --git a/packages/pds/tests/preferences.test.ts b/packages/pds/tests/preferences.test.ts index 77dc256f85f..3af5cba0384 100644 --- a/packages/pds/tests/preferences.test.ts +++ b/packages/pds/tests/preferences.test.ts @@ -42,16 +42,12 @@ describe('user preferences', () => { }) it('only gets preferences in app.bsky namespace.', async () => { - const { db, services } = network.pds.ctx - await db.transaction(async (tx) => { - await services - .account(tx) - .putPreferences( - sc.dids.alice, - [{ $type: 'com.atproto.server.defs#unknown' }], - 'com.atproto', - ) - }) + await network.pds.ctx.actorStore.transact(sc.dids.alice, (store) => + store.pref.putPreferences( + [{ $type: 'com.atproto.server.defs#unknown' }], + 'com.atproto', + ), + ) const { data } = await agent.api.app.bsky.actor.getPreferences( {}, { headers: sc.getHeaders(sc.dids.alice) }, @@ -98,10 +94,10 @@ describe('user preferences', () => { ], }) // Ensure other prefs were not clobbered - const { db, services } = network.pds.ctx - const otherPrefs = await services - .account(db) - .getPreferences(sc.dids.alice, 'com.atproto') + const otherPrefs = await network.pds.ctx.actorStore.read( + sc.dids.alice, + (store) => store.pref.getPreferences('com.atproto'), + ) expect(otherPrefs).toEqual([{ $type: 'com.atproto.server.defs#unknown' }]) }) diff --git a/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap b/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap index 15c63498ac1..85e2e3b99e5 100644 --- a/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap +++ b/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap @@ -195,11 +195,11 @@ Object { }, Object { "usedAt": "1970-01-01T00:00:00.000Z", - "usedBy": "user(0)", + "usedBy": "user(2)", }, Object { "usedAt": "1970-01-01T00:00:00.000Z", - "usedBy": "user(2)", + "usedBy": "user(0)", }, Object { "usedAt": "1970-01-01T00:00:00.000Z", diff --git a/packages/pds/tests/proxied/admin.test.ts b/packages/pds/tests/proxied/admin.test.ts index a51ec048c2d..fd8538e802a 100644 --- a/packages/pds/tests/proxied/admin.test.ts +++ b/packages/pds/tests/proxied/admin.test.ts @@ -317,18 +317,4 @@ describe('proxies admin requests', () => { expect.objectContaining({ uri: post.ref.uriStr, cid: post.ref.cidStr }), ) }) - - it('does not persist actions and reports on pds.', async () => { - const { db } = network.pds.ctx - const actions = await db.db - .selectFrom('moderation_action') - .selectAll() - .execute() - const reports = await db.db - .selectFrom('moderation_report') - .selectAll() - .execute() - expect(actions).toEqual([]) - expect(reports).toEqual([]) - }) }) diff --git a/packages/pds/tests/proxied/notif.test.ts b/packages/pds/tests/proxied/notif.test.ts index 4fc559ee120..62351f1a2b5 100644 --- a/packages/pds/tests/proxied/notif.test.ts +++ b/packages/pds/tests/proxied/notif.test.ts @@ -68,7 +68,10 @@ describe('notif service proxy', () => { const auth = await verifyJwt( spy.current?.['jwt'] as string, notifDid, - async () => network.pds.ctx.repoSigningKey.did(), + async (did) => { + const keypair = await network.pds.ctx.actorStore.keypair(did) + return keypair.did() + }, ) expect(auth.iss).toEqual(sc.dids.bob) }) diff --git a/packages/pds/tests/races.test.ts b/packages/pds/tests/races.test.ts index 220e9c252c8..fc65dab664c 100644 --- a/packages/pds/tests/races.test.ts +++ b/packages/pds/tests/races.test.ts @@ -1,17 +1,17 @@ import AtpAgent from '@atproto/api' import { wait } from '@atproto/common' import { TestNetworkNoAppView } from '@atproto/dev-env' -import { CommitData, readCarWithRoot, verifyRepo } from '@atproto/repo' +import { readCarWithRoot, verifyRepo } from '@atproto/repo' import AppContext from '../src/context' -import { PreparedWrite, prepareCreate } from '../src/repo' -import SqlRepoStorage from '../src/sql-repo-storage' -import { ConcurrentWriteError } from '../src/services/repo' +import { PreparedCreate, prepareCreate } from '../src/repo' +import { Keypair } from '@atproto/crypto' -describe('crud operations', () => { +describe('races', () => { let network: TestNetworkNoAppView let ctx: AppContext let agent: AtpAgent let did: string + let signingKey: Keypair beforeAll(async () => { network = await TestNetworkNoAppView.create({ @@ -25,58 +25,40 @@ describe('crud operations', () => { password: 'alice-pass', }) did = agent.session?.did || '' + signingKey = await network.pds.ctx.actorStore.keypair(did) }) afterAll(async () => { await network.close() }) - const formatWrite = async () => { - const write = await prepareCreate({ - did, - collection: 'app.bsky.feed.post', - record: { - text: 'one', - createdAt: new Date().toISOString(), - }, - validate: true, - }) - const storage = new SqlRepoStorage(ctx.db, did) - const commit = await ctx.services - .repo(ctx.db) - .formatCommit(storage, did, [write]) - return { write, commit } - } - const processCommitWithWait = async ( did: string, - writes: PreparedWrite[], - commitData: CommitData, + write: PreparedCreate, waitMs: number, ) => { const now = new Date().toISOString() - await ctx.db.transaction(async (dbTxn) => { - const storage = new SqlRepoStorage(dbTxn, did, now) - const locked = await storage.lockRepo() - if (!locked) { - throw new ConcurrentWriteError() - } + return ctx.actorStore.transact(did, async (store) => { + const commitData = await store.repo.formatCommit([write]) + await store.repo.storage.applyCommit(commitData) await wait(waitMs) - const srvc = ctx.services.repo(dbTxn) - await Promise.all([ - // persist the commit to repo storage - storage.applyCommit(commitData), - // & send to indexing - srvc.indexWrites(writes, now), - // do any other processing needed after write - srvc.afterWriteProcessing(did, commitData, writes), - ]) + await store.repo.indexWrites([write], now) + return write }) } it('handles races in record routes', async () => { - const { write, commit } = await formatWrite() - const processPromise = processCommitWithWait(did, [write], commit, 500) + const write = await prepareCreate({ + did, + collection: 'app.bsky.feed.post', + record: { + text: 'one', + createdAt: new Date().toISOString(), + }, + validate: true, + }) + + const processPromise = processCommitWithWait(did, write, 500) const createdPost = await agent.api.app.bsky.feed.post.create( { repo: did }, @@ -94,10 +76,10 @@ describe('crud operations', () => { car.blocks, car.root, did, - ctx.repoSigningKey.did(), + signingKey.did(), ) expect(verified.creates.length).toBe(2) - expect(verified.creates[0].cid.equals(write.cid)).toBeTruthy() + expect(verified.creates[0].cid.toString()).toEqual(write.cid.toString()) expect(verified.creates[1].cid.toString()).toEqual( createdPost.cid.toString(), ) diff --git a/packages/pds/tests/seeds/basic.ts b/packages/pds/tests/seeds/basic.ts index bce8c1b3b92..1085e2b381e 100644 --- a/packages/pds/tests/seeds/basic.ts +++ b/packages/pds/tests/seeds/basic.ts @@ -106,6 +106,8 @@ export default async ( 'tests/sample-img/key-landscape-small.jpg', 'image/jpeg', ) + // must ensure ordering of replies in indexing + await sc.network.processAll() await sc.reply( bob, sc.posts[alice][1].ref, @@ -120,6 +122,7 @@ export default async ( sc.posts[alice][1].ref, replies.carol[0], ) + await sc.network.processAll() const alicesReplyToBob = await sc.reply( alice, sc.posts[alice][1].ref, diff --git a/packages/pds/tests/seeds/users.ts b/packages/pds/tests/seeds/users.ts index a142954ac68..fa169bcf826 100644 --- a/packages/pds/tests/seeds/users.ts +++ b/packages/pds/tests/seeds/users.ts @@ -28,7 +28,7 @@ export default async (sc: SeedClient, opts?: { inviteCode?: string }) => { return sc } -const users = { +export const users = { alice: { email: 'alice@test.com', handle: 'alice.test', diff --git a/packages/pds/tests/sequencer.test.ts b/packages/pds/tests/sequencer.test.ts index d48ba1797d6..3d9fed7a152 100644 --- a/packages/pds/tests/sequencer.test.ts +++ b/packages/pds/tests/sequencer.test.ts @@ -3,12 +3,10 @@ import { randomStr } from '@atproto/crypto' import { cborEncode, readFromGenerator, wait } from '@atproto/common' import { Sequencer, SeqEvt } from '../src/sequencer' import Outbox from '../src/sequencer/outbox' -import { Database } from '../src' import userSeed from './seeds/users' describe('sequencer', () => { let network: TestNetworkNoAppView - let db: Database let sequencer: Sequencer let sc: SeedClient let alice: string @@ -21,7 +19,6 @@ describe('sequencer', () => { network = await TestNetworkNoAppView.create({ dbPostgresSchema: 'sequencer', }) - db = network.pds.ctx.db sequencer = network.pds.ctx.sequencer sc = network.getSeedClient() await userSeed(sc) @@ -49,7 +46,7 @@ describe('sequencer', () => { } const loadFromDb = (lastSeen: number) => { - return db.db + return sequencer.db.db .selectFrom('repo_seq') .select([ 'seq', @@ -78,11 +75,9 @@ describe('sequencer', () => { const caughtUp = (outbox: Outbox): (() => Promise) => { return async () => { - const leaderCaughtUp = await network.pds.ctx.sequencerLeader?.isCaughtUp() - if (!leaderCaughtUp) return false const lastEvt = await outbox.sequencer.curr() - if (!lastEvt) return true - return outbox.lastSeen >= (lastEvt.seq ?? 0) + if (lastEvt === null) return true + return outbox.lastSeen >= (lastEvt ?? 0) } } @@ -183,6 +178,8 @@ describe('sequencer', () => { } await expect(overloadBuffer).rejects.toThrow('Stream consumer too slow') + await createPromise + const fromDb = await loadFromDb(lastSeen) lastSeen = fromDb.at(-1)?.seq ?? lastSeen }) diff --git a/packages/pds/tests/server.test.ts b/packages/pds/tests/server.test.ts index 23298a7d731..fff61fbede1 100644 --- a/packages/pds/tests/server.test.ts +++ b/packages/pds/tests/server.test.ts @@ -1,28 +1,25 @@ import { AddressInfo } from 'net' import express from 'express' import axios, { AxiosError } from 'axios' -import { TestNetwork, SeedClient } from '@atproto/dev-env' +import { SeedClient, TestNetworkNoAppView } from '@atproto/dev-env' import AtpAgent, { AtUri } from '@atproto/api' import { handler as errorHandler } from '../src/error' import basicSeed from './seeds/basic' -import { Database } from '../src' import { randomStr } from '@atproto/crypto' describe('server', () => { - let network: TestNetwork - let db: Database + let network: TestNetworkNoAppView let agent: AtpAgent let sc: SeedClient let alice: string beforeAll(async () => { - network = await TestNetwork.create({ + network = await TestNetworkNoAppView.create({ dbPostgresSchema: 'server', pds: { version: '0.0.0', }, }) - db = network.pds.ctx.db agent = network.pds.getClient() sc = network.getSeedClient() await basicSeed(sc) @@ -138,11 +135,9 @@ describe('server', () => { expect(data).toEqual({ version: '0.0.0' }) }) - it('healthcheck fails when database is unavailable.', async () => { - // destroy to release lock & allow db to close - await network.pds.ctx.sequencerLeader?.destroy() - - await db.close() + // @TODO this is hanging for some unknown reason + it.skip('healthcheck fails when database is unavailable.', async () => { + await network.pds.ctx.accountManager.db.close() let error: AxiosError try { await axios.get(`${network.pds.url}/xrpc/_health`) diff --git a/packages/pds/tests/sql-repo-storage.test.ts b/packages/pds/tests/sql-repo-storage.test.ts deleted file mode 100644 index ea63cf07e06..00000000000 --- a/packages/pds/tests/sql-repo-storage.test.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { TestNetworkNoAppView } from '@atproto/dev-env' -import { range, dataToCborBlock, TID } from '@atproto/common' -import { CidSet, def } from '@atproto/repo' -import BlockMap from '@atproto/repo/src/block-map' -import { CID } from 'multiformats/cid' -import { Database } from '../src' -import SqlRepoStorage from '../src/sql-repo-storage' - -describe('sql repo storage', () => { - let network: TestNetworkNoAppView - let db: Database - - beforeAll(async () => { - network = await TestNetworkNoAppView.create({ - dbPostgresSchema: 'sql_repo_storage', - }) - db = network.pds.ctx.db - }) - - afterAll(async () => { - await network.close() - }) - - it('puts and gets blocks.', async () => { - const did = 'did:key:zQ3shokFTS3brHcDQrn82RUDfCZESWL1ZdCEJwekUDPQiYBme' - - const cid = await db.transaction(async (dbTxn) => { - const storage = new SqlRepoStorage(dbTxn, did) - const block = await dataToCborBlock({ my: 'block' }) - await storage.putBlock(block.cid, block.bytes, TID.nextStr()) - return block.cid - }) - - const storage = new SqlRepoStorage(db, did) - const value = await storage.readObj(cid, def.unknown) - - expect(value).toEqual({ my: 'block' }) - }) - - it('allows same content to be put multiple times by the same did.', async () => { - const did = 'did:key:zQ3shtxV1FrJfhqE1dvxYRcCknWNjHc3c5X1y3ZSoPDi2aur2' - - const cidA = await db.transaction(async (dbTxn) => { - const storage = new SqlRepoStorage(dbTxn, did) - const block = await dataToCborBlock({ my: 'block' }) - await storage.putBlock(block.cid, block.bytes, TID.nextStr()) - return block.cid - }) - - const cidB = await db.transaction(async (dbTxn) => { - const storage = new SqlRepoStorage(dbTxn, did) - const block = await dataToCborBlock({ my: 'block' }) - await storage.putBlock(block.cid, block.bytes, TID.nextStr()) - return block.cid - }) - - expect(cidA.equals(cidB)).toBe(true) - }) - - it('applies commits', async () => { - const did = 'did:key:zQ3shtxV1FrJfhqE1dvxYRcCknWNjHc3c5X1y3ZSoPDi2aur3' - const blocks = await Promise.all( - range(10).map((num) => dataToCborBlock({ my: `block-${num}` })), - ) - const commits = await Promise.all( - range(2).map((num) => dataToCborBlock({ my: `commit-${num}` })), - ) - const blocks0 = new BlockMap() - blocks0.set(commits[0].cid, commits[0].bytes) - blocks.slice(0, 5).forEach((block) => { - blocks0.set(block.cid, block.bytes) - }) - const blocks1 = new BlockMap() - blocks1.set(commits[1].cid, commits[1].bytes) - blocks.slice(5, 10).forEach((block) => { - blocks1.set(block.cid, block.bytes) - }) - const toRemoveList = blocks0 - .entries() - .slice(0, 2) - .map((b) => b.cid) - const toRemove = new CidSet(toRemoveList) - await db.transaction(async (dbTxn) => { - const storage = new SqlRepoStorage(dbTxn, did) - await storage.applyCommit({ - cid: commits[0].cid, - rev: TID.nextStr(), - prev: null, - since: null, - newBlocks: blocks0, - removedCids: new CidSet(), - }) - await storage.applyCommit({ - cid: commits[1].cid, - rev: TID.nextStr(), - prev: commits[0].cid, - since: null, - newBlocks: blocks1, - removedCids: toRemove, - }) - }) - - const storage = new SqlRepoStorage(db, did) - const head = await storage.getRoot() - if (!head) { - throw new Error('could not get repo head') - } - expect(head.toString()).toEqual(commits[1].cid.toString()) - - const cidsRes = await db.db - .selectFrom('ipld_block') - .where('creator', '=', did) - .select('cid') - .execute() - const allCids = new CidSet(cidsRes.map((row) => CID.parse(row.cid))) - for (const entry of blocks1.entries()) { - expect(allCids.has(entry.cid)).toBe(true) - } - for (const entry of blocks0.entries()) { - const shouldHave = !toRemove.has(entry.cid) - expect(allCids.has(entry.cid)).toBe(shouldHave) - } - }) -}) diff --git a/packages/pds/tests/sync/subscribe-repos.test.ts b/packages/pds/tests/sync/subscribe-repos.test.ts index 58745b7fe1e..bc877ec11ee 100644 --- a/packages/pds/tests/sync/subscribe-repos.test.ts +++ b/packages/pds/tests/sync/subscribe-repos.test.ts @@ -17,7 +17,7 @@ import { Handle as HandleEvt, Tombstone as TombstoneEvt, } from '../../src/lexicon/types/com/atproto/sync/subscribeRepos' -import { AppContext, Database } from '../../src' +import { AppContext } from '../../src' import basicSeed from '../seeds/basic' import { CID } from 'multiformats/cid' @@ -25,7 +25,6 @@ describe('repo subscribe repos', () => { let serverHost: string let network: TestNetworkNoAppView - let db: Database let ctx: AppContext let agent: AtpAgent @@ -44,7 +43,6 @@ describe('repo subscribe repos', () => { }) serverHost = network.pds.url.replace('http://', '') ctx = network.pds.ctx - db = network.pds.ctx.db agent = network.pds.getClient() sc = network.getSeedClient() await basicSeed(sc) @@ -61,7 +59,8 @@ describe('repo subscribe repos', () => { const getRepo = async (did: string): Promise => { const carRes = await agent.api.com.atproto.sync.getRepo({ did }) const car = await repo.readCarWithRoot(carRes.data) - return repo.verifyRepo(car.blocks, car.root, did, ctx.repoSigningKey.did()) + const signingKey = await network.pds.ctx.actorStore.keypair(did) + return repo.verifyRepo(car.blocks, car.root, did, signingKey.did()) } const getHandleEvts = (frames: Frame[]): HandleEvt[] => { @@ -74,6 +73,25 @@ describe('repo subscribe repos', () => { return evts } + const getAllEvents = (userDid: string, frames: Frame[]) => { + const types: unknown[] = [] + for (const frame of frames) { + if (frame instanceof MessageFrame) { + if ( + (frame.header.t === '#commit' && + (frame.body as CommitEvt).repo === userDid) || + (frame.header.t === '#handle' && + (frame.body as HandleEvt).did === userDid) || + (frame.header.t === '#tombstone' && + (frame.body as TombstoneEvt).did === userDid) + ) { + types.push(frame.body) + } + } + } + return types + } + const getTombstoneEvts = (frames: Frame[]): TombstoneEvt[] => { const evts: TombstoneEvt[] = [] for (const frame of frames) { @@ -110,25 +128,6 @@ describe('repo subscribe repos', () => { return evts } - const getAllEvents = (userDid: string, frames: Frame[]) => { - const types: unknown[] = [] - for (const frame of frames) { - if (frame instanceof MessageFrame) { - if ( - (frame.header.t === '#commit' && - (frame.body as CommitEvt).repo === userDid) || - (frame.header.t === '#handle' && - (frame.body as HandleEvt).did === userDid) || - (frame.header.t === '#tombstone' && - (frame.body as TombstoneEvt).did === userDid) - ) { - types.push(frame.body) - } - } - } - return types - } - const verifyCommitEvents = async (frames: Frame[]) => { await verifyRepo(alice, getCommitEvents(alice, frames)) await verifyRepo(bob, getCommitEvents(bob, frames)) @@ -164,11 +163,12 @@ describe('repo subscribe repos', () => { if (!lastCommit) { throw new Error('no last commit') } + const signingKey = await network.pds.ctx.actorStore.keypair(did) const fromStream = await repo.verifyRepo( allBlocks, lastCommit, did, - ctx.repoSigningKey.did(), + signingKey.did(), ) const fromRpcOps = fromRpc.creates const fromStreamOps = fromStream.creates @@ -199,16 +199,8 @@ describe('repo subscribe repos', () => { const isDone = async (evt: any) => { if (evt === undefined) return false if (evt instanceof ErrorFrame) return true - const caughtUp = await ctx.sequencerLeader?.isCaughtUp() - if (!caughtUp) return false - const curr = await db.db - .selectFrom('repo_seq') - .where('seq', 'is not', null) - .select('seq') - .limit(1) - .orderBy('seq', 'desc') - .executeTakeFirst() - return curr !== undefined && evt.body.seq === curr.seq + const curr = await ctx.sequencer.curr() + return evt.body.seq === curr } return readFromGenerator(gen, isDone, waitFor) @@ -271,9 +263,8 @@ describe('repo subscribe repos', () => { }) it('backfills only from provided cursor', async () => { - const seqs = await db.db + const seqs = await ctx.sequencer.db.db .selectFrom('repo_seq') - .where('seq', 'is not', null) .selectAll() .orderBy('seq', 'asc') .execute() @@ -348,9 +339,7 @@ describe('repo subscribe repos', () => { ).did for (const did of [baddie1, baddie2]) { - await ctx.services.record(db).deleteForActor(did) - await ctx.services.repo(db).deleteRepo(did) - await ctx.services.account(db).deleteAccount(did) + await ctx.sequencer.sequenceTombstone(did) } const ws = new WebSocket( @@ -368,7 +357,7 @@ describe('repo subscribe repos', () => { it('account deletions invalidate all seq ops', async () => { const baddie3 = ( - await sc.createAccount('baddie3.test', { + await sc.createAccount('baddie3', { email: 'baddie3@test.com', handle: 'baddie3.test', password: 'baddie3-pass', @@ -377,10 +366,15 @@ describe('repo subscribe repos', () => { await randomPost(baddie3) await sc.updateHandle(baddie3, 'baddie3-update.test') - - await ctx.services.record(db).deleteForActor(baddie3) - await ctx.services.repo(db).deleteRepo(baddie3) - await ctx.services.account(db).deleteAccount(baddie3) + const token = await network.pds.ctx.accountManager.createEmailToken( + baddie3, + 'delete_account', + ) + await agent.api.com.atproto.server.deleteAccount({ + token, + did: baddie3, + password: sc.accounts[baddie3].password, + }) const ws = new WebSocket( `ws://${serverHost}/xrpc/com.atproto.sync.subscribeRepos?cursor=${-1}`, @@ -395,33 +389,11 @@ describe('repo subscribe repos', () => { verifyTombstoneEvent(didEvts[0], baddie3) }) - it('does not return invalidated events', async () => { - await sc.updateHandle(alice, 'alice3.test') - await sc.updateHandle(alice, 'alice4.test') - await sc.updateHandle(alice, 'alice5.test') - await sc.updateHandle(bob, 'bob3.test') - await sc.updateHandle(bob, 'bob4.test') - await sc.updateHandle(bob, 'bob5.test') - - const ws = new WebSocket( - `ws://${serverHost}/xrpc/com.atproto.sync.subscribeRepos?cursor=${-1}`, - ) - - const gen = byFrame(ws) - const evts = await readTillCaughtUp(gen) - ws.terminate() - - const handleEvts = getHandleEvts(evts) - expect(handleEvts.length).toBe(2) - verifyHandleEvent(handleEvts[0], alice, 'alice5.test') - verifyHandleEvent(handleEvts[1], bob, 'bob5.test') - }) - it('sends info frame on out of date cursor', async () => { // we rewrite the sequenceAt time for existing seqs to be past the backfill cutoff // then we create some new posts const overAnHourAgo = new Date(Date.now() - HOUR - MINUTE).toISOString() - await db.db + await ctx.sequencer.db.db .updateTable('repo_seq') .set({ sequencedAt: overAnHourAgo }) .execute() diff --git a/packages/pds/tests/sync/sync.test.ts b/packages/pds/tests/sync/sync.test.ts index 4f99b3bb08c..e71f8093a05 100644 --- a/packages/pds/tests/sync/sync.test.ts +++ b/packages/pds/tests/sync/sync.test.ts @@ -1,30 +1,28 @@ import { TestNetworkNoAppView, SeedClient } from '@atproto/dev-env' import AtpAgent from '@atproto/api' import { TID } from '@atproto/common' -import { randomStr } from '@atproto/crypto' +import { Keypair, randomStr } from '@atproto/crypto' import * as repo from '@atproto/repo' import { MemoryBlockstore } from '@atproto/repo' import { AtUri } from '@atproto/syntax' import { CID } from 'multiformats/cid' -import { AppContext } from '../../src' describe('repo sync', () => { let network: TestNetworkNoAppView let agent: AtpAgent let sc: SeedClient let did: string + let signingKey: Keypair const repoData: repo.RepoContents = {} const uris: AtUri[] = [] const storage = new MemoryBlockstore() let currRoot: CID | undefined - let ctx: AppContext beforeAll(async () => { network = await TestNetworkNoAppView.create({ dbPostgresSchema: 'repo_sync', }) - ctx = network.pds.ctx agent = network.pds.getClient() sc = network.getSeedClient() await sc.createAccount('alice', { @@ -33,6 +31,7 @@ describe('repo sync', () => { password: 'alice-pass', }) did = sc.dids.alice + signingKey = await network.pds.ctx.actorStore.keypair(did) }) afterAll(async () => { @@ -56,7 +55,7 @@ describe('repo sync', () => { car.blocks, car.root, did, - ctx.repoSigningKey.did(), + signingKey.did(), ) await storage.applyCommit(synced.commit) expect(synced.creates.length).toBe(ADD_COUNT) @@ -93,7 +92,7 @@ describe('repo sync', () => { car.blocks, car.root, did, - ctx.repoSigningKey.did(), + signingKey.did(), ) expect(synced.writes.length).toBe(ADD_COUNT) // -2 because of dels of new records, +2 because of dels of old records await storage.applyCommit(synced.commit) @@ -131,7 +130,7 @@ describe('repo sync', () => { car.blocks, car.root, did, - ctx.repoSigningKey.did(), + signingKey.did(), ) expect(synced.writes.length).toBe(1) await storage.applyCommit(synced.commit) @@ -153,7 +152,7 @@ describe('repo sync', () => { const records = await repo.verifyRecords( new Uint8Array(car.data), did, - ctx.repoSigningKey.did(), + signingKey.did(), ) const claim = { collection, @@ -166,7 +165,7 @@ describe('repo sync', () => { new Uint8Array(car.data), [claim], did, - ctx.repoSigningKey.did(), + signingKey.did(), ) expect(result.verified.length).toBe(1) expect(result.unverified.length).toBe(0) @@ -189,7 +188,7 @@ describe('repo sync', () => { new Uint8Array(car.data), [claim], did, - ctx.repoSigningKey.did(), + signingKey.did(), ) expect(result.verified.length).toBe(1) expect(result.unverified.length).toBe(0) diff --git a/packages/pds/tests/transfer-repo.test.ts b/packages/pds/tests/transfer-repo.test.ts new file mode 100644 index 00000000000..f2e6dca0bfd --- /dev/null +++ b/packages/pds/tests/transfer-repo.test.ts @@ -0,0 +1,217 @@ +import path from 'node:path' +import os from 'node:os' +import axios from 'axios' +import * as ui8 from 'uint8arrays' +import { SeedClient, TestPds, TestPlc, mockResolvers } from '@atproto/dev-env' +import AtpAgent from '@atproto/api' +import * as pdsEntryway from '@atproto/pds-entryway' +import { Secp256k1Keypair, randomStr } from '@atproto/crypto' +import * as plcLib from '@did-plc/lib' +import getPort from 'get-port' + +describe('transfer repo', () => { + let plc: TestPlc + let pds: TestPds + let entryway: pdsEntryway.PDS + let entrywaySc: SeedClient + let pdsAgent: AtpAgent + let entrywayAgent: AtpAgent + + let did: string + const accountDetail = { + email: 'alice@test.com', + handle: 'alice.test', + password: 'test123', + } + + beforeAll(async () => { + const jwtSigningKey = await Secp256k1Keypair.create({ exportable: true }) + const plcRotationKey = await Secp256k1Keypair.create({ exportable: true }) + const entrywayPort = await getPort() + plc = await TestPlc.create({}) + pds = await TestPds.create({ + entrywayUrl: `http://localhost:${entrywayPort}`, + entrywayDid: 'did:example:entryway', + entrywayJwtVerifyKeyK256PublicKeyHex: getPublicHex(jwtSigningKey), + entrywayPlcRotationKey: plcRotationKey.did(), + adminPassword: 'admin-pass', + serviceHandleDomains: [], + didPlcUrl: plc.url, + serviceDid: 'did:example:pds', + inviteRequired: false, + }) + entryway = await createEntryway({ + dbPostgresSchema: 'transfer_repo', + port: entrywayPort, + adminPassword: 'admin-pass', + jwtSigningKeyK256PrivateKeyHex: await getPrivateHex(jwtSigningKey), + plcRotationKeyK256PrivateKeyHex: await getPrivateHex(plcRotationKey), + inviteRequired: false, + serviceDid: 'did:example:entryway', + didPlcUrl: plc.url, + }) + mockResolvers(pds.ctx.idResolver, pds) + mockResolvers(entryway.ctx.idResolver, pds) + await entryway.ctx.db.db + .insertInto('pds') + .values({ + did: pds.ctx.cfg.service.did, + host: new URL(pds.ctx.cfg.service.publicUrl).host, + weight: 0, + }) + .execute() + pdsAgent = pds.getClient() + entrywayAgent = new AtpAgent({ + service: entryway.ctx.cfg.service.publicUrl, + }) + + // @ts-ignore network not needed + entrywaySc = new SeedClient({}, entrywayAgent) + await entrywaySc.createAccount('alice', accountDetail) + did = entrywaySc.dids.alice + for (let i = 0; i < 50; i++) { + const post = await entrywaySc.post(did, 'blah') + await entrywaySc.like(did, post.ref) + } + const img = await entrywaySc.uploadFile( + did, + 'tests/sample-img/key-landscape-small.jpg', + 'image/jpeg', + ) + await entrywaySc.post(did, 'img post', undefined, [img]) + }) + + afterAll(async () => { + await plc.close() + await entryway.destroy() + await pds.close() + }) + + it('transfers', async () => { + const signingKeyRes = + await pdsAgent.api.com.atproto.server.reserveSigningKey({ did }) + const signingKey = signingKeyRes.data.signingKey + + const repo = await entrywayAgent.api.com.atproto.sync.getRepo({ did }) + const importRes = await axios.post( + `${pds.url}/xrpc/com.atproto.temp.importRepo`, + repo.data, + { + params: { did }, + headers: { + 'content-type': 'application/vnd.ipld.car', + ...pds.adminAuthHeaders('admin'), + }, + decompress: true, + responseType: 'stream', + }, + ) + + for await (const _log of importRes.data) { + // noop just wait till import is finished + } + + const lastOp = await pds.ctx.plcClient.getLastOp(did) + if (!lastOp || lastOp.type === 'plc_tombstone') { + throw new Error('could not find last plc op') + } + const plcOp = await plcLib.createUpdateOp( + lastOp, + entryway.ctx.plcRotationKey, + (normalized) => ({ + ...normalized, + rotationKeys: [pds.ctx.plcRotationKey.did()], + verificationMethods: { + atproto: signingKey, + }, + services: { + atproto_pds: { + type: 'AtprotoPersonalDataServer', + endpoint: pds.ctx.cfg.service.publicUrl, + }, + }, + }), + ) + await pdsAgent.api.com.atproto.temp.transferAccount( + { + did, + handle: accountDetail.handle, + plcOp, + }, + { headers: pds.adminAuthHeaders('admin'), encoding: 'application/json' }, + ) + + await entryway.ctx.db.db + .updateTable('user_account') + .set({ + pdsId: entryway.ctx.db.db.selectFrom('pds').select('id').limit(1), + }) + .where('did', '=', did) + .execute() + + await pdsAgent.login({ + identifier: accountDetail.handle, + password: accountDetail.password, + }) + + await pdsAgent.api.app.bsky.feed.post.create( + { repo: did }, + { + text: 'asdflsidkfu', + createdAt: new Date().toISOString(), + }, + ) + + const listPosts = await pdsAgent.api.com.atproto.repo.listRecords({ + repo: did, + collection: 'app.bsky.feed.post', + limit: 100, + }) + const listLikes = await pdsAgent.api.com.atproto.repo.listRecords({ + repo: did, + collection: 'app.bsky.feed.like', + limit: 100, + }) + + expect(listPosts.data.records.length).toBe(52) + expect(listLikes.data.records.length).toBe(50) + }) +}) + +const createEntryway = async ( + config: pdsEntryway.ServerEnvironment & { + adminPassword: string + jwtSigningKeyK256PrivateKeyHex: string + plcRotationKeyK256PrivateKeyHex: string + }, +) => { + const signingKey = await Secp256k1Keypair.create({ exportable: true }) + const recoveryKey = await Secp256k1Keypair.create({ exportable: true }) + const env: pdsEntryway.ServerEnvironment = { + isEntryway: true, + recoveryDidKey: recoveryKey.did(), + serviceHandleDomains: ['.test'], + dbPostgresUrl: process.env.DB_POSTGRES_URL, + blobstoreDiskLocation: path.join(os.tmpdir(), randomStr(8, 'base32')), + bskyAppViewUrl: 'https://appview.invalid', + bskyAppViewDid: 'did:example:invalid', + bskyAppViewCdnUrlPattern: 'http://cdn.appview.com/%s/%s/%s', + jwtSecret: randomStr(8, 'base32'), + repoSigningKeyK256PrivateKeyHex: await getPrivateHex(signingKey), + ...config, + } + const cfg = pdsEntryway.envToCfg(env) + const secrets = pdsEntryway.envToSecrets(env) + const server = await pdsEntryway.PDS.create(cfg, secrets) + await server.ctx.db.migrateToLatestOrThrow() + await server.start() + return server +} + +const getPublicHex = (key: Secp256k1Keypair) => { + return key.publicKeyStr('hex') +} + +const getPrivateHex = async (key: Secp256k1Keypair) => { + return ui8.toString(await key.export(), 'hex') +} diff --git a/packages/repo/src/index.ts b/packages/repo/src/index.ts index 82ed2115ad9..111e3546264 100644 --- a/packages/repo/src/index.ts +++ b/packages/repo/src/index.ts @@ -2,6 +2,7 @@ export * from './block-map' export * from './cid-set' export * from './repo' export * from './mst' +export * from './parse' export * from './storage' export * from './sync' export * from './types' diff --git a/packages/repo/src/readable-repo.ts b/packages/repo/src/readable-repo.ts index 4381bd5e0e4..7e53a7e5a55 100644 --- a/packages/repo/src/readable-repo.ts +++ b/packages/repo/src/readable-repo.ts @@ -6,6 +6,7 @@ import log from './logger' import * as util from './util' import * as parse from './parse' import { MissingBlocksError } from './error' +import { RepoRecord } from '@atproto/lexicon' type Params = { storage: ReadableBlockstore @@ -47,6 +48,19 @@ export class ReadableRepo { return this.commit.version } + async *walkRecords(from?: string): AsyncIterable<{ + collection: string + rkey: string + cid: CID + record: RepoRecord + }> { + for await (const leaf of this.data.walkLeavesFrom(from ?? '')) { + const { collection, rkey } = util.parseDataKey(leaf.key) + const record = await this.storage.readRecord(leaf.value) + yield { collection, rkey, cid: leaf.value, record } + } + } + async getRecord(collection: string, rkey: string): Promise { const dataKey = collection + '/' + rkey const cid = await this.data.get(dataKey) diff --git a/packages/repo/src/repo.ts b/packages/repo/src/repo.ts index 49e8ef24810..b1031cdd127 100644 --- a/packages/repo/src/repo.ts +++ b/packages/repo/src/repo.ts @@ -16,6 +16,7 @@ import log from './logger' import BlockMap from './block-map' import { ReadableRepo } from './readable-repo' import * as util from './util' +import CidSet from './cid-set' type Params = { storage: RepoStorage @@ -187,6 +188,34 @@ export class Repo extends ReadableRepo { const commit = await this.formatCommit(toWrite, keypair) return this.applyCommit(commit) } + + async formatResignCommit(rev: string, keypair: crypto.Keypair) { + const commit = await util.signCommit( + { + did: this.did, + version: 3, + rev, + prev: null, // added for backwards compatibility with v2 + data: this.commit.data, + }, + keypair, + ) + const newBlocks = new BlockMap() + const commitCid = await newBlocks.add(commit) + return { + cid: commitCid, + rev, + since: null, + prev: null, + newBlocks, + removedCids: new CidSet([this.cid]), + } + } + + async resignCommit(rev: string, keypair: crypto.Keypair) { + const formatted = await this.formatResignCommit(rev, keypair) + return this.applyCommit(formatted) + } } export default Repo diff --git a/packages/repo/src/storage/memory-blockstore.ts b/packages/repo/src/storage/memory-blockstore.ts index 5f91311c345..1426d962dc0 100644 --- a/packages/repo/src/storage/memory-blockstore.ts +++ b/packages/repo/src/storage/memory-blockstore.ts @@ -10,6 +10,7 @@ export class MemoryBlockstore { blocks: BlockMap root: CID | null = null + rev: string | null = null constructor(blocks?: BlockMap) { super() @@ -43,8 +44,9 @@ export class MemoryBlockstore this.blocks.addMap(blocks) } - async updateRoot(cid: CID): Promise { + async updateRoot(cid: CID, rev: string): Promise { this.root = cid + this.rev = rev } async applyCommit(commit: CommitData): Promise { diff --git a/packages/repo/src/storage/types.ts b/packages/repo/src/storage/types.ts index 804be48cbc8..b1eebd5d983 100644 --- a/packages/repo/src/storage/types.ts +++ b/packages/repo/src/storage/types.ts @@ -10,7 +10,7 @@ export interface RepoStorage { getRoot(): Promise putBlock(cid: CID, block: Uint8Array, rev: string): Promise putMany(blocks: BlockMap, rev: string): Promise - updateRoot(cid: CID): Promise + updateRoot(cid: CID, rev: string): Promise applyCommit(commit: CommitData) // Readable @@ -38,8 +38,10 @@ export interface BlobStore { unquarantine(cid: CID): Promise getBytes(cid: CID): Promise getStream(cid: CID): Promise + hasTemp(key: string): Promise hasStored(cid: CID): Promise delete(cid: CID): Promise + deleteMany(cid: CID[]): Promise } export class BlobNotFoundError extends Error {} diff --git a/packages/repo/src/sync/consumer.ts b/packages/repo/src/sync/consumer.ts index 08ca98195f2..1c0fdd796af 100644 --- a/packages/repo/src/sync/consumer.ts +++ b/packages/repo/src/sync/consumer.ts @@ -23,8 +23,9 @@ export const verifyRepo = async ( head: CID, did?: string, signingKey?: string, + opts?: { ensureLeaves?: boolean }, ): Promise => { - const diff = await verifyDiff(null, blocks, head, did, signingKey) + const diff = await verifyDiff(null, blocks, head, did, signingKey, opts) const creates = util.ensureCreates(diff.writes) return { creates, @@ -37,9 +38,10 @@ export const verifyDiffCar = async ( carBytes: Uint8Array, did?: string, signingKey?: string, + opts?: { ensureLeaves?: boolean }, ): Promise => { const car = await util.readCarWithRoot(carBytes) - return verifyDiff(repo, car.blocks, car.root, did, signingKey) + return verifyDiff(repo, car.blocks, car.root, did, signingKey, opts) } export const verifyDiff = async ( @@ -48,7 +50,9 @@ export const verifyDiff = async ( updateRoot: CID, did?: string, signingKey?: string, + opts?: { ensureLeaves?: boolean }, ): Promise => { + const { ensureLeaves = true } = opts ?? {} const stagedStorage = new MemoryBlockstore(updateBlocks) const updateStorage = repo ? new SyncStorage(stagedStorage, repo.storage) @@ -60,10 +64,10 @@ export const verifyDiff = async ( signingKey, ) const diff = await DataDiff.of(updated.data, repo?.data ?? null) - const writes = await util.diffToWriteDescripts(diff, updateBlocks) + const writes = await util.diffToWriteDescripts(diff) const newBlocks = diff.newMstBlocks const leaves = updateBlocks.getMany(diff.newLeafCids.toList()) - if (leaves.missing.length > 0) { + if (leaves.missing.length > 0 && ensureLeaves) { throw new Error(`missing leaf blocks: ${leaves.missing}`) } newBlocks.addMap(leaves.blocks) diff --git a/packages/repo/src/sync/provider.ts b/packages/repo/src/sync/provider.ts index ef6a586b15a..215481c6aa0 100644 --- a/packages/repo/src/sync/provider.ts +++ b/packages/repo/src/sync/provider.ts @@ -3,7 +3,7 @@ import { BlockWriter } from '@ipld/car/writer' import { CID } from 'multiformats/cid' import CidSet from '../cid-set' import { MissingBlocksError } from '../error' -import { RepoStorage } from '../storage' +import { ReadableBlockstore, RepoStorage } from '../storage' import * as util from '../util' import { MST } from '../mst' @@ -26,7 +26,7 @@ export const getFullRepo = ( // ------------- export const getRecords = ( - storage: RepoStorage, + storage: ReadableBlockstore, commitCid: CID, paths: RecordPath[], ): AsyncIterable => { diff --git a/packages/repo/src/types.ts b/packages/repo/src/types.ts index 4b796193ef3..7aeaba03fca 100644 --- a/packages/repo/src/types.ts +++ b/packages/repo/src/types.ts @@ -2,6 +2,7 @@ import { z } from 'zod' import { def as commonDef } from '@atproto/common-web' import { schema as common } from '@atproto/common' import { CID } from 'multiformats' +import * as car from '@ipld/car/api' import BlockMap from './block-map' import { RepoRecord } from '@atproto/lexicon' import CidSet from './cid-set' @@ -95,16 +96,25 @@ export type RecordDeleteOp = { export type RecordWriteOp = RecordCreateOp | RecordUpdateOp | RecordDeleteOp -export type RecordCreateDescript = RecordCreateOp & { +export type RecordCreateDescript = { + action: WriteOpAction.Create + collection: string + rkey: string cid: CID } -export type RecordUpdateDescript = RecordUpdateOp & { +export type RecordUpdateDescript = { + action: WriteOpAction.Update + collection: string + rkey: string prev: CID cid: CID } -export type RecordDeleteDescript = RecordDeleteOp & { +export type RecordDeleteDescript = { + action: WriteOpAction.Delete + collection: string + rkey: string cid: CID } @@ -163,3 +173,5 @@ export type VerifiedRepo = { creates: RecordCreateDescript[] commit: CommitData } + +export type CarBlock = car.Block diff --git a/packages/repo/src/util.ts b/packages/repo/src/util.ts index 89f85a097d5..5d52eb565a5 100644 --- a/packages/repo/src/util.ts +++ b/packages/repo/src/util.ts @@ -1,8 +1,8 @@ +import { setImmediate } from 'node:timers/promises' import { CID } from 'multiformats/cid' import * as cbor from '@ipld/dag-cbor' -import { CarReader } from '@ipld/car/reader' +import { CarBlockIterator } from '@ipld/car' import { BlockWriter, CarWriter } from '@ipld/car/writer' -import { Block as CarBlock } from '@ipld/car/api' import { streamToBuffer, verifyCidForBytes, @@ -18,6 +18,7 @@ import { ipldToLex, lexToIpld, LexValue, RepoRecord } from '@atproto/lexicon' import * as crypto from '@atproto/crypto' import DataDiff from './data-diff' import { + CarBlock, Commit, LegacyV2Commit, RecordCreateDescript, @@ -29,7 +30,6 @@ import { WriteOpAction, } from './types' import BlockMap from './block-map' -import * as parse from './parse' import { Keypair } from '@atproto/crypto' import { Readable } from 'stream' @@ -88,14 +88,15 @@ export const blocksToCarFile = ( return streamToBuffer(carStream) } -export const readCar = async ( - bytes: Uint8Array, +export const carToBlocks = async ( + car: CarBlockIterator, ): Promise<{ roots: CID[]; blocks: BlockMap }> => { - const car = await CarReader.fromBytes(bytes) const roots = await car.getRoots() const blocks = new BlockMap() - for await (const block of verifyIncomingCarBlocks(car.blocks())) { + for await (const block of verifyIncomingCarBlocks(car)) { blocks.set(block.cid, block.bytes) + // break up otherwise "synchronous" work in car parsing + await setImmediate() } return { roots, @@ -103,6 +104,18 @@ export const readCar = async ( } } +export const readCar = async ( + bytes: Uint8Array, +): Promise<{ roots: CID[]; blocks: BlockMap }> => { + const car = await CarBlockIterator.fromBytes(bytes) + return carToBlocks(car) +} + +export const readCarStream = async (stream: AsyncIterable) => { + const car = await CarBlockIterator.fromIterable(stream) + return carToBlocks(car) +} + export const readCarWithRoot = async ( bytes: Uint8Array, ): Promise<{ root: CID; blocks: BlockMap }> => { @@ -119,30 +132,25 @@ export const readCarWithRoot = async ( export const diffToWriteDescripts = ( diff: DataDiff, - blocks: BlockMap, ): Promise => { return Promise.all([ ...diff.addList().map(async (add) => { const { collection, rkey } = parseDataKey(add.key) - const value = await parse.getAndParseRecord(blocks, add.cid) return { action: WriteOpAction.Create, collection, rkey, cid: add.cid, - record: value.record, } as RecordCreateDescript }), ...diff.updateList().map(async (upd) => { const { collection, rkey } = parseDataKey(upd.key) - const value = await parse.getAndParseRecord(blocks, upd.cid) return { action: WriteOpAction.Update, collection, rkey, cid: upd.cid, prev: upd.prev, - record: value.record, } as RecordUpdateDescript }), ...diff.deleteList().map((del) => { diff --git a/packages/repo/tests/sync.test.ts b/packages/repo/tests/sync.test.ts index 9c8597a0228..5018d8f58ef 100644 --- a/packages/repo/tests/sync.test.ts +++ b/packages/repo/tests/sync.test.ts @@ -4,6 +4,7 @@ import { Repo, RepoContents, RepoVerificationError, + getAndParseRecord, readCarWithRoot, } from '../src' import { MemoryBlockstore } from '../src/storage' @@ -47,7 +48,8 @@ describe('Repo Sync', () => { const contentsFromOps: RepoContents = {} for (const write of verified.creates) { contentsFromOps[write.collection] ??= {} - contentsFromOps[write.collection][write.rkey] = write.record + const parsed = await getAndParseRecord(car.blocks, write.cid) + contentsFromOps[write.collection][write.rkey] = parsed.record } expect(contentsFromOps).toEqual(repoData) }) diff --git a/packages/xrpc-server/src/server.ts b/packages/xrpc-server/src/server.ts index 22281fc9740..9a666488c1c 100644 --- a/packages/xrpc-server/src/server.ts +++ b/packages/xrpc-server/src/server.ts @@ -173,7 +173,7 @@ export class Server { this.routes[verb]( `/xrpc/${nsid}`, ...middleware, - this.createHandler(nsid, def, config.handler), + this.createHandler(nsid, def, config), ) } @@ -206,10 +206,13 @@ export class Server { createHandler( nsid: string, def: LexXrpcQuery | LexXrpcProcedure, - handler: XRPCHandler, + routeCfg: XRPCHandlerConfig, ): RequestHandler { + const routeOpts = { + blobLimit: routeCfg.opts?.blobLimit ?? this.options.payload?.blobLimit, + } const validateReqInput = (req: express.Request) => - validateInput(nsid, def, req, this.options, this.lex) + validateInput(nsid, def, req, routeOpts, this.lex) const validateResOutput = this.options.validateResponse === false ? (output?: HandlerSuccess) => output @@ -254,7 +257,7 @@ export class Server { } // run the handler - const outputUnvalidated = await handler(reqCtx) + const outputUnvalidated = await routeCfg.handler(reqCtx) if (isHandlerError(outputUnvalidated)) { throw XRPCError.fromError(outputUnvalidated) diff --git a/packages/xrpc-server/src/types.ts b/packages/xrpc-server/src/types.ts index dc75627a95f..35e88268975 100644 --- a/packages/xrpc-server/src/types.ts +++ b/packages/xrpc-server/src/types.ts @@ -143,7 +143,12 @@ export type RateLimiterStatus = { isFirstInDuration: boolean } +export type RouteOpts = { + blobLimit?: number +} + export type XRPCHandlerConfig = { + opts?: RouteOpts rateLimit?: HandlerRateLimitOpts | HandlerRateLimitOpts[] auth?: AuthVerifier handler: XRPCHandler diff --git a/packages/xrpc-server/src/util.ts b/packages/xrpc-server/src/util.ts index f306d1944aa..cb389d859c2 100644 --- a/packages/xrpc-server/src/util.ts +++ b/packages/xrpc-server/src/util.ts @@ -19,8 +19,8 @@ import { handlerSuccess, InvalidRequestError, InternalServerError, - Options, XRPCError, + RouteOpts, } from './types' export function decodeQueryParams( @@ -82,7 +82,7 @@ export function validateInput( nsid: string, def: LexXrpcProcedure | LexXrpcQuery, req: express.Request, - opts: Options, + opts: RouteOpts, lexicons: Lexicons, ): HandlerInput | undefined { // request expectation @@ -139,7 +139,7 @@ export function validateInput( if (req.readableEnded) { body = req.body } else { - body = decodeBodyStream(req, opts.payload?.blobLimit) + body = decodeBodyStream(req, opts.blobLimit) } return { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 422867c73f4..0b554e7612c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -328,6 +328,10 @@ importers: uint8arrays: specifier: 3.0.0 version: 3.0.0 + devDependencies: + '@atproto/common': + specifier: workspace:^ + version: link:../common packages/dev-env: dependencies: @@ -540,9 +544,12 @@ importers: ioredis: specifier: ^5.3.2 version: 5.3.2 - jsonwebtoken: - specifier: ^8.5.1 - version: 8.5.1 + jose: + specifier: ^5.0.1 + version: 5.1.3 + key-encoder: + specifier: ^2.0.3 + version: 2.0.3 kysely: specifier: ^0.22.0 version: 0.22.0 @@ -589,6 +596,9 @@ importers: '@atproto/lex-cli': specifier: workspace:^ version: link:../lex-cli + '@atproto/pds-entryway': + specifier: npm:@atproto/pds@0.3.0-entryway.2 + version: /@atproto/pds@0.3.0-entryway.2 '@did-plc/server': specifier: ^0.0.1 version: 0.0.1 @@ -604,9 +614,6 @@ importers: '@types/express-serve-static-core': specifier: ^4.17.36 version: 4.17.36 - '@types/jsonwebtoken': - specifier: ^8.5.9 - version: 8.5.9 '@types/nodemailer': specifier: ^6.4.6 version: 6.4.6 @@ -619,6 +626,9 @@ importers: axios: specifier: ^0.27.2 version: 0.27.2 + get-port: + specifier: ^6.1.2 + version: 6.1.2 ws: specifier: ^8.12.0 version: 8.12.0 @@ -761,9 +771,15 @@ importers: '@atproto/pds': specifier: workspace:^ version: link:../../packages/pds + '@opentelemetry/instrumentation': + specifier: ^0.45.0 + version: 0.45.1(@opentelemetry/api@1.7.0) dd-trace: - specifier: 3.13.2 - version: 3.13.2 + specifier: ^4.18.0 + version: 4.20.0 + opentelemetry-plugin-better-sqlite3: + specifier: ^1.1.0 + version: 1.1.0(better-sqlite3@7.6.2) packages: @@ -797,6 +813,54 @@ packages: one-webcrypto: 1.0.3 uint8arrays: 3.0.0 + /@atproto/pds@0.3.0-entryway.2: + resolution: {integrity: sha512-nj3cOgPBiX0PLMG8Wn6Vy9mpRa891nGDXiOURoeSzQPSJMkWlk/4SlfYEFSrGSHlBnkUNd1fKE3NsJMMQJ/Utg==} + hasBin: true + dependencies: + '@atproto/api': link:packages/api + '@atproto/aws': link:packages/aws + '@atproto/common': link:packages/common + '@atproto/crypto': link:packages/crypto + '@atproto/identity': link:packages/identity + '@atproto/lexicon': link:packages/lexicon + '@atproto/repo': link:packages/repo + '@atproto/syntax': link:packages/syntax + '@atproto/xrpc': link:packages/xrpc + '@atproto/xrpc-server': link:packages/xrpc-server + '@did-plc/lib': 0.0.1 + better-sqlite3: 7.6.2 + bytes: 3.1.2 + compression: 1.7.4 + cors: 2.8.5 + disposable-email: 0.2.3 + express: 4.18.2 + express-async-errors: 3.1.1(express@4.18.2) + file-type: 16.5.4 + form-data: 4.0.0 + handlebars: 4.7.7 + http-errors: 2.0.0 + http-terminator: 3.2.0 + ioredis: 5.3.2 + jose: 4.15.4 + key-encoder: 2.0.3 + kysely: 0.22.0 + multiformats: 9.9.0 + nodemailer: 6.8.0 + nodemailer-html-to-text: 3.2.0 + p-queue: 6.6.2 + pg: 8.10.0 + pino: 8.15.0 + pino-http: 8.4.0 + sharp: 0.31.3 + typed-emitter: 2.1.0 + uint8arrays: 3.0.0 + zod: 3.21.4 + transitivePeerDependencies: + - debug + - pg-native + - supports-color + dev: true + /@aws-crypto/crc32@2.0.0: resolution: {integrity: sha512-TvE1r2CUueyXOuHdEigYjIZVesInd9KN+K/TFFNfkkxRThiNxO6i4ZqqAVMoEjAamZZ1AA8WXJkjCz7YShHPQA==} dependencies: @@ -4529,6 +4593,14 @@ packages: node-gyp-build: 3.9.0 dev: false + /@datadog/native-appsec@4.0.0: + resolution: {integrity: sha512-myTguXJ3VQHS2E1ylNsSF1avNpDmq5t+K4Q47wdzeakGc3sDIDDyEbvuFTujl9c9wBIkup94O1mZj5DR37ajzA==} + engines: {node: '>=12'} + requiresBuild: true + dependencies: + node-gyp-build: 3.9.0 + dev: false + /@datadog/native-iast-rewriter@1.1.2: resolution: {integrity: sha512-pigRfRtAjZjMjqIXyXb98S4aDnuHz/EmqpoxAajFZsNjBLM87YonwSY5zoBdCsOyA46ddKOJRoCQd5ZalpOFMQ==} engines: {node: '>= 10'} @@ -4536,12 +4608,27 @@ packages: node-gyp-build: 4.6.1 dev: false + /@datadog/native-iast-rewriter@2.2.1: + resolution: {integrity: sha512-DyZlE8gNa5AoOFNKGRJU4RYF/Y/tJzv4bIAMuVBbEnMA0xhiIYqpYQG8T3OKkALl3VSEeBMjYwuOR2fCrJ6gzA==} + engines: {node: '>= 10'} + dependencies: + lru-cache: 7.18.3 + node-gyp-build: 4.6.1 + dev: false + /@datadog/native-iast-taint-tracking@1.1.0: resolution: {integrity: sha512-TOrngpt6Qh52zWFOz1CkFXw0g43rnuUziFBtIMUsOLGzSHr9wdnTnE6HAyuvKy3f3ecAoZESlMfilGRKP93hXQ==} dependencies: node-gyp-build: 3.9.0 dev: false + /@datadog/native-iast-taint-tracking@1.6.4: + resolution: {integrity: sha512-Owxk7hQ4Dxwv4zJAoMjRga0IvE6lhvxnNc8pJCHsemCWBXchjr/9bqg05Zy5JnMbKUWn4XuZeJD6RFZpRa8bfw==} + requiresBuild: true + dependencies: + node-gyp-build: 3.9.0 + dev: false + /@datadog/native-metrics@1.6.0: resolution: {integrity: sha512-+8jBzd0nlLV+ay3Vb87DLwz8JHAS817hRhSRQ6zxhud9TyvvcNTNN+VA2sb2fe5UK4aMDvj/sGVJjEtgr4RHew==} engines: {node: '>=12'} @@ -4550,6 +4637,15 @@ packages: node-gyp-build: 3.9.0 dev: false + /@datadog/native-metrics@2.0.0: + resolution: {integrity: sha512-YklGVwUtmKGYqFf1MNZuOHvTYdKuR4+Af1XkWcMD8BwOAjxmd9Z+97328rCOY8TFUJzlGUPaXzB8j2qgG/BMwA==} + engines: {node: '>=12'} + requiresBuild: true + dependencies: + node-addon-api: 6.1.0 + node-gyp-build: 3.9.0 + dev: false + /@datadog/pprof@1.1.1: resolution: {integrity: sha512-5lYXUpikQhrJwzODtJ7aFM0oKmPccISnTCecuWhjxIj4/7UJv0DamkLak634bgEW+kiChgkKFDapHSesuXRDXQ==} engines: {node: '>=12'} @@ -4565,6 +4661,18 @@ packages: split: 1.0.1 dev: false + /@datadog/pprof@4.0.1: + resolution: {integrity: sha512-TavqyiyQZOaUM9eQB07r8+K/T1CqKyOdsUGxpN79+BF+eOQBpTj/Cte6KdlhcUSKL3h5hSjC+vlgA7uW2qtVhA==} + engines: {node: '>=14'} + requiresBuild: true + dependencies: + delay: 5.0.0 + node-gyp-build: 3.9.0 + p-limit: 3.1.0 + pprof-format: 2.0.7 + source-map: 0.7.4 + dev: false + /@datadog/sketches-js@2.1.0: resolution: {integrity: sha512-smLocSfrt3s53H/XSVP3/1kP42oqvrkjUPtyaFd1F79ux24oE31BKt+q0c6lsa6hOYrFzsIwyc5GXAI5JmfOew==} dev: false @@ -4653,7 +4761,6 @@ packages: /@ioredis/commands@1.2.0: resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==} - dev: false /@ipld/car@3.2.3: resolution: {integrity: sha512-pXE5mFJlXzJVaBwqAJKGlKqMmxq8H2SLEWBJgkeBDPBIN8ZbscPc3I9itkSQSlS/s6Fgx35Ri3LDTDtodQjCCQ==} @@ -5035,6 +5142,58 @@ packages: json-parse-even-better-errors: 3.0.0 dev: true + /@opentelemetry/api@1.7.0: + resolution: {integrity: sha512-AdY5wvN0P2vXBi3b29hxZgSFvdhdxPB9+f0B6s//P9Q8nibRWeA3cHm8UmLpio9ABigkVHJ5NMPk+Mz8VCCyrw==} + engines: {node: '>=8.0.0'} + dev: false + + /@opentelemetry/core@1.18.1(@opentelemetry/api@1.7.0): + resolution: {integrity: sha512-kvnUqezHMhsQvdsnhnqTNfAJs3ox/isB0SVrM1dhVFw7SsB7TstuVa6fgWnN2GdPyilIFLUvvbTZoVRmx6eiRg==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.8.0' + dependencies: + '@opentelemetry/api': 1.7.0 + '@opentelemetry/semantic-conventions': 1.18.1 + dev: false + + /@opentelemetry/instrumentation@0.44.0(@opentelemetry/api@1.7.0): + resolution: {integrity: sha512-B6OxJTRRCceAhhnPDBshyQO7K07/ltX3quOLu0icEvPK9QZ7r9P1y0RQX8O5DxB4vTv4URRkxkg+aFU/plNtQw==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + dependencies: + '@opentelemetry/api': 1.7.0 + '@types/shimmer': 1.0.5 + import-in-the-middle: 1.4.2 + require-in-the-middle: 7.2.0 + semver: 7.5.4 + shimmer: 1.2.1 + transitivePeerDependencies: + - supports-color + dev: false + + /@opentelemetry/instrumentation@0.45.1(@opentelemetry/api@1.7.0): + resolution: {integrity: sha512-V1Cr0g8hSg35lpW3G/GYVZurrhHrQZJdmP68WyJ83f1FDn3iru+/Vnlto9kiOSm7PHhW+pZGdb9Fbv+mkQ31CA==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + dependencies: + '@opentelemetry/api': 1.7.0 + '@types/shimmer': 1.0.5 + import-in-the-middle: 1.4.2 + require-in-the-middle: 7.2.0 + semver: 7.5.4 + shimmer: 1.2.1 + transitivePeerDependencies: + - supports-color + dev: false + + /@opentelemetry/semantic-conventions@1.18.1: + resolution: {integrity: sha512-+NLGHr6VZwcgE/2lw8zDIufOCGnzsA5CbQIMleXZTrgkBd0TanCX+MiDYJ1TOS4KL/Tqk0nFRxawnaYr6pkZkA==} + engines: {node: '>=14'} + dev: false + /@protobufjs/aspromise@1.1.2: resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} dev: false @@ -5229,7 +5388,6 @@ packages: /@tokenizer/token@0.3.0: resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==} - dev: false /@tootallnate/once@2.0.0: resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==} @@ -5382,12 +5540,6 @@ packages: resolution: {integrity: sha512-Hr5Jfhc9eYOQNPYO5WLDq/n4jqijdHNlDXjuAQkkt+mWdQR+XJToOHrsD4cPaMXpn6KO7y2+wM8AZEs8VpBLVA==} dev: true - /@types/jsonwebtoken@8.5.9: - resolution: {integrity: sha512-272FMnFGzAVMGtu9tkr29hRL6bZj4Zs1KZNeHLnKqAvp06tAIcarTMwOh8/8bz4FmKRcMxZhZNeUAQsNLoiPhg==} - dependencies: - '@types/node': 18.17.8 - dev: true - /@types/mime@1.3.2: resolution: {integrity: sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==} dev: true @@ -5460,6 +5612,10 @@ packages: '@types/node': 18.17.8 dev: true + /@types/shimmer@1.0.5: + resolution: {integrity: sha512-9Hp0ObzwwO57DpLFF0InUjUm/II8GmKAvzbefxQTihCb7KI6yc9yzf0nLc4mVdby5N4DRCgQM2wCup9KTieeww==} + dev: false + /@types/stack-utils@2.0.1: resolution: {integrity: sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==} dev: true @@ -6004,7 +6160,6 @@ packages: dependencies: bindings: 1.5.0 prebuild-install: 7.1.1 - dev: false /big-integer@1.6.51: resolution: {integrity: sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==} @@ -6014,7 +6169,6 @@ packages: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} dependencies: file-uri-to-path: 1.0.0 - dev: false /bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} @@ -6022,7 +6176,6 @@ packages: buffer: 5.7.1 inherits: 2.0.4 readable-stream: 3.6.2 - dev: false /bn.js@4.12.0: resolution: {integrity: sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==} @@ -6097,10 +6250,6 @@ packages: node-int64: 0.4.0 dev: true - /buffer-equal-constant-time@1.0.1: - resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} - dev: false - /buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} dev: true @@ -6121,7 +6270,6 @@ packages: dependencies: base64-js: 1.5.1 ieee754: 1.2.1 - dev: false /buffer@6.0.3: resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} @@ -6132,7 +6280,6 @@ packages: /bytes@3.0.0: resolution: {integrity: sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==} engines: {node: '>= 0.8'} - dev: false /bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} @@ -6257,7 +6404,6 @@ packages: /chownr@1.1.4: resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} - dev: false /chownr@2.0.0: resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==} @@ -6302,7 +6448,6 @@ packages: /cluster-key-slot@1.1.2: resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} engines: {node: '>=0.10.0'} - dev: false /co@4.6.0: resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} @@ -6341,7 +6486,6 @@ packages: dependencies: color-name: 1.1.4 simple-swizzle: 0.2.2 - dev: false /color-support@1.1.3: resolution: {integrity: sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==} @@ -6354,7 +6498,6 @@ packages: dependencies: color-convert: 2.0.1 color-string: 1.9.1 - dev: false /colorette@2.0.20: resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} @@ -6381,7 +6524,6 @@ packages: engines: {node: '>= 0.6'} dependencies: mime-db: 1.52.0 - dev: false /compression@1.7.4: resolution: {integrity: sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==} @@ -6396,7 +6538,6 @@ packages: vary: 1.1.2 transitivePeerDependencies: - supports-color - dev: false /concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} @@ -6495,6 +6636,11 @@ packages: resolution: {integrity: sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==} dev: true + /dc-polyfill@0.1.3: + resolution: {integrity: sha512-Wyk5n/5KUj3GfVKV2jtDbtChC/Ff9fjKsBcg4ZtYW1yQe3DXNHcGURvmoxhqQdfOQ9TwyMjnfyv1lyYcOkFkFA==} + engines: {node: '>=12.17'} + dev: false + /dd-trace@3.13.2: resolution: {integrity: sha512-POO9nEcAufe5pgp2xV1X3PfWip6wh+6TpEcRSlSgZJCIIMvWVCkcIVL/J2a6KAZq6V3Yjbkl8Ktfe+MOzQf5kw==} engines: {node: '>=14'} @@ -6529,6 +6675,46 @@ packages: semver: 5.7.2 dev: false + /dd-trace@4.20.0: + resolution: {integrity: sha512-y7IDLSSt6nww6zMdw/I8oLdfgOQADIOkERCNdfSzlBrXHz5CHimEOFfsHN87ag0mn6vusr06aoi+CQRZSNJz2g==} + engines: {node: '>=16'} + requiresBuild: true + dependencies: + '@datadog/native-appsec': 4.0.0 + '@datadog/native-iast-rewriter': 2.2.1 + '@datadog/native-iast-taint-tracking': 1.6.4 + '@datadog/native-metrics': 2.0.0 + '@datadog/pprof': 4.0.1 + '@datadog/sketches-js': 2.1.0 + '@opentelemetry/api': 1.7.0 + '@opentelemetry/core': 1.18.1(@opentelemetry/api@1.7.0) + crypto-randomuuid: 1.0.0 + dc-polyfill: 0.1.3 + ignore: 5.2.4 + import-in-the-middle: 1.4.2 + int64-buffer: 0.1.10 + ipaddr.js: 2.1.0 + istanbul-lib-coverage: 3.2.0 + jest-docblock: 29.7.0 + koalas: 1.0.2 + limiter: 1.1.5 + lodash.kebabcase: 4.1.1 + lodash.pick: 4.4.0 + lodash.sortby: 4.7.0 + lodash.uniq: 4.5.0 + lru-cache: 7.18.3 + methods: 1.1.2 + module-details-from-path: 1.0.3 + msgpack-lite: 0.1.26 + node-abort-controller: 3.1.1 + opentracing: 0.14.7 + path-to-regexp: 0.1.7 + pprof-format: 2.0.7 + protobufjs: 7.2.5 + retry: 0.13.1 + semver: 7.5.4 + dev: false + /debug@2.6.9: resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} peerDependencies: @@ -6568,7 +6754,6 @@ packages: engines: {node: '>=10'} dependencies: mimic-response: 3.1.0 - dev: false /dedent@0.7.0: resolution: {integrity: sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA==} @@ -6577,7 +6762,6 @@ packages: /deep-extend@0.6.0: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} - dev: false /deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} @@ -6615,7 +6799,6 @@ packages: /denque@2.1.0: resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} engines: {node: '>=0.10'} - dev: false /depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} @@ -6633,12 +6816,10 @@ packages: /detect-libc@2.0.2: resolution: {integrity: sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==} engines: {node: '>=8'} - dev: false /detect-newline@3.1.0: resolution: {integrity: sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==} engines: {node: '>=8'} - dev: true /diagnostics_channel@1.1.0: resolution: {integrity: sha512-OE1ngLDjSBPG6Tx0YATELzYzy3RKHC+7veQ8gLa8yS7AAgw65mFbVdcsu3501abqOZCEZqZyAIemB0zXlqDSuw==} @@ -6664,7 +6845,6 @@ packages: /disposable-email@0.2.3: resolution: {integrity: sha512-gkBQQ5Res431ZXqLlAafrXHizG7/1FWmi8U2RTtriD78Vc10HhBUvdJun3R4eSF0KRIQQJs+wHlxjkED/Hr1EQ==} - dev: false /doctrine@3.0.0: resolution: {integrity: sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==} @@ -6679,18 +6859,15 @@ packages: domelementtype: 2.3.0 domhandler: 4.3.1 entities: 2.2.0 - dev: false /domelementtype@2.3.0: resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} - dev: false /domhandler@4.3.1: resolution: {integrity: sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==} engines: {node: '>= 4'} dependencies: domelementtype: 2.3.0 - dev: false /domutils@2.8.0: resolution: {integrity: sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==} @@ -6698,7 +6875,6 @@ packages: dom-serializer: 1.4.1 domelementtype: 2.3.0 domhandler: 4.3.1 - dev: false /dotenv@16.0.3: resolution: {integrity: sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==} @@ -6709,12 +6885,6 @@ packages: engines: {node: '>=10'} dev: true - /ecdsa-sig-formatter@1.0.11: - resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} - dependencies: - safe-buffer: 5.2.1 - dev: false - /ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} @@ -6769,7 +6939,6 @@ packages: /entities@2.2.0: resolution: {integrity: sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==} - dev: false /env-paths@2.2.1: resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==} @@ -7270,13 +7439,16 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} + /event-lite@0.1.3: + resolution: {integrity: sha512-8qz9nOz5VeD2z96elrEKD2U433+L3DWdUdDkOINLGOJvx1GsMBbMn0aCeu28y8/e85A6mCigBiFlYMnTBEGlSw==} + dev: false + /event-target-shim@5.0.1: resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} engines: {node: '>=6'} /eventemitter3@4.0.7: resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} - dev: false /events@3.3.0: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} @@ -7305,7 +7477,6 @@ packages: /expand-template@2.0.3: resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} engines: {node: '>=6'} - dev: false /expect@28.1.3: resolution: {integrity: sha512-eEh0xn8HlsuOBxFgIss+2mX85VAS4Qy3OSkjV7rlBWljtA4oWH37glVGyOZSZvErDT/yBywZdPGwCXuTvSG85g==} @@ -7474,11 +7645,9 @@ packages: readable-web-to-node-stream: 3.0.2 strtok3: 6.3.0 token-types: 4.2.1 - dev: false /file-uri-to-path@1.0.0: resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} - dev: false /fill-range@7.0.1: resolution: {integrity: sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==} @@ -7573,7 +7742,6 @@ packages: /fs-constants@1.0.0: resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} - dev: false /fs-extra@10.1.0: resolution: {integrity: sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==} @@ -7693,7 +7861,6 @@ packages: /github-from-package@0.0.0: resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} - dev: false /glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} @@ -7789,7 +7956,6 @@ packages: wordwrap: 1.0.0 optionalDependencies: uglify-js: 3.17.4 - dev: false /hard-rejection@2.1.0: resolution: {integrity: sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==} @@ -7849,7 +8015,6 @@ packages: /he@1.2.0: resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==} hasBin: true - dev: false /help-me@4.2.0: resolution: {integrity: sha512-TAOnTB8Tz5Dw8penUuzHVrKNKlCIbwwbHnXraNJxPwf8LRtE2HlM84RYuezMFcwOJmoYOCWVDyJ8TQGxn9PgxA==} @@ -7882,7 +8047,6 @@ packages: he: 1.2.0 htmlparser2: 6.1.0 minimist: 1.2.8 - dev: false /htmlparser2@6.1.0: resolution: {integrity: sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==} @@ -7891,7 +8055,6 @@ packages: domhandler: 4.3.1 domutils: 2.8.0 entities: 2.2.0 - dev: false /http-cache-semantics@4.1.1: resolution: {integrity: sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==} @@ -8026,6 +8189,9 @@ packages: /ini@1.3.8: resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + + /int64-buffer@0.1.10: + resolution: {integrity: sha512-v7cSY1J8ydZ0GyjUHqF+1bshJ6cnEVLo9EnjB8p+4HDRPZc9N5jjmvUV7NvEsqQOKyH0pmIBFWXVQbiS0+OBbA==} dev: false /internal-slot@1.0.5: @@ -8052,7 +8218,6 @@ packages: standard-as-callback: 2.1.0 transitivePeerDependencies: - supports-color - dev: false /ip@2.0.0: resolution: {integrity: sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==} @@ -8081,7 +8246,6 @@ packages: /is-arrayish@0.3.2: resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==} - dev: false /is-bigint@1.0.4: resolution: {integrity: sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==} @@ -8113,7 +8277,6 @@ packages: resolution: {integrity: sha512-Z7dk6Qo8pOCp3l4tsX2C5ZVas4V+UxwQodwZhLopL91TX8UyyHEXafPcyoeeWuLrwzHcr3igO78wNLwHJHsMCQ==} dependencies: has: 1.0.3 - dev: true /is-date-object@1.0.5: resolution: {integrity: sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==} @@ -8225,6 +8388,10 @@ packages: engines: {node: '>=0.10.0'} dev: true + /isarray@1.0.0: + resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} + dev: false + /isarray@2.0.5: resolution: {integrity: sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==} dev: true @@ -8442,6 +8609,13 @@ packages: detect-newline: 3.1.0 dev: true + /jest-docblock@29.7.0: + resolution: {integrity: sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + detect-newline: 3.1.0 + dev: false + /jest-each@28.1.3: resolution: {integrity: sha512-arT1z4sg2yABU5uogObVPvSlSMQlDA48owx07BDPAiasW0yYpYHYOo4HHLz9q0BVzDVU4hILFjzJw0So9aCL/g==} engines: {node: ^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0} @@ -8733,6 +8907,10 @@ packages: resolution: {integrity: sha512-W+oqK4H+r5sITxfxpSU+MMdr/YSWGvgZMQDIsNoBDGGy4i7GBPTtvFKibQzW06n3U3TqHjhvBJsirShsEJ6eeQ==} dev: true + /jose@5.1.3: + resolution: {integrity: sha512-GPExOkcMsCLBTi1YetY2LmkoY559fss0+0KVa6kOfb2YFe84nAM7Nm/XzuZozah4iHgmBGrCOHL5/cy670SBRw==} + dev: false + /joycon@3.1.1: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} @@ -8816,37 +8994,6 @@ packages: graceful-fs: 4.2.11 dev: true - /jsonwebtoken@8.5.1: - resolution: {integrity: sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==} - engines: {node: '>=4', npm: '>=1.4.28'} - dependencies: - jws: 3.2.2 - lodash.includes: 4.3.0 - lodash.isboolean: 3.0.3 - lodash.isinteger: 4.0.4 - lodash.isnumber: 3.0.3 - lodash.isplainobject: 4.0.6 - lodash.isstring: 4.0.1 - lodash.once: 4.1.1 - ms: 2.1.3 - semver: 5.7.2 - dev: false - - /jwa@1.4.1: - resolution: {integrity: sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==} - dependencies: - buffer-equal-constant-time: 1.0.1 - ecdsa-sig-formatter: 1.0.11 - safe-buffer: 5.2.1 - dev: false - - /jws@3.2.2: - resolution: {integrity: sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==} - dependencies: - jwa: 1.4.1 - safe-buffer: 5.2.1 - dev: false - /key-encoder@2.0.3: resolution: {integrity: sha512-fgBtpAGIr/Fy5/+ZLQZIPPhsZEcbSlYu/Wu96tNDFNSjSACw5lEIOFeaVdQ/iwrb8oxjlWi6wmWdH76hV6GZjg==} dependencies: @@ -8878,7 +9025,6 @@ packages: /kysely@0.22.0: resolution: {integrity: sha512-ZE3qWtnqLOalodzfK5QUEcm7AEulhxsPNuKaGFsC3XiqO92vMLm+mAHk/NnbSIOtC4RmGm0nsv700i8KDp1gfQ==} engines: {node: '>=14.0.0'} - dev: false /kysely@0.23.5: resolution: {integrity: sha512-TH+b56pVXQq0tsyooYLeNfV11j6ih7D50dyN8tkM0e7ndiUH28Nziojiog3qRFlmEj9XePYdZUrNJ2079Qjdow==} @@ -8935,35 +9081,9 @@ packages: /lodash.defaults@4.2.0: resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} - dev: false - - /lodash.includes@4.3.0: - resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} - dev: false /lodash.isarguments@3.1.0: resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} - dev: false - - /lodash.isboolean@3.0.3: - resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==} - dev: false - - /lodash.isinteger@4.0.4: - resolution: {integrity: sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==} - dev: false - - /lodash.isnumber@3.0.3: - resolution: {integrity: sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==} - dev: false - - /lodash.isplainobject@4.0.6: - resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==} - dev: false - - /lodash.isstring@4.0.1: - resolution: {integrity: sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==} - dev: false /lodash.kebabcase@4.1.1: resolution: {integrity: sha512-N8XRTIMMqqDgSy4VLKPnJ/+hpGZN+PHQiJnSenYqPaVV/NCqEogTnAdZLQiGKhxX+JCs8waWq2t1XHWKOmlY8g==} @@ -8973,10 +9093,6 @@ packages: resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} dev: true - /lodash.once@4.1.1: - resolution: {integrity: sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==} - dev: false - /lodash.pick@4.4.0: resolution: {integrity: sha512-hXt6Ul/5yWjfklSGvLQl8vM//l3FtyHZeuelpzK6mm99pNvN9yTDruNZPEJZD1oWrqo+izBmB7oUfWgcCX7s4Q==} dev: false @@ -9138,7 +9254,6 @@ packages: /mimic-response@3.1.0: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} - dev: false /min-indent@1.0.1: resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} @@ -9241,7 +9356,6 @@ packages: /mkdirp-classic@0.5.3: resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} - dev: false /mkdirp@1.0.4: resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} @@ -9261,12 +9375,21 @@ packages: /ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + /msgpack-lite@0.1.26: + resolution: {integrity: sha512-SZ2IxeqZ1oRFGo0xFGbvBJWMp3yLIY9rlIJyxy8CGrwZn1f0ZK4r6jV/AM1r0FZMDUkWkglOk/eeKIL9g77Nxw==} + hasBin: true + dependencies: + event-lite: 0.1.3 + ieee754: 1.2.1 + int64-buffer: 0.1.10 + isarray: 1.0.0 + dev: false + /multiformats@9.9.0: resolution: {integrity: sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==} /napi-build-utils@1.0.2: resolution: {integrity: sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==} - dev: false /natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -9278,19 +9401,21 @@ packages: /neo-async@2.6.2: resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==} - dev: false /node-abi@3.47.0: resolution: {integrity: sha512-2s6B2CWZM//kPgwnuI0KrYwNjfdByE25zvAaEpq9IH4zcNsarH8Ihu/UuX6XMPEogDAxkuUFeZn60pXNHAqn3A==} engines: {node: '>=10'} dependencies: semver: 7.5.4 - dev: false /node-abort-controller@3.1.1: resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} dev: false + /node-addon-api@5.1.0: + resolution: {integrity: sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA==} + dev: true + /node-addon-api@6.1.0: resolution: {integrity: sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==} dev: false @@ -9357,12 +9482,10 @@ packages: engines: {node: '>= 10.23.0'} dependencies: html-to-text: 7.1.1 - dev: false /nodemailer@6.8.0: resolution: {integrity: sha512-EjYvSmHzekz6VNkNd12aUqAco+bOkRe3Of5jVhltqKhEsjw/y0PYPJfp83+s9Wzh1dspYAkUW/YNQ350NATbSQ==} engines: {node: '>=6.0.0'} - dev: false /nopt@6.0.0: resolution: {integrity: sha512-ZwLpbTgdhuZUnZzjd7nb1ZV+4DoiC6/sfiVKok72ym/4Tlf+DFdlHYmT2JPmcNNWV6Pi3SDf1kT+A4r9RTuT9g==} @@ -9436,7 +9559,6 @@ packages: /on-headers@1.0.2: resolution: {integrity: sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==} engines: {node: '>= 0.8'} - dev: false /once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} @@ -9453,6 +9575,20 @@ packages: mimic-fn: 2.1.0 dev: true + /opentelemetry-plugin-better-sqlite3@1.1.0(better-sqlite3@7.6.2): + resolution: {integrity: sha512-yd+mgaB5W5JxzcQt9TvX1VIrusqtbbeuxSoZ6KQe4Ra0J/Kqkp6kz7dg0VQUU5+cenOWkza6xtvsT0KGXI03HA==} + peerDependencies: + better-sqlite3: ^7.1.1 || ^8.0.0 || ^9.0.0 + dependencies: + '@opentelemetry/api': 1.7.0 + '@opentelemetry/core': 1.18.1(@opentelemetry/api@1.7.0) + '@opentelemetry/instrumentation': 0.44.0(@opentelemetry/api@1.7.0) + '@opentelemetry/semantic-conventions': 1.18.1 + better-sqlite3: 7.6.2 + transitivePeerDependencies: + - supports-color + dev: false + /opentracing@0.14.7: resolution: {integrity: sha512-vz9iS7MJ5+Bp1URw8Khvdyw1H/hGvzHWlKQ7eRrQojSCDL1/SrWfrY9QebLw97n2deyRtzHRC3MkQfVNUCo91Q==} engines: {node: '>=0.10'} @@ -9535,7 +9671,6 @@ packages: dependencies: eventemitter3: 4.0.7 p-timeout: 3.2.0 - dev: false /p-timeout@3.2.0: resolution: {integrity: sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==} @@ -9599,7 +9734,6 @@ packages: /path-parse@1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} - dev: true /path-to-regexp@0.1.7: resolution: {integrity: sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==} @@ -9612,7 +9746,6 @@ packages: /peek-readable@4.1.0: resolution: {integrity: sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg==} engines: {node: '>=8'} - dev: false /pg-connection-string@2.6.2: resolution: {integrity: sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==} @@ -9774,6 +9907,10 @@ packages: dependencies: xtend: 4.0.2 + /pprof-format@2.0.7: + resolution: {integrity: sha512-1qWaGAzwMpaXJP9opRa23nPnt2Egi7RMNoNBptEE/XwHbcn4fC2b/4U4bKc5arkGkIh2ZabpF2bEb+c5GNHEKA==} + dev: false + /prebuild-install@7.1.1: resolution: {integrity: sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==} engines: {node: '>=10'} @@ -9791,7 +9928,6 @@ packages: simple-get: 4.0.1 tar-fs: 2.1.1 tunnel-agent: 0.6.0 - dev: false /preferred-pm@3.0.3: resolution: {integrity: sha512-+wZgbxNES/KlJs9q40F/1sfOd/j7f1O9JaHcW5Dsn3aUUOZg3L2bjpVUcKV2jvtElYfoTuQiNeMfQJ4kwUAhCQ==} @@ -9964,7 +10100,6 @@ packages: ini: 1.3.8 minimist: 1.2.8 strip-json-comments: 2.0.1 - dev: false /react-is@18.2.0: resolution: {integrity: sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==} @@ -10022,7 +10157,6 @@ packages: engines: {node: '>=8'} dependencies: readable-stream: 3.6.2 - dev: false /real-require@0.2.0: resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} @@ -10039,14 +10173,12 @@ packages: /redis-errors@1.2.0: resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} engines: {node: '>=4'} - dev: false /redis-parser@3.0.0: resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} engines: {node: '>=4'} dependencies: redis-errors: 1.2.0 - dev: false /regenerate-unicode-properties@10.1.0: resolution: {integrity: sha512-d1VudCLoIGitcU/hEg2QqvyGZQmdC0Lf8BqdOMXGFSvJP4bNV1+XqbPQeHHLD51Jh4QJJ225dlIFvY4Ly6MXmQ==} @@ -10111,6 +10243,17 @@ packages: resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} engines: {node: '>=0.10.0'} + /require-in-the-middle@7.2.0: + resolution: {integrity: sha512-3TLx5TGyAY6AOqLBoXmHkNql0HIf2RGbuMgCDT2WO/uGVAPJs6h7Kl+bN6TIZGd9bWhWPwnDnTHGtW8Iu77sdw==} + engines: {node: '>=8.6.0'} + dependencies: + debug: 4.3.4 + module-details-from-path: 1.0.3 + resolve: 1.22.4 + transitivePeerDependencies: + - supports-color + dev: false + /require-main-filename@2.0.0: resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==} dev: true @@ -10144,7 +10287,6 @@ packages: is-core-module: 2.13.0 path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 - dev: true /retry@0.10.1: resolution: {integrity: sha512-ZXUSQYTHdl3uS7IuCehYfMzKyIDBNoAuUblvy5oGO5UJSUTmStUUVPXbA9Qxd173Bgre53yCQczQuHgRWAdvJQ==} @@ -10155,6 +10297,11 @@ packages: engines: {node: '>= 4'} dev: true + /retry@0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + dev: false + /reusify@1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -10190,7 +10337,6 @@ packages: requiresBuild: true dependencies: tslib: 2.6.2 - dev: false optional: true /safe-array-concat@1.0.1: @@ -10205,7 +10351,6 @@ packages: /safe-buffer@5.1.2: resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} - dev: false /safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} @@ -10287,6 +10432,21 @@ packages: /setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + /sharp@0.31.3: + resolution: {integrity: sha512-XcR4+FCLBFKw1bdB+GEhnUNXNXvnt0tDo4WsBsraKymuo/IAuPuCBVAL2wIkUw2r/dwFW5Q5+g66Kwl2dgDFVg==} + engines: {node: '>=14.15.0'} + requiresBuild: true + dependencies: + color: 4.2.3 + detect-libc: 2.0.2 + node-addon-api: 5.1.0 + prebuild-install: 7.1.1 + semver: 7.5.4 + simple-get: 4.0.1 + tar-fs: 2.1.1 + tunnel-agent: 0.6.0 + dev: true + /sharp@0.32.6: resolution: {integrity: sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w==} engines: {node: '>=14.15.0'} @@ -10326,6 +10486,10 @@ packages: engines: {node: '>=8'} dev: true + /shimmer@1.2.1: + resolution: {integrity: sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==} + dev: false + /side-channel@1.0.4: resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==} dependencies: @@ -10339,7 +10503,6 @@ packages: /simple-concat@1.0.1: resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} - dev: false /simple-get@4.0.1: resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} @@ -10347,13 +10510,11 @@ packages: decompress-response: 6.0.0 once: 1.4.0 simple-concat: 1.0.1 - dev: false /simple-swizzle@0.2.2: resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==} dependencies: is-arrayish: 0.3.2 - dev: false /sisteransi@1.0.5: resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} @@ -10481,7 +10642,6 @@ packages: /standard-as-callback@2.1.0: resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} - dev: false /statuses@2.0.1: resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} @@ -10586,7 +10746,6 @@ packages: /strip-json-comments@2.0.1: resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} engines: {node: '>=0.10.0'} - dev: false /strip-json-comments@3.1.1: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} @@ -10603,7 +10762,6 @@ packages: dependencies: '@tokenizer/token': 0.3.0 peek-readable: 4.1.0 - dev: false /supports-color@5.5.0: resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} @@ -10637,7 +10795,6 @@ packages: /supports-preserve-symlinks-flag@1.0.0: resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} engines: {node: '>= 0.4'} - dev: true /tar-fs@2.1.1: resolution: {integrity: sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==} @@ -10646,7 +10803,6 @@ packages: mkdirp-classic: 0.5.3 pump: 3.0.0 tar-stream: 2.2.0 - dev: false /tar-fs@3.0.4: resolution: {integrity: sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w==} @@ -10665,7 +10821,6 @@ packages: fs-constants: 1.0.0 inherits: 2.0.4 readable-stream: 3.6.2 - dev: false /tar-stream@3.1.6: resolution: {integrity: sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==} @@ -10759,7 +10914,6 @@ packages: dependencies: '@tokenizer/token': 0.3.0 ieee754: 1.2.1 - dev: false /tr46@0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} @@ -10851,7 +11005,6 @@ packages: /tslib@2.6.2: resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} requiresBuild: true - dev: false /tsutils@3.21.0(typescript@4.8.4): resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==} @@ -10881,7 +11034,6 @@ packages: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} dependencies: safe-buffer: 5.2.1 - dev: false /type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} @@ -10973,7 +11125,6 @@ packages: resolution: {integrity: sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==} optionalDependencies: rxjs: 7.8.1 - dev: false /typescript@4.8.4: resolution: {integrity: sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==} @@ -10992,7 +11143,6 @@ packages: engines: {node: '>=0.8.0'} hasBin: true requiresBuild: true - dev: false optional: true /uint8arrays@3.0.0: @@ -11195,7 +11345,6 @@ packages: /wordwrap@1.0.0: resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==} - dev: false /wrap-ansi@6.2.0: resolution: {integrity: sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==} diff --git a/services/pds/index.js b/services/pds/index.js index 112d63edf90..54742f7865e 100644 --- a/services/pds/index.js +++ b/services/pds/index.js @@ -1,8 +1,14 @@ 'use strict' /* eslint-disable */ -require('dd-trace') // Only works with commonjs +const { registerInstrumentations } = require('@opentelemetry/instrumentation') + +const { + BetterSqlite3Instrumentation, +} = require('opentelemetry-plugin-better-sqlite3') + +const { TracerProvider } = require('dd-trace') // Only works with commonjs .init({ logInjection: true }) - .tracer.use('express', { + .use('express', { hooks: { request: (span, req) => { maintainXrpcResource(span, req) @@ -10,6 +16,14 @@ require('dd-trace') // Only works with commonjs }, }) +const tracer = new TracerProvider() +tracer.register() + +registerInstrumentations({ + tracerProvider: tracer, + instrumentations: [new BetterSqlite3Instrumentation()], +}) + // Tracer code above must come before anything else const path = require('path') const { @@ -18,7 +32,6 @@ const { envToSecrets, readEnv, httpLogger, - PeriodicModerationActionReversal, } = require('@atproto/pds') const pkg = require('@atproto/pds/package.json') @@ -29,16 +42,6 @@ const main = async () => { const secrets = envToSecrets(env) const pds = await PDS.create(cfg, secrets) - // If the PDS is configured to proxy moderation, this will be running on appview instead of pds. - // Also don't run this on the sequencer leader, which may not be configured regarding moderation proxying at all. - const periodicModerationActionReversal = - pds.ctx.cfg.bskyAppView.proxyModeration || - pds.ctx.cfg.sequencerLeaderEnabled - ? null - : new PeriodicModerationActionReversal(pds.ctx) - const periodicModerationActionReversalRunning = - periodicModerationActionReversal?.run() - await pds.start() httpLogger.info('pds is running') diff --git a/services/pds/package.json b/services/pds/package.json index 96729d806e6..1490e2419da 100644 --- a/services/pds/package.json +++ b/services/pds/package.json @@ -5,6 +5,8 @@ "@atproto/aws": "workspace:^", "@atproto/crypto": "workspace:^", "@atproto/pds": "workspace:^", - "dd-trace": "3.13.2" + "@opentelemetry/instrumentation": "^0.45.0", + "dd-trace": "^4.18.0", + "opentelemetry-plugin-better-sqlite3": "^1.1.0" } } From 49e7f98fdb568a27a515572a67a18fd930f3f0c2 Mon Sep 17 00:00:00 2001 From: devin ivy Date: Tue, 5 Dec 2023 10:39:57 -0500 Subject: [PATCH 59/59] Config to start notifications daemon from a specific did (#1922) config to start notifications daemon from a specific did --- packages/bsky/src/daemon/config.ts | 10 ++++++++++ packages/bsky/src/daemon/index.ts | 6 ++++-- packages/bsky/src/daemon/notifications.ts | 6 +++++- packages/bsky/src/services/actor/index.ts | 20 +++++++++++++++----- services/bsky/daemon.js | 3 +++ 5 files changed, 37 insertions(+), 8 deletions(-) diff --git a/packages/bsky/src/daemon/config.ts b/packages/bsky/src/daemon/config.ts index e0e789203e4..3dd7d557652 100644 --- a/packages/bsky/src/daemon/config.ts +++ b/packages/bsky/src/daemon/config.ts @@ -4,6 +4,7 @@ export interface DaemonConfigValues { version: string dbPostgresUrl: string dbPostgresSchema?: string + notificationsDaemonFromDid?: string } export class DaemonConfig { @@ -15,11 +16,16 @@ export class DaemonConfig { overrides?.dbPostgresUrl || process.env.DB_PRIMARY_POSTGRES_URL const dbPostgresSchema = overrides?.dbPostgresSchema || process.env.DB_POSTGRES_SCHEMA + const notificationsDaemonFromDid = + overrides?.notificationsDaemonFromDid || + process.env.BSKY_NOTIFS_DAEMON_FROM_DID || + undefined assert(dbPostgresUrl) return new DaemonConfig({ version, dbPostgresUrl, dbPostgresSchema, + notificationsDaemonFromDid, ...stripUndefineds(overrides ?? {}), }) } @@ -35,6 +41,10 @@ export class DaemonConfig { get dbPostgresSchema() { return this.cfg.dbPostgresSchema } + + get notificationsDaemonFromDid() { + return this.cfg.notificationsDaemonFromDid + } } function stripUndefineds( diff --git a/packages/bsky/src/daemon/index.ts b/packages/bsky/src/daemon/index.ts index 61bcd8568f4..9d6388dd381 100644 --- a/packages/bsky/src/daemon/index.ts +++ b/packages/bsky/src/daemon/index.ts @@ -43,9 +43,11 @@ export class BskyDaemon { } async start() { - const { db } = this.ctx + const { db, cfg } = this.ctx const pool = db.pool - this.notifications.run() + this.notifications.run({ + startFromDid: cfg.notificationsDaemonFromDid, + }) this.dbStatsInterval = setInterval(() => { dbLogger.info( { diff --git a/packages/bsky/src/daemon/notifications.ts b/packages/bsky/src/daemon/notifications.ts index e8e884b37c2..96431af8c1f 100644 --- a/packages/bsky/src/daemon/notifications.ts +++ b/packages/bsky/src/daemon/notifications.ts @@ -47,4 +47,8 @@ export class NotificationsDaemon { } } -type RunOptions = { forever?: boolean; batchSize?: number } +type RunOptions = { + forever?: boolean + batchSize?: number + startFromDid?: string +} diff --git a/packages/bsky/src/services/actor/index.ts b/packages/bsky/src/services/actor/index.ts index 51be90892fc..a44e81e8f8f 100644 --- a/packages/bsky/src/services/actor/index.ts +++ b/packages/bsky/src/services/actor/index.ts @@ -147,24 +147,34 @@ export class ActorService { } async *all( - opts: { batchSize?: number; forever?: boolean; cooldownMs?: number } = {}, + opts: { + batchSize?: number + forever?: boolean + cooldownMs?: number + startFromDid?: string + } = {}, ) { - const { cooldownMs = 1000, batchSize = 1000, forever = false } = opts + const { + cooldownMs = 1000, + batchSize = 1000, + forever = false, + startFromDid, + } = opts const baseQuery = this.db.db .selectFrom('actor') .selectAll() .orderBy('did') .limit(batchSize) while (true) { - let cursor: ActorResult | undefined + let cursor = startFromDid do { const actors = cursor - ? await baseQuery.where('did', '>', cursor.did).execute() + ? await baseQuery.where('did', '>', cursor).execute() : await baseQuery.execute() for (const actor of actors) { yield actor } - cursor = actors.at(-1) + cursor = actors.at(-1)?.did } while (cursor) if (forever) { await wait(cooldownMs) diff --git a/services/bsky/daemon.js b/services/bsky/daemon.js index bd8322ab58f..38b2fdb59e4 100644 --- a/services/bsky/daemon.js +++ b/services/bsky/daemon.js @@ -18,6 +18,7 @@ const main = async () => { version: env.version, dbPostgresUrl: env.dbPostgresUrl, dbPostgresSchema: env.dbPostgresSchema, + notificationsDaemonFromDid: env.notificationsDaemonFromDid, }) const daemon = BskyDaemon.create({ db, cfg }) await daemon.start() @@ -34,6 +35,8 @@ const getEnv = () => ({ dbPoolSize: maybeParseInt(process.env.DB_POOL_SIZE), dbPoolMaxUses: maybeParseInt(process.env.DB_POOL_MAX_USES), dbPoolIdleTimeoutMs: maybeParseInt(process.env.DB_POOL_IDLE_TIMEOUT_MS), + notificationsDaemonFromDid: + process.env.BSKY_NOTIFS_DAEMON_FROM_DID || undefined, }) const maybeParseInt = (str) => {