From 2802880a97935bd465c65f9038cf732b929291f0 Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Tue, 12 Mar 2024 14:40:32 +0000 Subject: [PATCH 1/5] :sparkles: Add modEventDivertBlobs event to send blobs to abyss (#2238) * :construction: Working through an nullable review state * :white_check_mark: Update snapshots on some tests * :white_check_mark: Update snapshots on some tests * :white_check_mark: Add test for reviewOptional status mutation * :sparkles: Add divertBlobs event to send blobs to abyss * :recycle: Rename reviewOptional -> reviewNone * :recycle: Rename modEventDivertBlobs -> modEventDivert * :bug: Rename event type checker * :sparkles: Use pds resolver to get blob straight from pds * :white_check_mark: Use FOR UPDATE to respect db transactions * :recycle: Refactor to use event_pusher table instead of new table * :sparkles: Bring back missing lines in pnpm-lock * :hammer: Rebuild? * :rotating_light: Formatting * :recycle: Refactor to divert blob sync * :broom: Cleanup * :white_check_mark: Use modClient seed client in blob-divert test * update divert blob config to use basic admin auth * fix * build --------- Co-authored-by: Devin Ivy --- .../workflows/build-and-push-ozone-aws.yaml | 1 + lexicons/com/atproto/admin/defs.json | 13 +- packages/api/src/client/lexicons.ts | 12 + .../client/types/com/atproto/admin/defs.ts | 20 ++ packages/bsky/src/lexicon/lexicons.ts | 12 + .../lexicon/types/com/atproto/admin/defs.ts | 20 ++ .../src/api/admin/emitModerationEvent.ts | 298 ++++++++++-------- packages/ozone/src/auth-verifier.ts | 4 +- packages/ozone/src/config/config.ts | 14 + packages/ozone/src/config/env.ts | 4 + packages/ozone/src/context.ts | 22 +- packages/ozone/src/daemon/blob-diverter.ts | 150 +++++++++ packages/ozone/src/daemon/context.ts | 9 +- packages/ozone/src/daemon/event-pusher.ts | 63 +++- packages/ozone/src/daemon/index.ts | 1 + packages/ozone/src/lexicon/lexicons.ts | 12 + .../lexicon/types/com/atproto/admin/defs.ts | 20 ++ packages/ozone/src/mod-service/index.ts | 22 +- .../__snapshots__/blob-divert.test.ts.snap | 22 ++ packages/ozone/tests/blob-divert.test.ts | 90 ++++++ packages/pds/src/lexicon/lexicons.ts | 12 + .../lexicon/types/com/atproto/admin/defs.ts | 20 ++ pnpm-lock.yaml | 8 +- 23 files changed, 675 insertions(+), 174 deletions(-) create mode 100644 packages/ozone/src/daemon/blob-diverter.ts create mode 100644 packages/ozone/tests/__snapshots__/blob-divert.test.ts.snap create mode 100644 packages/ozone/tests/blob-divert.test.ts diff --git a/.github/workflows/build-and-push-ozone-aws.yaml b/.github/workflows/build-and-push-ozone-aws.yaml index 53f95c5b731..ff8162bb941 100644 --- a/.github/workflows/build-and-push-ozone-aws.yaml +++ b/.github/workflows/build-and-push-ozone-aws.yaml @@ -3,6 +3,7 @@ on: push: branches: - main + - divert-blobs env: REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }} USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }} diff --git a/lexicons/com/atproto/admin/defs.json b/lexicons/com/atproto/admin/defs.json index 1dc4944417d..f8173634f78 100644 --- a/lexicons/com/atproto/admin/defs.json +++ b/lexicons/com/atproto/admin/defs.json @@ -34,7 +34,8 @@ "#modEventEscalate", "#modEventMute", "#modEventEmail", - "#modEventResolveAppeal" + "#modEventResolveAppeal", + "#modEventDivert" ] }, "subject": { @@ -72,7 +73,8 @@ "#modEventEscalate", "#modEventMute", "#modEventEmail", - "#modEventResolveAppeal" + "#modEventResolveAppeal", + "#modEventDivert" ] }, "subject": { @@ -625,6 +627,13 @@ } } }, + "modEventDivert": { + "type": "object", + "description": "Divert a record's blobs to a 3rd party service for further scanning/tagging", + "properties": { + "comment": { "type": "string" } + } + }, "communicationTemplateView": { "type": "object", "required": [ diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index a7934d471d5..e9a585f5a21 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -92,6 +92,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', + 'lex:com.atproto.admin.defs#modEventDivert', ], }, subject: { @@ -150,6 +151,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', + 'lex:com.atproto.admin.defs#modEventDivert', ], }, subject: { @@ -940,6 +942,16 @@ export const schemaDict = { }, }, }, + modEventDivert: { + type: 'object', + description: + "Divert a record's blobs to a 3rd party service for further scanning/tagging", + properties: { + comment: { + type: 'string', + }, + }, + }, communicationTemplateView: { type: 'object', required: [ diff --git a/packages/api/src/client/types/com/atproto/admin/defs.ts b/packages/api/src/client/types/com/atproto/admin/defs.ts index af94ecceaff..ffec7bc21d8 100644 --- a/packages/api/src/client/types/com/atproto/admin/defs.ts +++ b/packages/api/src/client/types/com/atproto/admin/defs.ts @@ -41,6 +41,7 @@ export interface ModEventView { | ModEventMute | ModEventEmail | ModEventResolveAppeal + | ModEventDivert | { $type: string; [k: string]: unknown } subject: | RepoRef @@ -79,6 +80,7 @@ export interface ModEventViewDetail { | ModEventMute | ModEventEmail | ModEventResolveAppeal + | ModEventDivert | { $type: string; [k: string]: unknown } subject: | RepoView @@ -749,6 +751,24 @@ export function validateModEventTag(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#modEventTag', v) } +/** Divert a record's blobs to a 3rd party service for further scanning/tagging */ +export interface ModEventDivert { + comment?: string + [k: string]: unknown +} + +export function isModEventDivert(v: unknown): v is ModEventDivert { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventDivert' + ) +} + +export function validateModEventDivert(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventDivert', v) +} + export interface CommunicationTemplateView { id: string /** Name of the template. */ diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index a7934d471d5..e9a585f5a21 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -92,6 +92,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', + 'lex:com.atproto.admin.defs#modEventDivert', ], }, subject: { @@ -150,6 +151,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', + 'lex:com.atproto.admin.defs#modEventDivert', ], }, subject: { @@ -940,6 +942,16 @@ export const schemaDict = { }, }, }, + modEventDivert: { + type: 'object', + description: + "Divert a record's blobs to a 3rd party service for further scanning/tagging", + properties: { + comment: { + type: 'string', + }, + }, + }, communicationTemplateView: { type: 'object', required: [ diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts index a860e6bcfa0..bc374807c83 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts @@ -41,6 +41,7 @@ export interface ModEventView { | ModEventMute | ModEventEmail | ModEventResolveAppeal + | ModEventDivert | { $type: string; [k: string]: unknown } subject: | RepoRef @@ -79,6 +80,7 @@ export interface ModEventViewDetail { | ModEventMute | ModEventEmail | ModEventResolveAppeal + | ModEventDivert | { $type: string; [k: string]: unknown } subject: | RepoView @@ -749,6 +751,24 @@ export function validateModEventTag(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#modEventTag', v) } +/** Divert a record's blobs to a 3rd party service for further scanning/tagging */ +export interface ModEventDivert { + comment?: string + [k: string]: unknown +} + +export function isModEventDivert(v: unknown): v is ModEventDivert { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventDivert' + ) +} + +export function validateModEventDivert(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventDivert', v) +} + export interface CommunicationTemplateView { id: string /** Name of the template. */ diff --git a/packages/ozone/src/api/admin/emitModerationEvent.ts b/packages/ozone/src/api/admin/emitModerationEvent.ts index f2885e67e4c..f913e25cb34 100644 --- a/packages/ozone/src/api/admin/emitModerationEvent.ts +++ b/packages/ozone/src/api/admin/emitModerationEvent.ts @@ -2,160 +2,204 @@ import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../lexicon' import AppContext from '../../context' import { + isModEventDivert, isModEventEmail, isModEventLabel, isModEventReverseTakedown, isModEventTakedown, } from '../../lexicon/types/com/atproto/admin/defs' +import { HandlerInput } from '../../lexicon/types/com/atproto/admin/emitModerationEvent' import { subjectFromInput } from '../../mod-service/subject' import { ModerationLangService } from '../../mod-service/lang' import { retryHttp } from '../../util' +import { ModeratorOutput, AdminTokenOutput } from '../../auth-verifier' + +const handleModerationEvent = async ({ + ctx, + input, + auth, +}: { + ctx: AppContext + input: HandlerInput + auth: ModeratorOutput | AdminTokenOutput +}) => { + const access = auth.credentials + const createdBy = + auth.credentials.type === 'moderator' + ? auth.credentials.iss + : input.body.createdBy + const db = ctx.db + const moderationService = ctx.modService(db) + const { event } = input.body + const isTakedownEvent = isModEventTakedown(event) + const isReverseTakedownEvent = isModEventReverseTakedown(event) + const isLabelEvent = isModEventLabel(event) + const subject = subjectFromInput( + input.body.subject, + input.body.subjectBlobCids, + ) + + // apply access rules + + // if less than moderator access then can only take ack and escalation actions + if (isTakedownEvent || isReverseTakedownEvent) { + if (!access.isModerator) { + throw new AuthRequiredError( + 'Must be a full moderator to take this type of action', + ) + } -export default function (server: Server, ctx: AppContext) { - server.com.atproto.admin.emitModerationEvent({ - auth: ctx.authVerifier.modOrAdminToken, - handler: async ({ input, auth }) => { - const access = auth.credentials - const createdBy = - auth.credentials.type === 'moderator' - ? auth.credentials.iss - : input.body.createdBy - const db = ctx.db - const moderationService = ctx.modService(db) - const { event } = input.body - const isTakedownEvent = isModEventTakedown(event) - const isReverseTakedownEvent = isModEventReverseTakedown(event) - const isLabelEvent = isModEventLabel(event) - const subject = subjectFromInput( - input.body.subject, - input.body.subjectBlobCids, + // Non admins should not be able to take down feed generators + if ( + !access.isAdmin && + subject.recordPath?.includes('app.bsky.feed.generator/') + ) { + throw new AuthRequiredError( + 'Must be a full admin to take this type of action on feed generators', ) + } + } + // if less than moderator access then can not apply labels + if (!access.isModerator && isLabelEvent) { + throw new AuthRequiredError('Must be a full moderator to label content') + } - // apply access rules - - // if less than moderator access then can only take ack and escalation actions - if (isTakedownEvent || isReverseTakedownEvent) { - if (!access.isModerator) { - throw new AuthRequiredError( - 'Must be a full moderator to take this type of action', - ) - } - - // Non admins should not be able to take down feed generators - if ( - !access.isAdmin && - subject.recordPath?.includes('app.bsky.feed.generator/') - ) { - throw new AuthRequiredError( - 'Must be a full admin to take this type of action on feed generators', - ) - } - } - // if less than moderator access then can not apply labels - if (!access.isModerator && isLabelEvent) { - throw new AuthRequiredError('Must be a full moderator to label content') - } + if (isLabelEvent) { + validateLabels([ + ...(event.createLabelVals ?? []), + ...(event.negateLabelVals ?? []), + ]) + } - if (isLabelEvent) { - validateLabels([ - ...(event.createLabelVals ?? []), - ...(event.negateLabelVals ?? []), - ]) - } + if (isTakedownEvent || isReverseTakedownEvent) { + const status = await moderationService.getStatus(subject) - if (isTakedownEvent || isReverseTakedownEvent) { - const status = await moderationService.getStatus(subject) + if (status?.takendown && isTakedownEvent) { + throw new InvalidRequestError(`Subject is already taken down`) + } + + if (!status?.takendown && isReverseTakedownEvent) { + throw new InvalidRequestError(`Subject is not taken down`) + } - if (status?.takendown && isTakedownEvent) { - throw new InvalidRequestError(`Subject is already taken down`) - } + if (status?.takendown && isReverseTakedownEvent && subject.isRecord()) { + // due to the way blob status is modeled, we should reverse takedown on all + // blobs for the record being restored, which aren't taken down on another record. + subject.blobCids = status.blobCids ?? [] + } + } - if (!status?.takendown && isReverseTakedownEvent) { - throw new InvalidRequestError(`Subject is not taken down`) - } + if (isModEventEmail(event) && event.content) { + // sending email prior to logging the event to avoid a long transaction below + if (!subject.isRepo()) { + throw new InvalidRequestError('Email can only be sent to a repo subject') + } + const { content, subjectLine } = event + await retryHttp(() => + ctx.modService(db).sendEmail({ + subject: subjectLine, + content, + recipientDid: subject.did, + }), + ) + } - if (status?.takendown && isReverseTakedownEvent && subject.isRecord()) { - // due to the way blob status is modeled, we should reverse takedown on all - // blobs for the record being restored, which aren't taken down on another record. - subject.blobCids = status.blobCids ?? [] - } - } + if (isModEventDivert(event) && subject.isRecord()) { + if (!ctx.blobDiverter) { + throw new InvalidRequestError( + 'BlobDiverter not configured for this service', + ) + } + await ctx.blobDiverter.uploadBlobOnService(subject.info()) + } - if (isModEventEmail(event) && event.content) { - // sending email prior to logging the event to avoid a long transaction below - if (!subject.isRepo()) { - throw new InvalidRequestError( - 'Email can only be sent to a repo subject', - ) - } - const { content, subjectLine } = event - await retryHttp(() => - ctx.modService(db).sendEmail({ - subject: subjectLine, - content, - recipientDid: subject.did, - }), - ) + const moderationEvent = await db.transaction(async (dbTxn) => { + const moderationTxn = ctx.modService(dbTxn) + + const result = await moderationTxn.logEvent({ + event, + subject, + createdBy, + }) + + const moderationLangService = new ModerationLangService(moderationTxn) + await moderationLangService.tagSubjectWithLang({ + subject, + createdBy: ctx.cfg.service.did, + subjectStatus: result.subjectStatus, + }) + + if (subject.isRepo()) { + if (isTakedownEvent) { + const isSuspend = !!result.event.durationInHours + await moderationTxn.takedownRepo(subject, result.event.id, isSuspend) + } else if (isReverseTakedownEvent) { + await moderationTxn.reverseTakedownRepo(subject) } + } - const moderationEvent = await db.transaction(async (dbTxn) => { - const moderationTxn = ctx.modService(dbTxn) + if (subject.isRecord()) { + if (isTakedownEvent) { + await moderationTxn.takedownRecord(subject, result.event.id) + } else if (isReverseTakedownEvent) { + await moderationTxn.reverseTakedownRecord(subject) + } + } - const result = await moderationTxn.logEvent({ - event, - subject, - createdBy, - }) + if (isLabelEvent) { + await moderationTxn.formatAndCreateLabels( + result.event.subjectUri ?? result.event.subjectDid, + result.event.subjectCid, + { + create: result.event.createLabelVals?.length + ? result.event.createLabelVals.split(' ') + : undefined, + negate: result.event.negateLabelVals?.length + ? result.event.negateLabelVals.split(' ') + : undefined, + }, + ) + } - const moderationLangService = new ModerationLangService(moderationTxn) - await moderationLangService.tagSubjectWithLang({ - subject, - createdBy: ctx.cfg.service.did, - subjectStatus: result.subjectStatus, - }) + return result.event + }) - if (subject.isRepo()) { - if (isTakedownEvent) { - const isSuspend = !!result.event.durationInHours - await moderationTxn.takedownRepo( - subject, - result.event.id, - isSuspend, - ) - } else if (isReverseTakedownEvent) { - await moderationTxn.reverseTakedownRepo(subject) - } - } - - if (subject.isRecord()) { - if (isTakedownEvent) { - await moderationTxn.takedownRecord(subject, result.event.id) - } else if (isReverseTakedownEvent) { - await moderationTxn.reverseTakedownRecord(subject) - } - } - - if (isLabelEvent) { - await moderationTxn.formatAndCreateLabels( - result.event.subjectUri ?? result.event.subjectDid, - result.event.subjectCid, - { - create: result.event.createLabelVals?.length - ? result.event.createLabelVals.split(' ') - : undefined, - negate: result.event.negateLabelVals?.length - ? result.event.negateLabelVals.split(' ') - : undefined, - }, - ) - } + return moderationService.views.formatEvent(moderationEvent) +} - return result.event +export default function (server: Server, ctx: AppContext) { + server.com.atproto.admin.emitModerationEvent({ + auth: ctx.authVerifier.modOrAdminToken, + handler: async ({ input, auth }) => { + const moderationEvent = await handleModerationEvent({ + input, + auth, + ctx, }) + // On divert events, we need to automatically take down the blobs + if (isModEventDivert(input.body.event)) { + await handleModerationEvent({ + auth, + ctx, + input: { + ...input, + body: { + ...input.body, + event: { + ...input.body.event, + $type: 'com.atproto.admin.defs#modEventTakedown', + comment: + '[DIVERT_SIDE_EFFECT]: Automatically taking down after divert event', + }, + }, + }, + }) + } + return { encoding: 'application/json', - body: moderationService.views.formatEvent(moderationEvent), + body: moderationEvent, } }, }) diff --git a/packages/ozone/src/auth-verifier.ts b/packages/ozone/src/auth-verifier.ts index 2eec84cb683..b3ab579d754 100644 --- a/packages/ozone/src/auth-verifier.ts +++ b/packages/ozone/src/auth-verifier.ts @@ -7,7 +7,7 @@ type ReqCtx = { req: express.Request } -type AdminTokenOutput = { +export type AdminTokenOutput = { credentials: { type: 'admin_token' isAdmin: true @@ -16,7 +16,7 @@ type AdminTokenOutput = { } } -type ModeratorOutput = { +export type ModeratorOutput = { credentials: { type: 'moderator' aud: string diff --git a/packages/ozone/src/config/config.ts b/packages/ozone/src/config/config.ts index 8621dfd2957..b3fb1c8dbe9 100644 --- a/packages/ozone/src/config/config.ts +++ b/packages/ozone/src/config/config.ts @@ -50,6 +50,13 @@ export const envToCfg = (env: OzoneEnvironment): OzoneConfig => { plcUrl: env.didPlcUrl, } + const blobDivertServiceCfg = + env.blobDivertUrl && env.blobDivertAdminPassword + ? { + url: env.blobDivertUrl, + adminPassword: env.blobDivertAdminPassword, + } + : null const accessCfg: OzoneConfig['access'] = { admins: env.adminDids, moderators: env.moderatorDids, @@ -63,6 +70,7 @@ export const envToCfg = (env: OzoneEnvironment): OzoneConfig => { pds: pdsCfg, cdn: cdnCfg, identity: identityCfg, + blobDivert: blobDivertServiceCfg, access: accessCfg, } } @@ -74,6 +82,7 @@ export type OzoneConfig = { pds: PdsConfig | null cdn: CdnConfig identity: IdentityConfig + blobDivert: BlobDivertConfig | null access: AccessConfig } @@ -85,6 +94,11 @@ export type ServiceConfig = { devMode?: boolean } +export type BlobDivertConfig = { + url: string + adminPassword: string +} + export type DatabaseConfig = { postgresUrl: string postgresSchema?: string diff --git a/packages/ozone/src/config/env.ts b/packages/ozone/src/config/env.ts index 06850df0c5e..a879339aacd 100644 --- a/packages/ozone/src/config/env.ts +++ b/packages/ozone/src/config/env.ts @@ -25,6 +25,8 @@ export const readEnv = (): OzoneEnvironment => { triageDids: envList('OZONE_TRIAGE_DIDS'), adminPassword: envStr('OZONE_ADMIN_PASSWORD'), signingKeyHex: envStr('OZONE_SIGNING_KEY_HEX'), + blobDivertUrl: envStr('OZONE_BLOB_DIVERT_URL'), + blobDivertAdminPassword: envStr('OZONE_BLOB_DIVERT_ADMIN_PASSWORD'), } } @@ -52,4 +54,6 @@ export type OzoneEnvironment = { triageDids: string[] adminPassword?: string signingKeyHex?: string + blobDivertUrl?: string + blobDivertAdminPassword?: string } diff --git a/packages/ozone/src/context.ts b/packages/ozone/src/context.ts index b7d9a3e9e89..d0cbd9ae347 100644 --- a/packages/ozone/src/context.ts +++ b/packages/ozone/src/context.ts @@ -14,6 +14,7 @@ import { CommunicationTemplateService, CommunicationTemplateServiceCreator, } from './communication-service/template' +import { BlobDiverter } from './daemon/blob-diverter' import { AuthVerifier } from './auth-verifier' import { ImageInvalidator } from './image-invalidator' import { getSigningKeyId } from './util' @@ -25,6 +26,7 @@ export type AppContextOptions = { communicationTemplateService: CommunicationTemplateServiceCreator appviewAgent: AtpAgent pdsAgent: AtpAgent | undefined + blobDiverter?: BlobDiverter signingKey: Keypair signingKeyId: number idResolver: IdResolver @@ -56,6 +58,10 @@ export class AppContext { ? new AtpAgent({ service: cfg.pds.url }) : undefined + const idResolver = new IdResolver({ + plcUrl: cfg.identity.plcUrl, + }) + const createAuthHeaders = (aud: string) => createServiceAuthHeaders({ iss: `${cfg.service.did}#atproto_labeler`, @@ -64,15 +70,16 @@ export class AppContext { }) const backgroundQueue = new BackgroundQueue(db) + const blobDiverter = cfg.blobDivert + ? new BlobDiverter(db, { + idResolver, + serviceConfig: cfg.blobDivert, + }) + : undefined const eventPusher = new EventPusher(db, createAuthHeaders, { appview: cfg.appview.pushEvents ? cfg.appview : undefined, pds: cfg.pds ?? undefined, }) - - const idResolver = new IdResolver({ - plcUrl: cfg.identity.plcUrl, - }) - const modService = ModerationService.creator( signingKey, signingKeyId, @@ -111,6 +118,7 @@ export class AppContext { backgroundQueue, sequencer, authVerifier, + blobDiverter, ...(overrides ?? {}), }, secrets, @@ -137,6 +145,10 @@ export class AppContext { return this.opts.modService } + get blobDiverter(): BlobDiverter | undefined { + return this.opts.blobDiverter + } + get communicationTemplateService(): CommunicationTemplateServiceCreator { return this.opts.communicationTemplateService } diff --git a/packages/ozone/src/daemon/blob-diverter.ts b/packages/ozone/src/daemon/blob-diverter.ts new file mode 100644 index 00000000000..386c3a090cb --- /dev/null +++ b/packages/ozone/src/daemon/blob-diverter.ts @@ -0,0 +1,150 @@ +import { + VerifyCidTransform, + forwardStreamErrors, + getPdsEndpoint, +} from '@atproto/common' +import { IdResolver } from '@atproto/identity' +import axios from 'axios' +import { Readable } from 'stream' +import { CID } from 'multiformats/cid' + +import Database from '../db' +import { retryHttp } from '../util' +import { BlobDivertConfig } from '../config' + +export class BlobDiverter { + serviceConfig: BlobDivertConfig + idResolver: IdResolver + + constructor( + public db: Database, + services: { + idResolver: IdResolver + serviceConfig: BlobDivertConfig + }, + ) { + this.serviceConfig = services.serviceConfig + this.idResolver = services.idResolver + } + + private async getBlob({ + pds, + did, + cid, + }: { + pds: string + did: string + cid: string + }) { + const blobResponse = await axios.get( + `${pds}/xrpc/com.atproto.sync.getBlob`, + { + params: { did, cid }, + decompress: true, + responseType: 'stream', + timeout: 5000, // 5sec of inactivity on the connection + }, + ) + const imageStream: Readable = blobResponse.data + const verifyCid = new VerifyCidTransform(CID.parse(cid)) + forwardStreamErrors(imageStream, verifyCid) + + return { + contentType: + blobResponse.headers['content-type'] || 'application/octet-stream', + imageStream: imageStream.pipe(verifyCid), + } + } + + async sendImage({ + url, + imageStream, + contentType, + }: { + url: string + imageStream: Readable + contentType: string + }) { + const result = await axios(url, { + method: 'POST', + data: imageStream, + headers: { + Authorization: basicAuth('admin', this.serviceConfig.adminPassword), + 'Content-Type': contentType, + }, + }) + + return result.status === 200 + } + + private async uploadBlob( + { + imageStream, + contentType, + }: { imageStream: Readable; contentType: string }, + { + subjectDid, + subjectUri, + }: { subjectDid: string; subjectUri: string | null }, + ) { + const url = new URL(this.serviceConfig.url) + url.searchParams.set('did', subjectDid) + if (subjectUri) url.searchParams.set('uri', subjectUri) + const result = await this.sendImage({ + url: url.toString(), + imageStream, + contentType, + }) + + return result + } + + async uploadBlobOnService({ + subjectDid, + subjectUri, + subjectBlobCids, + }: { + subjectDid: string + subjectUri: string + subjectBlobCids: string[] + }): Promise { + const didDoc = await this.idResolver.did.resolve(subjectDid) + + if (!didDoc) { + throw new Error('Error resolving DID') + } + + const pds = getPdsEndpoint(didDoc) + + if (!pds) { + throw new Error('Error resolving PDS') + } + + // attempt to download and upload within the same retry block since the imageStream is not reusable + const uploadResult = await Promise.all( + subjectBlobCids.map((cid) => + retryHttp(async () => { + const { imageStream, contentType } = await this.getBlob({ + pds, + cid, + did: subjectDid, + }) + return this.uploadBlob( + { imageStream, contentType }, + { subjectDid, subjectUri }, + ) + }), + ), + ) + + if (uploadResult.includes(false)) { + throw new Error(`Error uploading blob ${subjectUri}`) + } + + return true + } +} + +const basicAuth = (username: string, password: string) => { + return 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64') +} diff --git a/packages/ozone/src/daemon/context.ts b/packages/ozone/src/daemon/context.ts index 64cf3c9423e..3ffa021d37e 100644 --- a/packages/ozone/src/daemon/context.ts +++ b/packages/ozone/src/daemon/context.ts @@ -1,5 +1,6 @@ import { Keypair, Secp256k1Keypair } from '@atproto/crypto' import { createServiceAuthHeaders } from '@atproto/xrpc-server' +import { IdResolver } from '@atproto/identity' import AtpAgent from '@atproto/api' import { OzoneConfig, OzoneSecrets } from '../config' import { Database } from '../db' @@ -7,7 +8,6 @@ import { EventPusher } from './event-pusher' import { EventReverser } from './event-reverser' import { ModerationService, ModerationServiceCreator } from '../mod-service' import { BackgroundQueue } from '../background' -import { IdResolver } from '@atproto/identity' import { getSigningKeyId } from '../util' export type DaemonContextOptions = { @@ -34,6 +34,10 @@ export class DaemonContext { const signingKey = await Secp256k1Keypair.import(secrets.signingKeyHex) const signingKeyId = await getSigningKeyId(db, signingKey.did()) + const idResolver = new IdResolver({ + plcUrl: cfg.identity.plcUrl, + }) + const appviewAgent = new AtpAgent({ service: cfg.appview.url }) const createAuthHeaders = (aud: string) => createServiceAuthHeaders({ @@ -48,9 +52,6 @@ export class DaemonContext { }) const backgroundQueue = new BackgroundQueue(db) - const idResolver = new IdResolver({ - plcUrl: cfg.identity.plcUrl, - }) const modService = ModerationService.creator( signingKey, diff --git a/packages/ozone/src/daemon/event-pusher.ts b/packages/ozone/src/daemon/event-pusher.ts index d1ff52b7d14..ea4b5ecd35d 100644 --- a/packages/ozone/src/daemon/event-pusher.ts +++ b/packages/ozone/src/daemon/event-pusher.ts @@ -6,6 +6,8 @@ import { RepoPushEventType } from '../db/schema/repo_push_event' import { retryHttp } from '../util' import { dbLogger } from '../logger' import { InputSchema } from '../lexicon/types/com/atproto/admin/updateSubjectStatus' +import { BlobPushEvent } from '../db/schema/blob_push_event' +import { Insertable, Selectable } from 'kysely' type EventSubject = InputSchema['subject'] @@ -285,20 +287,53 @@ export class EventPusher { subject, evt.takedownRef, ) - await dbTxn.db - .updateTable('blob_push_event') - .set( - succeeded - ? { confirmedAt: new Date() } - : { - lastAttempted: new Date(), - attempts: (evt.attempts ?? 0) + 1, - }, - ) - .where('subjectDid', '=', evt.subjectDid) - .where('subjectBlobCid', '=', evt.subjectBlobCid) - .where('eventType', '=', evt.eventType) - .execute() + await this.markBlobEventAttempt(dbTxn, evt, succeeded) }) } + + async markBlobEventAttempt( + dbTxn: Database, + event: Selectable, + succeeded: boolean, + ) { + await dbTxn.db + .updateTable('blob_push_event') + .set( + succeeded + ? { confirmedAt: new Date() } + : { + lastAttempted: new Date(), + attempts: (event.attempts ?? 0) + 1, + }, + ) + .where('subjectDid', '=', event.subjectDid) + .where('subjectBlobCid', '=', event.subjectBlobCid) + .where('eventType', '=', event.eventType) + .execute() + } + + async logBlobPushEvent( + blobValues: Insertable[], + takedownRef?: string | null, + ) { + return this.db.db + .insertInto('blob_push_event') + .values(blobValues) + .onConflict((oc) => + oc.columns(['subjectDid', 'subjectBlobCid', 'eventType']).doUpdateSet({ + takedownRef, + confirmedAt: null, + attempts: 0, + lastAttempted: null, + }), + ) + .returning([ + 'id', + 'subjectDid', + 'subjectUri', + 'subjectBlobCid', + 'eventType', + ]) + .execute() + } } diff --git a/packages/ozone/src/daemon/index.ts b/packages/ozone/src/daemon/index.ts index aa5d7b12734..501b8caad5c 100644 --- a/packages/ozone/src/daemon/index.ts +++ b/packages/ozone/src/daemon/index.ts @@ -3,6 +3,7 @@ import DaemonContext from './context' import { AppContextOptions } from '../context' export { EventPusher } from './event-pusher' +export { BlobDiverter } from './blob-diverter' export { EventReverser } from './event-reverser' export class OzoneDaemon { diff --git a/packages/ozone/src/lexicon/lexicons.ts b/packages/ozone/src/lexicon/lexicons.ts index a7934d471d5..e9a585f5a21 100644 --- a/packages/ozone/src/lexicon/lexicons.ts +++ b/packages/ozone/src/lexicon/lexicons.ts @@ -92,6 +92,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', + 'lex:com.atproto.admin.defs#modEventDivert', ], }, subject: { @@ -150,6 +151,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', + 'lex:com.atproto.admin.defs#modEventDivert', ], }, subject: { @@ -940,6 +942,16 @@ export const schemaDict = { }, }, }, + modEventDivert: { + type: 'object', + description: + "Divert a record's blobs to a 3rd party service for further scanning/tagging", + properties: { + comment: { + type: 'string', + }, + }, + }, communicationTemplateView: { type: 'object', required: [ diff --git a/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts b/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts index a860e6bcfa0..bc374807c83 100644 --- a/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts @@ -41,6 +41,7 @@ export interface ModEventView { | ModEventMute | ModEventEmail | ModEventResolveAppeal + | ModEventDivert | { $type: string; [k: string]: unknown } subject: | RepoRef @@ -79,6 +80,7 @@ export interface ModEventViewDetail { | ModEventMute | ModEventEmail | ModEventResolveAppeal + | ModEventDivert | { $type: string; [k: string]: unknown } subject: | RepoView @@ -749,6 +751,24 @@ export function validateModEventTag(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#modEventTag', v) } +/** Divert a record's blobs to a 3rd party service for further scanning/tagging */ +export interface ModEventDivert { + comment?: string + [k: string]: unknown +} + +export function isModEventDivert(v: unknown): v is ModEventDivert { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventDivert' + ) +} + +export function validateModEventDivert(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventDivert', v) +} + export interface CommunicationTemplateView { id: string /** Name of the template. */ diff --git a/packages/ozone/src/mod-service/index.ts b/packages/ozone/src/mod-service/index.ts index 671c93990a6..0e878087a28 100644 --- a/packages/ozone/src/mod-service/index.ts +++ b/packages/ozone/src/mod-service/index.ts @@ -566,27 +566,17 @@ export class ModerationService { for (const cid of blobCids) { blobValues.push({ eventType, + takedownRef, subjectDid: subject.did, + subjectUri: subject.uri || null, subjectBlobCid: cid.toString(), - takedownRef, }) } } - const blobEvts = await this.db.db - .insertInto('blob_push_event') - .values(blobValues) - .onConflict((oc) => - oc - .columns(['subjectDid', 'subjectBlobCid', 'eventType']) - .doUpdateSet({ - takedownRef, - confirmedAt: null, - attempts: 0, - lastAttempted: null, - }), - ) - .returning(['id', 'subjectDid', 'subjectBlobCid', 'eventType']) - .execute() + const blobEvts = await this.eventPusher.logBlobPushEvent( + blobValues, + takedownRef, + ) this.db.onCommit(() => { this.backgroundQueue.add(async () => { diff --git a/packages/ozone/tests/__snapshots__/blob-divert.test.ts.snap b/packages/ozone/tests/__snapshots__/blob-divert.test.ts.snap new file mode 100644 index 00000000000..30477a59748 --- /dev/null +++ b/packages/ozone/tests/__snapshots__/blob-divert.test.ts.snap @@ -0,0 +1,22 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`blob divert sends blobs to configured divert service and marks divert date 1`] = ` +Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "createdBy": "user(0)", + "event": Object { + "$type": "com.atproto.admin.defs#modEventDivert", + "comment": "Diverting for test", + }, + "id": 1, + "subject": Object { + "$type": "com.atproto.repo.strongRef", + "cid": "cids(0)", + "uri": "record(0)", + }, + "subjectBlobCids": Array [ + "cids(1)", + "cids(2)", + ], +} +`; diff --git a/packages/ozone/tests/blob-divert.test.ts b/packages/ozone/tests/blob-divert.test.ts new file mode 100644 index 00000000000..0890ac8a136 --- /dev/null +++ b/packages/ozone/tests/blob-divert.test.ts @@ -0,0 +1,90 @@ +import { + ModeratorClient, + SeedClient, + TestNetwork, + basicSeed, +} from '@atproto/dev-env' +import AtpAgent from '@atproto/api' +import { BlobDiverter } from '../src/daemon' +import { forSnapshot } from './_util' + +describe('blob divert', () => { + let network: TestNetwork + let agent: AtpAgent + let sc: SeedClient + let modClient: ModeratorClient + + beforeAll(async () => { + network = await TestNetwork.create({ + dbPostgresSchema: 'ozone_blob_divert_test', + ozone: { + blobDivertUrl: `https://blob-report.com`, + blobDivertAdminPassword: 'test-auth-token', + }, + }) + agent = network.pds.getClient() + sc = network.getSeedClient() + modClient = network.ozone.getModClient() + await basicSeed(sc) + await network.processAll() + }) + + afterAll(async () => { + await network.close() + }) + + const mockReportServiceResponse = (result: boolean) => { + return jest + .spyOn(BlobDiverter.prototype, 'sendImage') + .mockImplementation(async () => { + return result + }) + } + + const getSubject = () => ({ + $type: 'com.atproto.repo.strongRef', + uri: sc.posts[sc.dids.carol][0].ref.uriStr, + cid: sc.posts[sc.dids.carol][0].ref.cidStr, + }) + + const emitDivertEvent = async () => + modClient.emitModerationEvent( + { + subject: getSubject(), + event: { + $type: 'com.atproto.admin.defs#modEventDivert', + comment: 'Diverting for test', + }, + createdBy: sc.dids.alice, + subjectBlobCids: sc.posts[sc.dids.carol][0].images.map((img) => + img.image.ref.toString(), + ), + }, + 'moderator', + ) + + it('fails and keeps attempt count when report service fails to accept upload.', async () => { + // Simulate failure to fail upload + const reportServiceRequest = mockReportServiceResponse(false) + + await expect(emitDivertEvent()).rejects.toThrow() + + expect(reportServiceRequest).toHaveBeenCalled() + }) + + it('sends blobs to configured divert service and marks divert date', async () => { + // Simulate failure to accept upload + const reportServiceRequest = mockReportServiceResponse(true) + + const divertEvent = await emitDivertEvent() + + expect(reportServiceRequest).toHaveBeenCalled() + expect(forSnapshot(divertEvent)).toMatchSnapshot() + + const { subjectStatuses } = await modClient.queryModerationStatuses({ + subject: getSubject().uri, + }) + + expect(subjectStatuses[0].takendown).toBe(true) + }) +}) diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index a7934d471d5..e9a585f5a21 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -92,6 +92,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', + 'lex:com.atproto.admin.defs#modEventDivert', ], }, subject: { @@ -150,6 +151,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', + 'lex:com.atproto.admin.defs#modEventDivert', ], }, subject: { @@ -940,6 +942,16 @@ export const schemaDict = { }, }, }, + modEventDivert: { + type: 'object', + description: + "Divert a record's blobs to a 3rd party service for further scanning/tagging", + properties: { + comment: { + type: 'string', + }, + }, + }, communicationTemplateView: { type: 'object', required: [ diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts index a860e6bcfa0..bc374807c83 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts @@ -41,6 +41,7 @@ export interface ModEventView { | ModEventMute | ModEventEmail | ModEventResolveAppeal + | ModEventDivert | { $type: string; [k: string]: unknown } subject: | RepoRef @@ -79,6 +80,7 @@ export interface ModEventViewDetail { | ModEventMute | ModEventEmail | ModEventResolveAppeal + | ModEventDivert | { $type: string; [k: string]: unknown } subject: | RepoView @@ -749,6 +751,24 @@ export function validateModEventTag(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#modEventTag', v) } +/** Divert a record's blobs to a 3rd party service for further scanning/tagging */ +export interface ModEventDivert { + comment?: string + [k: string]: unknown +} + +export function isModEventDivert(v: unknown): v is ModEventDivert { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventDivert' + ) +} + +export function validateModEventDivert(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventDivert', v) +} + export interface CommunicationTemplateView { id: string /** Name of the template. */ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 47fabe9593b..1f673eeaa28 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,9 +1,5 @@ lockfileVersion: '6.0' -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - importers: .: @@ -12055,3 +12051,7 @@ packages: /zod@3.21.4: resolution: {integrity: sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==} + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false From 31e67a5a0276778d5127a4f91d1dd4da95dc9fd5 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Tue, 12 Mar 2024 16:01:28 -0500 Subject: [PATCH 2/5] Remove sig in label hydration (#2295) remove sig in label hydration --- packages/bsky/src/hydration/label.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/bsky/src/hydration/label.ts b/packages/bsky/src/hydration/label.ts index 458d46fcfc5..9b47e0454ae 100644 --- a/packages/bsky/src/hydration/label.ts +++ b/packages/bsky/src/hydration/label.ts @@ -40,8 +40,9 @@ export class LabelHydrator { if (!subjects.length || !issuers.length) return new HydrationMap() const res = await this.dataplane.getLabels({ subjects, issuers }) return res.labels.reduce((acc, cur) => { - const label = parseJsonBytes(cur) as Label | undefined - if (!label || label.neg) return acc + const parsed = parseJsonBytes(cur) as Label | undefined + if (!parsed || parsed.neg) return acc + const { sig: _, ...label } = parsed const entry = acc.get(label.uri) if (entry) { entry.push(label) From 40a3537b0591deebaa072700e7015ef584812377 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Tue, 12 Mar 2024 16:01:40 -0500 Subject: [PATCH 3/5] Allow 3p labelers to query appview account infos (#2297) allow 3p labelers to query appview account infos --- .../src/api/com/atproto/admin/getAccountInfos.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/bsky/src/api/com/atproto/admin/getAccountInfos.ts b/packages/bsky/src/api/com/atproto/admin/getAccountInfos.ts index cee84d53177..971839d5afd 100644 --- a/packages/bsky/src/api/com/atproto/admin/getAccountInfos.ts +++ b/packages/bsky/src/api/com/atproto/admin/getAccountInfos.ts @@ -5,18 +5,25 @@ import { INVALID_HANDLE } from '@atproto/syntax' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.getAccountInfos({ - auth: ctx.authVerifier.roleOrModService, - handler: async ({ params }) => { + auth: ctx.authVerifier.optionalStandardOrRole, + handler: async ({ params, auth }) => { const { dids } = params + const { canViewTakedowns } = ctx.authVerifier.parseCreds(auth) + const actors = await ctx.hydrator.actor.getActors(dids, true) const infos = mapDefined(dids, (did) => { const info = actors.get(did) if (!info) return + if (info.takedownRef && !canViewTakedowns) return + const profileRecord = + !info.profileTakedownRef || canViewTakedowns + ? info.profile + : undefined return { did, handle: info.handle ?? INVALID_HANDLE, - relatedRecords: info.profile ? [info.profile] : undefined, + relatedRecords: profileRecord ? [profileRecord] : undefined, indexedAt: (info.sortedAt ?? new Date(0)).toISOString(), } }) From 8dd67f5c8160140dab9953dbba0621b972e9f8bb Mon Sep 17 00:00:00 2001 From: devin ivy Date: Tue, 12 Mar 2024 17:04:02 -0400 Subject: [PATCH 4/5] Appview: labels on feed gens and list views (#2298) * lexicon: add labels to feedgen and list views * appview: hydrate labels onto lists, feedgens * test hydration of labels on lists and feedgens * update pds snap --- lexicons/app/bsky/feed/defs.json | 4 + lexicons/app/bsky/graph/defs.json | 8 ++ packages/api/src/client/lexicons.ts | 21 ++++ .../src/client/types/app/bsky/feed/defs.ts | 1 + .../src/client/types/app/bsky/graph/defs.ts | 3 + packages/bsky/src/hydration/hydrator.ts | 9 +- packages/bsky/src/lexicon/lexicons.ts | 21 ++++ .../src/lexicon/types/app/bsky/feed/defs.ts | 1 + .../src/lexicon/types/app/bsky/graph/defs.ts | 3 + packages/bsky/src/views/index.ts | 4 + .../feed-generation.test.ts.snap | 12 +++ packages/bsky/tests/label-hydration.test.ts | 101 +++++++++++++----- .../__snapshots__/block-lists.test.ts.snap | 7 ++ .../__snapshots__/mute-lists.test.ts.snap | 10 ++ .../__snapshots__/threadgating.test.ts.snap | 2 + packages/ozone/src/lexicon/lexicons.ts | 21 ++++ .../src/lexicon/types/app/bsky/feed/defs.ts | 1 + .../src/lexicon/types/app/bsky/graph/defs.ts | 3 + packages/pds/src/lexicon/lexicons.ts | 21 ++++ .../src/lexicon/types/app/bsky/feed/defs.ts | 1 + .../src/lexicon/types/app/bsky/graph/defs.ts | 3 + .../proxied/__snapshots__/views.test.ts.snap | 6 ++ 22 files changed, 231 insertions(+), 32 deletions(-) diff --git a/lexicons/app/bsky/feed/defs.json b/lexicons/app/bsky/feed/defs.json index 7f121e88403..7c40ef3962f 100644 --- a/lexicons/app/bsky/feed/defs.json +++ b/lexicons/app/bsky/feed/defs.json @@ -137,6 +137,10 @@ }, "avatar": { "type": "string" }, "likeCount": { "type": "integer", "minimum": 0 }, + "labels": { + "type": "array", + "items": { "type": "ref", "ref": "com.atproto.label.defs#label" } + }, "viewer": { "type": "ref", "ref": "#generatorViewerState" }, "indexedAt": { "type": "string", "format": "datetime" } } diff --git a/lexicons/app/bsky/graph/defs.json b/lexicons/app/bsky/graph/defs.json index 737d984d08b..be718f20a16 100644 --- a/lexicons/app/bsky/graph/defs.json +++ b/lexicons/app/bsky/graph/defs.json @@ -11,6 +11,10 @@ "name": { "type": "string", "maxLength": 64, "minLength": 1 }, "purpose": { "type": "ref", "ref": "#listPurpose" }, "avatar": { "type": "string" }, + "labels": { + "type": "array", + "items": { "type": "ref", "ref": "com.atproto.label.defs#label" } + }, "viewer": { "type": "ref", "ref": "#listViewerState" }, "indexedAt": { "type": "string", "format": "datetime" } } @@ -34,6 +38,10 @@ "items": { "type": "ref", "ref": "app.bsky.richtext.facet" } }, "avatar": { "type": "string" }, + "labels": { + "type": "array", + "items": { "type": "ref", "ref": "com.atproto.label.defs#label" } + }, "viewer": { "type": "ref", "ref": "#listViewerState" }, "indexedAt": { "type": "string", "format": "datetime" } } diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index e9a585f5a21..1c9d18c1f17 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -6322,6 +6322,13 @@ export const schemaDict = { type: 'integer', minimum: 0, }, + labels: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.label.defs#label', + }, + }, viewer: { type: 'ref', ref: 'lex:app.bsky.feed.defs#generatorViewerState', @@ -7626,6 +7633,13 @@ export const schemaDict = { avatar: { type: 'string', }, + labels: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.label.defs#label', + }, + }, viewer: { type: 'ref', ref: 'lex:app.bsky.graph.defs#listViewerState', @@ -7676,6 +7690,13 @@ export const schemaDict = { avatar: { type: 'string', }, + labels: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.label.defs#label', + }, + }, viewer: { type: 'ref', ref: 'lex:app.bsky.graph.defs#listViewerState', diff --git a/packages/api/src/client/types/app/bsky/feed/defs.ts b/packages/api/src/client/types/app/bsky/feed/defs.ts index 949b8fb975e..856d5356086 100644 --- a/packages/api/src/client/types/app/bsky/feed/defs.ts +++ b/packages/api/src/client/types/app/bsky/feed/defs.ts @@ -219,6 +219,7 @@ export interface GeneratorView { descriptionFacets?: AppBskyRichtextFacet.Main[] avatar?: string likeCount?: number + labels?: ComAtprotoLabelDefs.Label[] viewer?: GeneratorViewerState indexedAt: string [k: string]: unknown diff --git a/packages/api/src/client/types/app/bsky/graph/defs.ts b/packages/api/src/client/types/app/bsky/graph/defs.ts index 0580b9bb158..78d3ac24d62 100644 --- a/packages/api/src/client/types/app/bsky/graph/defs.ts +++ b/packages/api/src/client/types/app/bsky/graph/defs.ts @@ -5,6 +5,7 @@ import { ValidationResult, BlobRef } from '@atproto/lexicon' import { isObj, hasProp } from '../../../../util' import { lexicons } from '../../../../lexicons' import { CID } from 'multiformats/cid' +import * as ComAtprotoLabelDefs from '../../../com/atproto/label/defs' import * as AppBskyActorDefs from '../actor/defs' import * as AppBskyRichtextFacet from '../richtext/facet' @@ -14,6 +15,7 @@ export interface ListViewBasic { name: string purpose: ListPurpose avatar?: string + labels?: ComAtprotoLabelDefs.Label[] viewer?: ListViewerState indexedAt?: string [k: string]: unknown @@ -40,6 +42,7 @@ export interface ListView { description?: string descriptionFacets?: AppBskyRichtextFacet.Main[] avatar?: string + labels?: ComAtprotoLabelDefs.Label[] viewer?: ListViewerState indexedAt: string [k: string]: unknown diff --git a/packages/bsky/src/hydration/hydrator.ts b/packages/bsky/src/hydration/hydrator.ts index 402e4985b5e..9181032b7da 100644 --- a/packages/bsky/src/hydration/hydrator.ts +++ b/packages/bsky/src/hydration/hydrator.ts @@ -194,11 +194,12 @@ export class Hydrator { uris: string[], ctx: HydrateCtx, ): Promise { - const [lists, listViewers] = await Promise.all([ + const [lists, listViewers, labels] = await Promise.all([ this.graph.getLists(uris), ctx.viewer ? this.graph.getListViewerStates(uris, ctx.viewer) : undefined, + this.label.getLabelsForSubjects(uris, ctx.labelers), ]) - return { lists, listViewers, ctx } + return { lists, listViewers, labels, ctx } } // app.bsky.graph.defs#listItemView @@ -430,7 +431,7 @@ export class Hydrator { uris: string[], // @TODO any way to get refs here? ctx: HydrateCtx, ): Promise { - const [feedgens, feedgenAggs, feedgenViewers, profileState] = + const [feedgens, feedgenAggs, feedgenViewers, profileState, labels] = await Promise.all([ this.feed.getFeedGens(uris), this.feed.getFeedGenAggregates(uris.map((uri) => ({ uri }))), @@ -438,11 +439,13 @@ export class Hydrator { ? this.feed.getFeedGenViewerStates(uris, ctx.viewer) : undefined, this.hydrateProfiles(uris.map(didFromUri), ctx), + this.label.getLabelsForSubjects(uris, ctx.labelers), ]) return mergeStates(profileState, { feedgens, feedgenAggs, feedgenViewers, + labels, ctx, }) } diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index e9a585f5a21..1c9d18c1f17 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -6322,6 +6322,13 @@ export const schemaDict = { type: 'integer', minimum: 0, }, + labels: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.label.defs#label', + }, + }, viewer: { type: 'ref', ref: 'lex:app.bsky.feed.defs#generatorViewerState', @@ -7626,6 +7633,13 @@ export const schemaDict = { avatar: { type: 'string', }, + labels: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.label.defs#label', + }, + }, viewer: { type: 'ref', ref: 'lex:app.bsky.graph.defs#listViewerState', @@ -7676,6 +7690,13 @@ export const schemaDict = { avatar: { type: 'string', }, + labels: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.label.defs#label', + }, + }, viewer: { type: 'ref', ref: 'lex:app.bsky.graph.defs#listViewerState', diff --git a/packages/bsky/src/lexicon/types/app/bsky/feed/defs.ts b/packages/bsky/src/lexicon/types/app/bsky/feed/defs.ts index 261d8a622ec..8c99f64f32e 100644 --- a/packages/bsky/src/lexicon/types/app/bsky/feed/defs.ts +++ b/packages/bsky/src/lexicon/types/app/bsky/feed/defs.ts @@ -219,6 +219,7 @@ export interface GeneratorView { descriptionFacets?: AppBskyRichtextFacet.Main[] avatar?: string likeCount?: number + labels?: ComAtprotoLabelDefs.Label[] viewer?: GeneratorViewerState indexedAt: string [k: string]: unknown diff --git a/packages/bsky/src/lexicon/types/app/bsky/graph/defs.ts b/packages/bsky/src/lexicon/types/app/bsky/graph/defs.ts index 31791dd3647..0ce8a364a8e 100644 --- a/packages/bsky/src/lexicon/types/app/bsky/graph/defs.ts +++ b/packages/bsky/src/lexicon/types/app/bsky/graph/defs.ts @@ -5,6 +5,7 @@ import { ValidationResult, BlobRef } from '@atproto/lexicon' import { lexicons } from '../../../../lexicons' import { isObj, hasProp } from '../../../../util' import { CID } from 'multiformats/cid' +import * as ComAtprotoLabelDefs from '../../../com/atproto/label/defs' import * as AppBskyActorDefs from '../actor/defs' import * as AppBskyRichtextFacet from '../richtext/facet' @@ -14,6 +15,7 @@ export interface ListViewBasic { name: string purpose: ListPurpose avatar?: string + labels?: ComAtprotoLabelDefs.Label[] viewer?: ListViewerState indexedAt?: string [k: string]: unknown @@ -40,6 +42,7 @@ export interface ListView { description?: string descriptionFacets?: AppBskyRichtextFacet.Main[] avatar?: string + labels?: ComAtprotoLabelDefs.Label[] viewer?: ListViewerState indexedAt: string [k: string]: unknown diff --git a/packages/bsky/src/views/index.ts b/packages/bsky/src/views/index.ts index 4c573d6ace6..29a5238d383 100644 --- a/packages/bsky/src/views/index.ts +++ b/packages/bsky/src/views/index.ts @@ -223,6 +223,7 @@ export class Views { return undefined } const listViewer = state.listViewers?.get(uri) + const labels = state.labels?.get(uri) ?? [] const creator = new AtUri(uri).hostname return { uri, @@ -237,6 +238,7 @@ export class Views { ) : undefined, indexedAt: list.sortedAt.toISOString(), + labels, viewer: listViewer ? { muted: !!listViewer.viewerMuted, @@ -347,6 +349,7 @@ export class Views { if (!creator) return const viewer = state.feedgenViewers?.get(uri) const aggs = state.feedgenAggs?.get(uri) + const labels = state.labels?.get(uri) ?? [] return { uri, @@ -364,6 +367,7 @@ export class Views { ) : undefined, likeCount: aggs?.likes, + labels, viewer: viewer ? { like: viewer.like, diff --git a/packages/bsky/tests/__snapshots__/feed-generation.test.ts.snap b/packages/bsky/tests/__snapshots__/feed-generation.test.ts.snap index 8aea15ddfc8..d4844cbfad5 100644 --- a/packages/bsky/tests/__snapshots__/feed-generation.test.ts.snap +++ b/packages/bsky/tests/__snapshots__/feed-generation.test.ts.snap @@ -99,6 +99,7 @@ Object { "did": "user(2)", "displayName": "All", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "likeCount": 2, "uri": "record(1)", "viewer": Object { @@ -168,6 +169,7 @@ Array [ "did": "user(0)", "displayName": "Odd", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "likeCount": 0, "uri": "record(0)", "viewer": Object {}, @@ -210,6 +212,7 @@ Array [ "did": "user(0)", "displayName": "Needs Auth", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "likeCount": 0, "uri": "record(4)", "viewer": Object {}, @@ -252,6 +255,7 @@ Array [ "did": "user(0)", "displayName": "Bad Pagination", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "likeCount": 0, "uri": "record(5)", "viewer": Object {}, @@ -294,6 +298,7 @@ Array [ "did": "user(0)", "displayName": "Even", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "likeCount": 0, "uri": "record(6)", "viewer": Object {}, @@ -336,6 +341,7 @@ Array [ "did": "user(0)", "displayName": "All", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "likeCount": 2, "uri": "record(7)", "viewer": Object { @@ -1493,6 +1499,7 @@ Object { "did": "user(0)", "displayName": "All", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "likeCount": 2, "uri": "record(0)", "viewer": Object { @@ -1543,6 +1550,7 @@ Object { "did": "user(0)", "displayName": "Even", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "likeCount": 0, "uri": "record(0)", "viewer": Object {}, @@ -1585,6 +1593,7 @@ Object { "did": "user(0)", "displayName": "All", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "likeCount": 2, "uri": "record(4)", "viewer": Object { @@ -1637,6 +1646,7 @@ Object { "did": "user(0)", "displayName": "All", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "likeCount": 2, "uri": "record(0)", "viewer": Object { @@ -1681,6 +1691,7 @@ Object { "did": "user(0)", "displayName": "Even", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "likeCount": 0, "uri": "record(5)", "viewer": Object {}, @@ -1723,6 +1734,7 @@ Object { "did": "user(0)", "displayName": "Bad Pagination", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "likeCount": 0, "uri": "record(6)", "viewer": Object {}, diff --git a/packages/bsky/tests/label-hydration.test.ts b/packages/bsky/tests/label-hydration.test.ts index ab672619f86..a7fe678581c 100644 --- a/packages/bsky/tests/label-hydration.test.ts +++ b/packages/bsky/tests/label-hydration.test.ts @@ -22,35 +22,14 @@ describe('label hydration', () => { bob = sc.dids.bob carol = sc.dids.carol labelerDid = network.bsky.ctx.cfg.labelsFromIssuerDids[0] - await network.bsky.db.db - .insertInto('label') - .values([ - { - src: alice, - uri: carol, - cid: '', - val: 'spam', - neg: false, - cts: new Date().toISOString(), - }, - { - src: bob, - uri: carol, - cid: '', - val: 'impersonation', - neg: false, - cts: new Date().toISOString(), - }, - { - src: labelerDid, - uri: carol, - cid: '', - val: 'misleading', - neg: false, - cts: new Date().toISOString(), - }, - ]) - .execute() + await createLabel({ src: alice, uri: carol, cid: '', val: 'spam' }) + await createLabel({ src: bob, uri: carol, cid: '', val: 'impersonation' }) + await createLabel({ + src: labelerDid, + uri: carol, + cid: '', + val: 'misleading', + }) await network.processAll() }) @@ -97,4 +76,68 @@ describe('label hydration', () => { expect(res.data.labels?.[0].src).toBe(labelerDid) expect(res.data.labels?.[0].val).toBe('misleading') }) + + it('hydrates labels onto list views.', async () => { + const list = await pdsAgent.api.app.bsky.graph.list.create( + { repo: alice }, + { + name: "alice's modlist", + purpose: 'app.bsky.graph.defs#modlist', + createdAt: new Date().toISOString(), + }, + sc.getHeaders(alice), + ) + await network.processAll() + await createLabel({ uri: list.uri, cid: list.cid, val: 'spam' }) + const res = await pdsAgent.api.app.bsky.graph.getList( + { list: list.uri }, + { headers: sc.getHeaders(alice) }, + ) + const [label, ...others] = res.data.list.labels ?? [] + expect(label?.src).toBe(labelerDid) + expect(label?.val).toBe('spam') + expect(others.length).toBe(0) + }) + + it('hydrates labels onto feed generator views.', async () => { + const feedgen = await pdsAgent.api.app.bsky.feed.generator.create( + { repo: alice }, + { + displayName: "alice's feedgen", + did: alice, + createdAt: new Date().toISOString(), + }, + sc.getHeaders(alice), + ) + await network.processAll() + await createLabel({ uri: feedgen.uri, cid: feedgen.cid, val: 'spam' }) + const res = await pdsAgent.api.app.bsky.feed.getFeedGenerators( + { feeds: [feedgen.uri] }, + { headers: sc.getHeaders(alice) }, + ) + expect(res.data.feeds.length).toBe(1) + const [label, ...others] = res.data.feeds[0].labels ?? [] + expect(label?.src).toBe(labelerDid) + expect(label?.val).toBe('spam') + expect(others.length).toBe(0) + }) + + const createLabel = async (opts: { + src?: string + uri: string + cid: string + val: string + }) => { + await network.bsky.db.db + .insertInto('label') + .values({ + uri: opts.uri, + cid: opts.cid, + val: opts.val, + cts: new Date().toISOString(), + neg: false, + src: opts.src ?? 'did:example:labeler', + }) + .execute() + } }) diff --git a/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap b/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap index c5ff586f5c2..0130e6eae31 100644 --- a/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap +++ b/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap @@ -325,6 +325,7 @@ Object { }, "description": "blah blah", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "new list", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(0)", @@ -369,6 +370,7 @@ Object { }, "description": "big list of blocks", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "alice blocks", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(4)", @@ -420,6 +422,7 @@ Object { }, "description": "blah blah", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "new list", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(0)", @@ -463,6 +466,7 @@ Object { }, "description": "big list of blocks", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "alice blocks", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(3)", @@ -503,6 +507,7 @@ Object { "avatar": "https://bsky.public.url/img/avatar/plain/user(0)/cids(1)@jpeg", "cid": "cids(0)", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "alice blocks", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(0)", @@ -532,6 +537,7 @@ Object { "avatar": "https://bsky.public.url/img/avatar/plain/user(0)/cids(1)@jpeg", "cid": "cids(0)", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "alice blocks", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(0)", @@ -582,6 +588,7 @@ Object { }, "description": "big list of blocks", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "alice blocks", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(0)", diff --git a/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap b/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap index 4cd94ec2efc..f3f0fc1d06c 100644 --- a/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap +++ b/packages/bsky/tests/views/__snapshots__/mute-lists.test.ts.snap @@ -68,6 +68,7 @@ Object { }, "description": "new descript", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "updated alice mutes", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(2)", @@ -165,6 +166,7 @@ Object { "avatar": "https://bsky.public.url/img/avatar/plain/user(1)/cids(1)@jpeg", "cid": "cids(4)", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "alice mutes", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(6)", @@ -216,6 +218,7 @@ Object { "avatar": "https://bsky.public.url/img/avatar/plain/user(1)/cids(1)@jpeg", "cid": "cids(4)", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "alice mutes", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(6)", @@ -336,6 +339,7 @@ Object { }, "description": "blah blah", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "new list", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(0)", @@ -379,6 +383,7 @@ Object { }, "description": "big list of mutes", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "alice mutes", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(3)", @@ -429,6 +434,7 @@ Object { }, "description": "blah blah", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "new list", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(0)", @@ -472,6 +478,7 @@ Object { }, "description": "big list of mutes", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "alice mutes", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(3)", @@ -512,6 +519,7 @@ Object { "avatar": "https://bsky.public.url/img/avatar/plain/user(0)/cids(1)@jpeg", "cid": "cids(0)", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "alice mutes", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(0)", @@ -540,6 +548,7 @@ Object { "avatar": "https://bsky.public.url/img/avatar/plain/user(0)/cids(1)@jpeg", "cid": "cids(0)", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "alice mutes", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(0)", @@ -588,6 +597,7 @@ Object { }, "description": "big list of mutes", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "alice mutes", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(0)", diff --git a/packages/bsky/tests/views/__snapshots__/threadgating.test.ts.snap b/packages/bsky/tests/views/__snapshots__/threadgating.test.ts.snap index 1545c19e9f4..9c460569d89 100644 --- a/packages/bsky/tests/views/__snapshots__/threadgating.test.ts.snap +++ b/packages/bsky/tests/views/__snapshots__/threadgating.test.ts.snap @@ -41,6 +41,7 @@ Object { Object { "cid": "cids(1)", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "list a", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(2)", @@ -51,6 +52,7 @@ Object { Object { "cid": "cids(2)", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "list b", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(3)", diff --git a/packages/ozone/src/lexicon/lexicons.ts b/packages/ozone/src/lexicon/lexicons.ts index e9a585f5a21..1c9d18c1f17 100644 --- a/packages/ozone/src/lexicon/lexicons.ts +++ b/packages/ozone/src/lexicon/lexicons.ts @@ -6322,6 +6322,13 @@ export const schemaDict = { type: 'integer', minimum: 0, }, + labels: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.label.defs#label', + }, + }, viewer: { type: 'ref', ref: 'lex:app.bsky.feed.defs#generatorViewerState', @@ -7626,6 +7633,13 @@ export const schemaDict = { avatar: { type: 'string', }, + labels: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.label.defs#label', + }, + }, viewer: { type: 'ref', ref: 'lex:app.bsky.graph.defs#listViewerState', @@ -7676,6 +7690,13 @@ export const schemaDict = { avatar: { type: 'string', }, + labels: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.label.defs#label', + }, + }, viewer: { type: 'ref', ref: 'lex:app.bsky.graph.defs#listViewerState', diff --git a/packages/ozone/src/lexicon/types/app/bsky/feed/defs.ts b/packages/ozone/src/lexicon/types/app/bsky/feed/defs.ts index 261d8a622ec..8c99f64f32e 100644 --- a/packages/ozone/src/lexicon/types/app/bsky/feed/defs.ts +++ b/packages/ozone/src/lexicon/types/app/bsky/feed/defs.ts @@ -219,6 +219,7 @@ export interface GeneratorView { descriptionFacets?: AppBskyRichtextFacet.Main[] avatar?: string likeCount?: number + labels?: ComAtprotoLabelDefs.Label[] viewer?: GeneratorViewerState indexedAt: string [k: string]: unknown diff --git a/packages/ozone/src/lexicon/types/app/bsky/graph/defs.ts b/packages/ozone/src/lexicon/types/app/bsky/graph/defs.ts index 31791dd3647..0ce8a364a8e 100644 --- a/packages/ozone/src/lexicon/types/app/bsky/graph/defs.ts +++ b/packages/ozone/src/lexicon/types/app/bsky/graph/defs.ts @@ -5,6 +5,7 @@ import { ValidationResult, BlobRef } from '@atproto/lexicon' import { lexicons } from '../../../../lexicons' import { isObj, hasProp } from '../../../../util' import { CID } from 'multiformats/cid' +import * as ComAtprotoLabelDefs from '../../../com/atproto/label/defs' import * as AppBskyActorDefs from '../actor/defs' import * as AppBskyRichtextFacet from '../richtext/facet' @@ -14,6 +15,7 @@ export interface ListViewBasic { name: string purpose: ListPurpose avatar?: string + labels?: ComAtprotoLabelDefs.Label[] viewer?: ListViewerState indexedAt?: string [k: string]: unknown @@ -40,6 +42,7 @@ export interface ListView { description?: string descriptionFacets?: AppBskyRichtextFacet.Main[] avatar?: string + labels?: ComAtprotoLabelDefs.Label[] viewer?: ListViewerState indexedAt: string [k: string]: unknown diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index e9a585f5a21..1c9d18c1f17 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -6322,6 +6322,13 @@ export const schemaDict = { type: 'integer', minimum: 0, }, + labels: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.label.defs#label', + }, + }, viewer: { type: 'ref', ref: 'lex:app.bsky.feed.defs#generatorViewerState', @@ -7626,6 +7633,13 @@ export const schemaDict = { avatar: { type: 'string', }, + labels: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.label.defs#label', + }, + }, viewer: { type: 'ref', ref: 'lex:app.bsky.graph.defs#listViewerState', @@ -7676,6 +7690,13 @@ export const schemaDict = { avatar: { type: 'string', }, + labels: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:com.atproto.label.defs#label', + }, + }, viewer: { type: 'ref', ref: 'lex:app.bsky.graph.defs#listViewerState', diff --git a/packages/pds/src/lexicon/types/app/bsky/feed/defs.ts b/packages/pds/src/lexicon/types/app/bsky/feed/defs.ts index 261d8a622ec..8c99f64f32e 100644 --- a/packages/pds/src/lexicon/types/app/bsky/feed/defs.ts +++ b/packages/pds/src/lexicon/types/app/bsky/feed/defs.ts @@ -219,6 +219,7 @@ export interface GeneratorView { descriptionFacets?: AppBskyRichtextFacet.Main[] avatar?: string likeCount?: number + labels?: ComAtprotoLabelDefs.Label[] viewer?: GeneratorViewerState indexedAt: string [k: string]: unknown diff --git a/packages/pds/src/lexicon/types/app/bsky/graph/defs.ts b/packages/pds/src/lexicon/types/app/bsky/graph/defs.ts index 31791dd3647..0ce8a364a8e 100644 --- a/packages/pds/src/lexicon/types/app/bsky/graph/defs.ts +++ b/packages/pds/src/lexicon/types/app/bsky/graph/defs.ts @@ -5,6 +5,7 @@ import { ValidationResult, BlobRef } from '@atproto/lexicon' import { lexicons } from '../../../../lexicons' import { isObj, hasProp } from '../../../../util' import { CID } from 'multiformats/cid' +import * as ComAtprotoLabelDefs from '../../../com/atproto/label/defs' import * as AppBskyActorDefs from '../actor/defs' import * as AppBskyRichtextFacet from '../richtext/facet' @@ -14,6 +15,7 @@ export interface ListViewBasic { name: string purpose: ListPurpose avatar?: string + labels?: ComAtprotoLabelDefs.Label[] viewer?: ListViewerState indexedAt?: string [k: string]: unknown @@ -40,6 +42,7 @@ export interface ListView { description?: string descriptionFacets?: AppBskyRichtextFacet.Main[] avatar?: string + labels?: ComAtprotoLabelDefs.Label[] viewer?: ListViewerState indexedAt: string [k: string]: unknown diff --git a/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap b/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap index 5daad7572b3..275cdea90b9 100644 --- a/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap +++ b/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap @@ -554,6 +554,7 @@ Object { "did": "user(0)", "displayName": "MyFeed", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "likeCount": 0, "uri": "record(0)", "viewer": Object {}, @@ -599,6 +600,7 @@ Object { "did": "user(0)", "displayName": "MyFeed", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "likeCount": 0, "uri": "record(0)", "viewer": Object {}, @@ -3309,6 +3311,7 @@ Object { }, "description": "bob's list of mutes", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "bob mutes", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(0)", @@ -3340,6 +3343,7 @@ Object { }, "description": "bob's list of mutes", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "bob mutes", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(0)", @@ -3375,6 +3379,7 @@ Object { }, "description": "a second list", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "another list", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(0)", @@ -3401,6 +3406,7 @@ Object { }, "description": "bob's list of mutes", "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], "name": "bob mutes", "purpose": "app.bsky.graph.defs#modlist", "uri": "record(3)", From 7b146605c9ad051ac7cdec3b2235dec28717501e Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Tue, 12 Mar 2024 16:04:12 -0500 Subject: [PATCH 5/5] Updated semantics for atproto labelers header (#2292) * update labelers header semantics * add response header * pds: pipe through res headers * fix up tests * revamp parsing --- packages/bsky/package.json | 1 + .../bsky/src/api/app/bsky/actor/getProfile.ts | 9 ++-- .../src/api/app/bsky/actor/getProfiles.ts | 9 ++-- .../src/api/app/bsky/actor/getSuggestions.ts | 2 + .../src/api/app/bsky/actor/searchActors.ts | 2 + .../app/bsky/actor/searchActorsTypeahead.ts | 2 + .../src/api/app/bsky/feed/getActorFeeds.ts | 3 +- .../src/api/app/bsky/feed/getActorLikes.ts | 9 ++-- .../src/api/app/bsky/feed/getAuthorFeed.ts | 9 ++-- .../bsky/src/api/app/bsky/feed/getFeed.ts | 14 +++--- .../src/api/app/bsky/feed/getFeedGenerator.ts | 2 + .../api/app/bsky/feed/getFeedGenerators.ts | 2 + .../bsky/src/api/app/bsky/feed/getLikes.ts | 3 +- .../bsky/src/api/app/bsky/feed/getListFeed.ts | 6 +-- .../src/api/app/bsky/feed/getPostThread.ts | 11 +++-- .../bsky/src/api/app/bsky/feed/getPosts.ts | 2 + .../src/api/app/bsky/feed/getRepostedBy.ts | 3 +- .../api/app/bsky/feed/getSuggestedFeeds.ts | 2 + .../bsky/src/api/app/bsky/feed/getTimeline.ts | 6 +-- .../bsky/src/api/app/bsky/feed/searchPosts.ts | 2 + .../bsky/src/api/app/bsky/graph/getBlocks.ts | 3 +- .../src/api/app/bsky/graph/getFollowers.ts | 3 +- .../bsky/src/api/app/bsky/graph/getFollows.ts | 3 +- .../bsky/src/api/app/bsky/graph/getList.ts | 3 +- .../src/api/app/bsky/graph/getListBlocks.ts | 3 +- .../src/api/app/bsky/graph/getListMutes.ts | 3 +- .../bsky/src/api/app/bsky/graph/getLists.ts | 3 +- .../bsky/src/api/app/bsky/graph/getMutes.ts | 3 +- .../bsky/graph/getSuggestedFollowsByActor.ts | 2 + .../src/api/app/bsky/labeler/getServices.ts | 2 + .../bsky/notification/listNotifications.ts | 3 +- .../unspecced/getPopularFeedGenerators.ts | 3 +- packages/bsky/src/api/util.ts | 23 ++++++++-- packages/bsky/src/context.ts | 27 +++++++----- packages/bsky/src/hydration/hydrator.ts | 12 +++-- packages/bsky/src/util.ts | 44 +++++++++++++++++++ packages/bsky/tests/label-hydration.test.ts | 17 ++++++- packages/pds/src/pipethrough.ts | 27 ++++++++---- pnpm-lock.yaml | 8 ++++ 39 files changed, 223 insertions(+), 68 deletions(-) create mode 100644 packages/bsky/src/util.ts diff --git a/packages/bsky/package.json b/packages/bsky/package.json index ff1725ca980..9138c16d0be 100644 --- a/packages/bsky/package.json +++ b/packages/bsky/package.json @@ -63,6 +63,7 @@ "pino": "^8.15.0", "pino-http": "^8.2.1", "sharp": "^0.32.6", + "structured-headers": "^1.0.1", "typed-emitter": "^2.1.0", "uint8arrays": "3.0.0" }, diff --git a/packages/bsky/src/api/app/bsky/actor/getProfile.ts b/packages/bsky/src/api/app/bsky/actor/getProfile.ts index b7860791507..f58ecc7dbee 100644 --- a/packages/bsky/src/api/app/bsky/actor/getProfile.ts +++ b/packages/bsky/src/api/app/bsky/actor/getProfile.ts @@ -2,7 +2,7 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import { QueryParams } from '../../../../lexicon/types/app/bsky/actor/getProfile' import AppContext from '../../../../context' -import { setRepoRev } from '../../../util' +import { resHeaders } from '../../../util' import { createPipeline, noRules } from '../../../../pipeline' import { HydrateCtx, @@ -15,7 +15,7 @@ export default function (server: Server, ctx: AppContext) { const getProfile = createPipeline(skeleton, hydration, noRules, presentation) server.app.bsky.actor.getProfile({ auth: ctx.authVerifier.optionalStandardOrRole, - handler: async ({ auth, params, req, res }) => { + handler: async ({ auth, params, req }) => { const { viewer, canViewTakedowns } = ctx.authVerifier.parseCreds(auth) const labelers = ctx.reqLabelers(req) const hydrateCtx = { labelers, viewer } @@ -26,11 +26,14 @@ export default function (server: Server, ctx: AppContext) { ) const repoRev = await ctx.hydrator.actor.getRepoRevSafe(viewer) - setRepoRev(res, repoRev) return { encoding: 'application/json', body: result, + headers: resHeaders({ + repoRev, + labelers, + }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/actor/getProfiles.ts b/packages/bsky/src/api/app/bsky/actor/getProfiles.ts index 862ac239c7f..014f9339fe8 100644 --- a/packages/bsky/src/api/app/bsky/actor/getProfiles.ts +++ b/packages/bsky/src/api/app/bsky/actor/getProfiles.ts @@ -2,7 +2,7 @@ import { mapDefined } from '@atproto/common' import { Server } from '../../../../lexicon' import { QueryParams } from '../../../../lexicon/types/app/bsky/actor/getProfiles' import AppContext from '../../../../context' -import { setRepoRev } from '../../../util' +import { resHeaders } from '../../../util' import { createPipeline, noRules } from '../../../../pipeline' import { HydrateCtx, @@ -15,7 +15,7 @@ export default function (server: Server, ctx: AppContext) { const getProfile = createPipeline(skeleton, hydration, noRules, presentation) server.app.bsky.actor.getProfiles({ auth: ctx.authVerifier.standardOptional, - handler: async ({ auth, params, req, res }) => { + handler: async ({ auth, params, req }) => { const viewer = auth.credentials.iss const labelers = ctx.reqLabelers(req) const hydrateCtx = { viewer, labelers } @@ -23,11 +23,14 @@ export default function (server: Server, ctx: AppContext) { const result = await getProfile({ ...params, hydrateCtx }, ctx) const repoRev = await ctx.hydrator.actor.getRepoRevSafe(viewer) - setRepoRev(res, repoRev) return { encoding: 'application/json', body: result, + headers: resHeaders({ + repoRev, + labelers, + }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/actor/getSuggestions.ts b/packages/bsky/src/api/app/bsky/actor/getSuggestions.ts index db312373ea0..451fcedcb5e 100644 --- a/packages/bsky/src/api/app/bsky/actor/getSuggestions.ts +++ b/packages/bsky/src/api/app/bsky/actor/getSuggestions.ts @@ -11,6 +11,7 @@ import { import { Views } from '../../../../views' import { DataPlaneClient } from '../../../../data-plane' import { parseString } from '../../../../hydration/util' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getSuggestions = createPipeline( @@ -30,6 +31,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/actor/searchActors.ts b/packages/bsky/src/api/app/bsky/actor/searchActors.ts index b7bf881ab3a..adb3b6704ca 100644 --- a/packages/bsky/src/api/app/bsky/actor/searchActors.ts +++ b/packages/bsky/src/api/app/bsky/actor/searchActors.ts @@ -14,6 +14,7 @@ import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator' import { Views } from '../../../../views' import { DataPlaneClient } from '../../../../data-plane' import { parseString } from '../../../../hydration/util' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const searchActors = createPipeline( @@ -32,6 +33,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: results, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/actor/searchActorsTypeahead.ts b/packages/bsky/src/api/app/bsky/actor/searchActorsTypeahead.ts index 0ac1506f86d..7517c58ab9d 100644 --- a/packages/bsky/src/api/app/bsky/actor/searchActorsTypeahead.ts +++ b/packages/bsky/src/api/app/bsky/actor/searchActorsTypeahead.ts @@ -14,6 +14,7 @@ import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator' import { Views } from '../../../../views' import { DataPlaneClient } from '../../../../data-plane' import { parseString } from '../../../../hydration/util' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const searchActorsTypeahead = createPipeline( @@ -35,6 +36,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: results, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getActorFeeds.ts b/packages/bsky/src/api/app/bsky/feed/getActorFeeds.ts index b94486940fa..0ba5c15409a 100644 --- a/packages/bsky/src/api/app/bsky/feed/getActorFeeds.ts +++ b/packages/bsky/src/api/app/bsky/feed/getActorFeeds.ts @@ -12,7 +12,7 @@ import { import { Views } from '../../../../views' import { DataPlaneClient } from '../../../../data-plane' import { parseString } from '../../../../hydration/util' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getActorFeeds = createPipeline( @@ -31,6 +31,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getActorLikes.ts b/packages/bsky/src/api/app/bsky/feed/getActorLikes.ts index a8b85a3c155..4411f7295ea 100644 --- a/packages/bsky/src/api/app/bsky/feed/getActorLikes.ts +++ b/packages/bsky/src/api/app/bsky/feed/getActorLikes.ts @@ -3,7 +3,7 @@ import { mapDefined } from '@atproto/common' import { Server } from '../../../../lexicon' import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getActorLikes' import AppContext from '../../../../context' -import { clearlyBadCursor, setRepoRev } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' import { createPipeline } from '../../../../pipeline' import { HydrateCtx, @@ -25,7 +25,7 @@ export default function (server: Server, ctx: AppContext) { ) server.app.bsky.feed.getActorLikes({ auth: ctx.authVerifier.standardOptional, - handler: async ({ params, auth, req, res }) => { + handler: async ({ params, auth, req }) => { const viewer = auth.credentials.iss const labelers = ctx.reqLabelers(req) const hydrateCtx = { labelers, viewer } @@ -33,11 +33,14 @@ export default function (server: Server, ctx: AppContext) { const result = await getActorLikes({ ...params, hydrateCtx }, ctx) const repoRev = await ctx.hydrator.actor.getRepoRevSafe(viewer) - setRepoRev(res, repoRev) return { encoding: 'application/json', body: result, + headers: resHeaders({ + repoRev, + labelers, + }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getAuthorFeed.ts b/packages/bsky/src/api/app/bsky/feed/getAuthorFeed.ts index a8174b164fc..eb686b80910 100644 --- a/packages/bsky/src/api/app/bsky/feed/getAuthorFeed.ts +++ b/packages/bsky/src/api/app/bsky/feed/getAuthorFeed.ts @@ -3,7 +3,7 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getAuthorFeed' import AppContext from '../../../../context' -import { clearlyBadCursor, setRepoRev } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' import { createPipeline } from '../../../../pipeline' import { HydrateCtx, @@ -27,7 +27,7 @@ export default function (server: Server, ctx: AppContext) { ) server.app.bsky.feed.getAuthorFeed({ auth: ctx.authVerifier.optionalStandardOrRole, - handler: async ({ params, auth, req, res }) => { + handler: async ({ params, auth, req }) => { const { viewer, canViewTakedowns } = ctx.authVerifier.parseCreds(auth) const labelers = ctx.reqLabelers(req) const hydrateCtx = { labelers, viewer } @@ -38,11 +38,14 @@ export default function (server: Server, ctx: AppContext) { ) const repoRev = await ctx.hydrator.actor.getRepoRevSafe(viewer) - setRepoRev(res, repoRev) return { encoding: 'application/json', body: result, + headers: resHeaders({ + repoRev, + labelers, + }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getFeed.ts b/packages/bsky/src/api/app/bsky/feed/getFeed.ts index 465348f0522..26e17ea3e2d 100644 --- a/packages/bsky/src/api/app/bsky/feed/getFeed.ts +++ b/packages/bsky/src/api/app/bsky/feed/getFeed.ts @@ -28,6 +28,7 @@ import { isDataplaneError, unpackIdentityServices, } from '../../../../data-plane' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getFeed = createPipeline( @@ -47,16 +48,19 @@ export default function (server: Server, ctx: AppContext) { 'accept-language': req.headers['accept-language'], }) // @NOTE feed cursors should not be affected by appview swap - const { timerSkele, timerHydr, resHeaders, ...result } = await getFeed( - { ...params, hydrateCtx, headers }, - ctx, - ) + const { + timerSkele, + timerHydr, + resHeaders: feedResHeaders, + ...result + } = await getFeed({ ...params, hydrateCtx, headers }, ctx) return { encoding: 'application/json', body: result, headers: { - ...(resHeaders ?? {}), + ...(feedResHeaders ?? {}), + ...resHeaders({ labelers }), 'server-timing': serverTimingHeader([timerSkele, timerHydr]), }, } diff --git a/packages/bsky/src/api/app/bsky/feed/getFeedGenerator.ts b/packages/bsky/src/api/app/bsky/feed/getFeedGenerator.ts index 171c497e2ae..9c94fb2b213 100644 --- a/packages/bsky/src/api/app/bsky/feed/getFeedGenerator.ts +++ b/packages/bsky/src/api/app/bsky/feed/getFeedGenerator.ts @@ -8,6 +8,7 @@ import { isDataplaneError, unpackIdentityServices, } from '../../../../data-plane' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { server.app.bsky.feed.getFeedGenerator({ @@ -63,6 +64,7 @@ export default function (server: Server, ctx: AppContext) { isOnline: true, isValid: true, }, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getFeedGenerators.ts b/packages/bsky/src/api/app/bsky/feed/getFeedGenerators.ts index d8225384c97..65d5a5dd316 100644 --- a/packages/bsky/src/api/app/bsky/feed/getFeedGenerators.ts +++ b/packages/bsky/src/api/app/bsky/feed/getFeedGenerators.ts @@ -9,6 +9,7 @@ import { Hydrator, } from '../../../../hydration/hydrator' import { Views } from '../../../../views' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getFeedGenerators = createPipeline( @@ -27,6 +28,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: view, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getLikes.ts b/packages/bsky/src/api/app/bsky/feed/getLikes.ts index 5e9ebf7d70c..402a6c44a8a 100644 --- a/packages/bsky/src/api/app/bsky/feed/getLikes.ts +++ b/packages/bsky/src/api/app/bsky/feed/getLikes.ts @@ -12,7 +12,7 @@ import { import { Views } from '../../../../views' import { parseString } from '../../../../hydration/util' import { creatorFromUri } from '../../../../views/util' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getLikes = createPipeline(skeleton, hydration, noBlocks, presentation) @@ -27,6 +27,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getListFeed.ts b/packages/bsky/src/api/app/bsky/feed/getListFeed.ts index daed2a99025..75ae9edf815 100644 --- a/packages/bsky/src/api/app/bsky/feed/getListFeed.ts +++ b/packages/bsky/src/api/app/bsky/feed/getListFeed.ts @@ -1,7 +1,7 @@ import { Server } from '../../../../lexicon' import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getListFeed' import AppContext from '../../../../context' -import { clearlyBadCursor, setRepoRev } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' import { createPipeline } from '../../../../pipeline' import { HydrateCtx, @@ -23,7 +23,7 @@ export default function (server: Server, ctx: AppContext) { ) server.app.bsky.feed.getListFeed({ auth: ctx.authVerifier.standardOptional, - handler: async ({ params, auth, req, res }) => { + handler: async ({ params, auth, req }) => { const viewer = auth.credentials.iss const labelers = ctx.reqLabelers(req) const hydrateCtx = { labelers, viewer } @@ -31,11 +31,11 @@ export default function (server: Server, ctx: AppContext) { const result = await getListFeed({ ...params, hydrateCtx }, ctx) const repoRev = await ctx.hydrator.actor.getRepoRevSafe(viewer) - setRepoRev(res, repoRev) return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers, repoRev }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getPostThread.ts b/packages/bsky/src/api/app/bsky/feed/getPostThread.ts index d7c4009bdaa..3cfd69f47f8 100644 --- a/packages/bsky/src/api/app/bsky/feed/getPostThread.ts +++ b/packages/bsky/src/api/app/bsky/feed/getPostThread.ts @@ -6,7 +6,7 @@ import { OutputSchema, } from '../../../../lexicon/types/app/bsky/feed/getPostThread' import AppContext from '../../../../context' -import { setRepoRev } from '../../../util' +import { ATPROTO_REPO_REV, resHeaders } from '../../../util' import { HydrationFnInput, PresentationFnInput, @@ -37,16 +37,21 @@ export default function (server: Server, ctx: AppContext) { result = await getPostThread({ ...params, hydrateCtx }, ctx) } catch (err) { const repoRev = await ctx.hydrator.actor.getRepoRevSafe(viewer) - setRepoRev(res, repoRev) + if (repoRev) { + res.setHeader(ATPROTO_REPO_REV, repoRev) + } throw err } const repoRev = await ctx.hydrator.actor.getRepoRevSafe(viewer) - setRepoRev(res, repoRev) return { encoding: 'application/json', body: result, + headers: resHeaders({ + repoRev, + labelers, + }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getPosts.ts b/packages/bsky/src/api/app/bsky/feed/getPosts.ts index fd5b89f96e0..322197c7e3f 100644 --- a/packages/bsky/src/api/app/bsky/feed/getPosts.ts +++ b/packages/bsky/src/api/app/bsky/feed/getPosts.ts @@ -10,6 +10,7 @@ import { } from '../../../../hydration/hydrator' import { Views } from '../../../../views' import { creatorFromUri } from '../../../../views/util' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getPosts = createPipeline(skeleton, hydration, noBlocks, presentation) @@ -25,6 +26,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: results, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getRepostedBy.ts b/packages/bsky/src/api/app/bsky/feed/getRepostedBy.ts index 887200476b7..eee518961b2 100644 --- a/packages/bsky/src/api/app/bsky/feed/getRepostedBy.ts +++ b/packages/bsky/src/api/app/bsky/feed/getRepostedBy.ts @@ -11,7 +11,7 @@ import { import { Views } from '../../../../views' import { parseString } from '../../../../hydration/util' import { creatorFromUri } from '../../../../views/util' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getRepostedBy = createPipeline( @@ -31,6 +31,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getSuggestedFeeds.ts b/packages/bsky/src/api/app/bsky/feed/getSuggestedFeeds.ts index ee42ea74dd3..e848eddd634 100644 --- a/packages/bsky/src/api/app/bsky/feed/getSuggestedFeeds.ts +++ b/packages/bsky/src/api/app/bsky/feed/getSuggestedFeeds.ts @@ -2,6 +2,7 @@ import { mapDefined } from '@atproto/common' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { parseString } from '../../../../hydration/util' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { server.app.bsky.feed.getSuggestedFeeds({ @@ -31,6 +32,7 @@ export default function (server: Server, ctx: AppContext) { feeds: feedViews, cursor: parseString(suggestedRes.cursor), }, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getTimeline.ts b/packages/bsky/src/api/app/bsky/feed/getTimeline.ts index e9e31bcaceb..c58199d195b 100644 --- a/packages/bsky/src/api/app/bsky/feed/getTimeline.ts +++ b/packages/bsky/src/api/app/bsky/feed/getTimeline.ts @@ -1,7 +1,7 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getTimeline' -import { clearlyBadCursor, setRepoRev } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' import { createPipeline } from '../../../../pipeline' import { HydrateCtx, @@ -23,7 +23,7 @@ export default function (server: Server, ctx: AppContext) { ) server.app.bsky.feed.getTimeline({ auth: ctx.authVerifier.standard, - handler: async ({ params, auth, req, res }) => { + handler: async ({ params, auth, req }) => { const viewer = auth.credentials.iss const labelers = ctx.reqLabelers(req) const hydrateCtx = { labelers, viewer } @@ -31,11 +31,11 @@ export default function (server: Server, ctx: AppContext) { const result = await getTimeline({ ...params, hydrateCtx }, ctx) const repoRev = await ctx.hydrator.actor.getRepoRevSafe(viewer) - setRepoRev(res, repoRev) return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers, repoRev }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/searchPosts.ts b/packages/bsky/src/api/app/bsky/feed/searchPosts.ts index c511883fbde..0ab4fb8ba3a 100644 --- a/packages/bsky/src/api/app/bsky/feed/searchPosts.ts +++ b/packages/bsky/src/api/app/bsky/feed/searchPosts.ts @@ -15,6 +15,7 @@ import { Views } from '../../../../views' import { DataPlaneClient } from '../../../../data-plane' import { parseString } from '../../../../hydration/util' import { creatorFromUri } from '../../../../views/util' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const searchPosts = createPipeline( @@ -33,6 +34,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: results, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/graph/getBlocks.ts b/packages/bsky/src/api/app/bsky/graph/getBlocks.ts index e7a9df2030e..b6723cb3c45 100644 --- a/packages/bsky/src/api/app/bsky/graph/getBlocks.ts +++ b/packages/bsky/src/api/app/bsky/graph/getBlocks.ts @@ -11,7 +11,7 @@ import { } from '../../../../pipeline' import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator' import { Views } from '../../../../views' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getBlocks = createPipeline(skeleton, hydration, noRules, presentation) @@ -25,6 +25,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/graph/getFollowers.ts b/packages/bsky/src/api/app/bsky/graph/getFollowers.ts index b2d3288b4bd..42e31f7288f 100644 --- a/packages/bsky/src/api/app/bsky/graph/getFollowers.ts +++ b/packages/bsky/src/api/app/bsky/graph/getFollowers.ts @@ -17,7 +17,7 @@ import { mergeStates, } from '../../../../hydration/hydrator' import { Views } from '../../../../views' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getFollowers = createPipeline( @@ -41,6 +41,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/graph/getFollows.ts b/packages/bsky/src/api/app/bsky/graph/getFollows.ts index d1840f9a19a..cfe8e220cc7 100644 --- a/packages/bsky/src/api/app/bsky/graph/getFollows.ts +++ b/packages/bsky/src/api/app/bsky/graph/getFollows.ts @@ -16,7 +16,7 @@ import { mergeStates, } from '../../../../hydration/hydrator' import { Views } from '../../../../views' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getFollows = createPipeline(skeleton, hydration, noBlocks, presentation) @@ -36,6 +36,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/graph/getList.ts b/packages/bsky/src/api/app/bsky/graph/getList.ts index 42c298ecbe2..92ea332d0ed 100644 --- a/packages/bsky/src/api/app/bsky/graph/getList.ts +++ b/packages/bsky/src/api/app/bsky/graph/getList.ts @@ -16,7 +16,7 @@ import { mergeStates, } from '../../../../hydration/hydrator' import { Views } from '../../../../views' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' import { ListItemInfo } from '../../../../proto/bsky_pb' export default function (server: Server, ctx: AppContext) { @@ -31,6 +31,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/graph/getListBlocks.ts b/packages/bsky/src/api/app/bsky/graph/getListBlocks.ts index 9b7e74f69b6..2360781a6ef 100644 --- a/packages/bsky/src/api/app/bsky/graph/getListBlocks.ts +++ b/packages/bsky/src/api/app/bsky/graph/getListBlocks.ts @@ -11,7 +11,7 @@ import { } from '../../../../pipeline' import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator' import { Views } from '../../../../views' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getListBlocks = createPipeline( @@ -30,6 +30,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/graph/getListMutes.ts b/packages/bsky/src/api/app/bsky/graph/getListMutes.ts index 7004e43249f..cc4f5a64942 100644 --- a/packages/bsky/src/api/app/bsky/graph/getListMutes.ts +++ b/packages/bsky/src/api/app/bsky/graph/getListMutes.ts @@ -11,7 +11,7 @@ import { } from '../../../../pipeline' import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator' import { Views } from '../../../../views' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getListMutes = createPipeline( @@ -30,6 +30,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/graph/getLists.ts b/packages/bsky/src/api/app/bsky/graph/getLists.ts index c53b7e14482..3ebc7fbb628 100644 --- a/packages/bsky/src/api/app/bsky/graph/getLists.ts +++ b/packages/bsky/src/api/app/bsky/graph/getLists.ts @@ -11,7 +11,7 @@ import { } from '../../../../pipeline' import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator' import { Views } from '../../../../views' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getLists = createPipeline(skeleton, hydration, noRules, presentation) @@ -26,6 +26,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/graph/getMutes.ts b/packages/bsky/src/api/app/bsky/graph/getMutes.ts index db81c46ab1b..6890087afa9 100644 --- a/packages/bsky/src/api/app/bsky/graph/getMutes.ts +++ b/packages/bsky/src/api/app/bsky/graph/getMutes.ts @@ -11,7 +11,7 @@ import { createPipeline, noRules, } from '../../../../pipeline' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getMutes = createPipeline(skeleton, hydration, noRules, presentation) @@ -25,6 +25,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/graph/getSuggestedFollowsByActor.ts b/packages/bsky/src/api/app/bsky/graph/getSuggestedFollowsByActor.ts index 579a9ab329b..356b33aa7fe 100644 --- a/packages/bsky/src/api/app/bsky/graph/getSuggestedFollowsByActor.ts +++ b/packages/bsky/src/api/app/bsky/graph/getSuggestedFollowsByActor.ts @@ -12,6 +12,7 @@ import { } from '../../../../pipeline' import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator' import { Views } from '../../../../views' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getSuggestedFollowsByActor = createPipeline( @@ -33,6 +34,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/labeler/getServices.ts b/packages/bsky/src/api/app/bsky/labeler/getServices.ts index 9f151757361..aeca1a10834 100644 --- a/packages/bsky/src/api/app/bsky/labeler/getServices.ts +++ b/packages/bsky/src/api/app/bsky/labeler/getServices.ts @@ -1,6 +1,7 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { mapDefined } from '@atproto/common' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { server.app.bsky.labeler.getServices({ @@ -38,6 +39,7 @@ export default function (server: Server, ctx: AppContext) { body: { views, }, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/notification/listNotifications.ts b/packages/bsky/src/api/app/bsky/notification/listNotifications.ts index 6ee8951fc66..8eb3996b2a2 100644 --- a/packages/bsky/src/api/app/bsky/notification/listNotifications.ts +++ b/packages/bsky/src/api/app/bsky/notification/listNotifications.ts @@ -14,7 +14,7 @@ import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator' import { Views } from '../../../../views' import { Notification } from '../../../../proto/bsky_pb' import { didFromUri } from '../../../../hydration/util' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const listNotifications = createPipeline( @@ -33,6 +33,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts b/packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts index c676e115908..883a451c8ac 100644 --- a/packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts +++ b/packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts @@ -2,7 +2,7 @@ import { mapDefined } from '@atproto/common' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { parseString } from '../../../../hydration/util' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' // THIS IS A TEMPORARY UNSPECCED ROUTE // @TODO currently mirrors getSuggestedFeeds and ignores the "query" param. @@ -53,6 +53,7 @@ export default function (server: Server, ctx: AppContext) { feeds: feedViews, cursor, }, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/util.ts b/packages/bsky/src/api/util.ts index 2fe54a8a7be..3ee0ea2c59b 100644 --- a/packages/bsky/src/api/util.ts +++ b/packages/bsky/src/api/util.ts @@ -1,9 +1,24 @@ -import express from 'express' +import { ParsedLabelers, formatLabelerHeader } from '../util' -export const setRepoRev = (res: express.Response, rev: string | null) => { - if (rev !== null) { - res.setHeader('Atproto-Repo-Rev', rev) +export const ATPROTO_CONTENT_LABELERS = 'Atproto-Content-Labelers' +export const ATPROTO_REPO_REV = 'Atproto-Repo-Rev' + +type ResHeaderOpts = { + labelers: ParsedLabelers + repoRev: string | null +} + +export const resHeaders = ( + opts: Partial, +): Record => { + const headers = {} + if (opts.labelers) { + headers[ATPROTO_CONTENT_LABELERS] = formatLabelerHeader(opts.labelers) + } + if (opts.repoRev) { + headers[ATPROTO_REPO_REV] = opts.repoRev } + return headers } export const clearlyBadCursor = (cursor?: string) => { diff --git a/packages/bsky/src/context.ts b/packages/bsky/src/context.ts index b166368cdf8..e8a15f5197a 100644 --- a/packages/bsky/src/context.ts +++ b/packages/bsky/src/context.ts @@ -9,9 +9,14 @@ import { DataPlaneClient } from './data-plane/client' import { Hydrator } from './hydration/hydrator' import { Views } from './views' import { AuthVerifier } from './auth-verifier' -import { dedupeStrs } from '@atproto/common' import { BsyncClient } from './bsync' import { CourierClient } from './courier' +import { + ParsedLabelers, + defaultLabelerHeader, + parseLabelerHeader, +} from './util' +import { httpLogger as log } from './logger' export class AppContext { constructor( @@ -82,15 +87,17 @@ export class AppContext { }) } - reqLabelers(req: express.Request): string[] { - const val = req.header('atproto-labelers') - if (!val) return this.cfg.labelsFromIssuerDids - return dedupeStrs( - val - .split(',') - .map((did) => did.trim()) - .slice(0, 10), - ) + reqLabelers(req: express.Request): ParsedLabelers { + const val = req.header('atproto-accept-labelers') + let parsed: ParsedLabelers | null + try { + parsed = parseLabelerHeader(val) + } catch (err) { + parsed = null + log.info({ err, val }, 'failed to parse labeler header') + } + if (!parsed) return defaultLabelerHeader(this.cfg.labelsFromIssuerDids) + return parsed } } diff --git a/packages/bsky/src/hydration/hydrator.ts b/packages/bsky/src/hydration/hydrator.ts index 9181032b7da..de64a333ace 100644 --- a/packages/bsky/src/hydration/hydrator.ts +++ b/packages/bsky/src/hydration/hydrator.ts @@ -45,9 +45,10 @@ import { FeedItem, ItemRef, } from './feed' +import { ParsedLabelers } from '../util' export type HydrateCtx = { - labelers: string[] + labelers: ParsedLabelers viewer: string | null } @@ -136,7 +137,10 @@ export class Hydrator { ): Promise { const [actors, labels, profileViewersState] = await Promise.all([ this.actor.getActors(dids, includeTakedowns), - this.label.getLabelsForSubjects(labelSubjectsForDid(dids), ctx.labelers), + this.label.getLabelsForSubjects( + labelSubjectsForDid(dids), + ctx.labelers.dids, + ), this.hydrateProfileViewers(dids, ctx), ]) return mergeStates(profileViewersState ?? {}, { @@ -298,7 +302,7 @@ export class Hydrator { ] = await Promise.all([ this.feed.getPostAggregates(refs), ctx.viewer ? this.feed.getPostViewerStates(refs, ctx.viewer) : undefined, - this.label.getLabelsForSubjects(allPostUris, ctx.labelers), + this.label.getLabelsForSubjects(allPostUris, ctx.labelers.dids), this.hydratePostBlocks(posts), this.hydrateProfiles(allPostUris.map(didFromUri), ctx, includeTakedowns), this.hydrateLists([...nestedListUris, ...gateListUris], ctx), @@ -494,7 +498,7 @@ export class Hydrator { this.feed.getLikes(likeUris), // reason: like this.feed.getReposts(repostUris), // reason: repost this.graph.getFollows(followUris), // reason: follow - this.label.getLabelsForSubjects(uris, ctx.labelers), + this.label.getLabelsForSubjects(uris, ctx.labelers.dids), this.hydrateProfiles(uris.map(didFromUri), ctx), ]) return mergeStates(profileState, { diff --git a/packages/bsky/src/util.ts b/packages/bsky/src/util.ts new file mode 100644 index 00000000000..50c0c1fa565 --- /dev/null +++ b/packages/bsky/src/util.ts @@ -0,0 +1,44 @@ +import { parseList } from 'structured-headers' + +export type ParsedLabelers = { + dids: string[] + redact: Set +} + +export const parseLabelerHeader = ( + header: string | undefined, +): ParsedLabelers | null => { + if (!header) return null + const labelerDids = new Set() + const redactDids = new Set() + const parsed = parseList(header) + for (const item of parsed) { + const did = item[0].toString() + if (!did) { + return null + } + labelerDids.add(did) + const redact = item[1].get('redact')?.valueOf() + if (redact === true) { + redactDids.add(did) + } + } + return { + dids: [...labelerDids], + redact: redactDids, + } +} + +export const defaultLabelerHeader = (dids: string[]): ParsedLabelers => { + return { + dids, + redact: new Set(dids), + } +} + +export const formatLabelerHeader = (parsed: ParsedLabelers): string => { + const parts = parsed.dids.map((did) => + parsed.redact.has(did) ? `${did};redact` : did, + ) + return parts.join(',') +} diff --git a/packages/bsky/tests/label-hydration.test.ts b/packages/bsky/tests/label-hydration.test.ts index a7fe678581c..b6927dd8afe 100644 --- a/packages/bsky/tests/label-hydration.test.ts +++ b/packages/bsky/tests/label-hydration.test.ts @@ -40,11 +40,17 @@ describe('label hydration', () => { it('hydrates labels based on a supplied labeler header', async () => { const res = await pdsAgent.api.app.bsky.actor.getProfile( { actor: carol }, - { headers: { ...sc.getHeaders(bob), 'atproto-labelers': alice } }, + { + headers: { + ...sc.getHeaders(bob), + 'atproto-accept-labelers': `${alice};redact`, + }, + }, ) expect(res.data.labels?.length).toBe(1) expect(res.data.labels?.[0].src).toBe(alice) expect(res.data.labels?.[0].val).toBe('spam') + expect(res.headers['atproto-content-labelers']).toEqual(`${alice};redact`) }) it('hydrates labels based on multiple a supplied labelers', async () => { @@ -53,7 +59,7 @@ describe('label hydration', () => { { headers: { ...sc.getHeaders(bob), - 'atproto-labelers': `${alice},${bob}, ${labelerDid}`, + 'atproto-accept-labelers': `${alice},${bob};redact, ${labelerDid}`, }, }, ) @@ -65,6 +71,10 @@ describe('label hydration', () => { expect(res.data.labels?.find((l) => l.src === labelerDid)?.val).toEqual( 'misleading', ) + const labelerHeaderDids = res.headers['atproto-content-labelers'].split(',') + expect(labelerHeaderDids.sort()).toEqual( + [alice, `${bob};redact`, labelerDid].sort(), + ) }) it('defaults to service labels when no labeler header is provided', async () => { @@ -75,6 +85,9 @@ describe('label hydration', () => { expect(res.data.labels?.length).toBe(1) expect(res.data.labels?.[0].src).toBe(labelerDid) expect(res.data.labels?.[0].val).toBe('misleading') + expect(res.headers['atproto-content-labelers']).toEqual( + `${labelerDid};redact`, + ) }) it('hydrates labels onto list views.', async () => { diff --git a/packages/pds/src/pipethrough.ts b/packages/pds/src/pipethrough.ts index 0d9c00737b5..8f633a7448f 100644 --- a/packages/pds/src/pipethrough.ts +++ b/packages/pds/src/pipethrough.ts @@ -95,10 +95,10 @@ export const parseProxyHeader = async ( return { did, serviceUrl } } -const HEADERS_TO_FORWARD = [ +const REQ_HEADERS_TO_FORWARD = [ 'accept-language', 'content-type', - 'atproto-labelers', + 'atproto-accept-labelers', ] export const createUrlAndHeaders = async ( @@ -122,7 +122,7 @@ export const createUrlAndHeaders = async ( ? (await ctx.serviceAuthHeaders(requester, aud)).headers : {} // forward select headers to upstream services - for (const header of HEADERS_TO_FORWARD) { + for (const header of REQ_HEADERS_TO_FORWARD) { const val = req.headers[header] if (val) { headers[header] = val @@ -152,15 +152,24 @@ export const doProxy = async (url: URL, reqInit: RequestInit) => { ) } const encoding = res.headers.get('content-type') ?? 'application/json' - const repoRevHeader = res.headers.get('atproto-repo-rev') - const contentLanguage = res.headers.get('content-language') - const resHeaders = noUndefinedVals({ - ['atproto-repo-rev']: repoRevHeader ?? undefined, - ['content-language']: contentLanguage ?? undefined, - }) + const resHeaders = makeResHeaders(res) return { encoding, buffer, headers: resHeaders } } +const RES_HEADERS_TO_FORWARD = [ + 'atproto-repo-rev', + 'content-language', + 'atproto-content-labelers', +] + +const makeResHeaders = (res: Response): Record => { + const headers = RES_HEADERS_TO_FORWARD.reduce((acc, cur) => { + acc[cur] = res.headers.get(cur) ?? undefined + return acc + }, {} as Record) + return noUndefinedVals(headers) +} + const isSafeUrl = (url: URL) => { if (url.protocol !== 'https:') return false if (!url.hostname || url.hostname === 'localhost') return false diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1f673eeaa28..e8c64766285 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -254,6 +254,9 @@ importers: sharp: specifier: ^0.32.6 version: 0.32.6 + structured-headers: + specifier: ^1.0.1 + version: 1.0.1 typed-emitter: specifier: ^2.1.0 version: 2.1.0 @@ -11317,6 +11320,11 @@ packages: '@tokenizer/token': 0.3.0 peek-readable: 4.1.0 + /structured-headers@1.0.1: + resolution: {integrity: sha512-QYBxdBtA4Tl5rFPuqmbmdrS9kbtren74RTJTcs0VSQNVV5iRhJD4QlYTLD0+81SBwUQctjEQzjTRI3WG4DzICA==} + engines: {node: '>= 14', npm: '>=6'} + dev: false + /supports-color@5.5.0: resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} engines: {node: '>=4'}