From 8c62af0410eb8d6006dd057dd57fab59dc1e1d47 Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Sun, 25 Feb 2024 18:22:14 +0100 Subject: [PATCH 01/20] :construction: Working through an nullable review state --- lexicons/com/atproto/admin/defs.json | 11 +++- packages/api/src/client/index.ts | 1 + packages/api/src/client/lexicons.ts | 6 ++ .../client/types/com/atproto/admin/defs.ts | 3 + packages/bsky/src/lexicon/index.ts | 1 + packages/bsky/src/lexicon/lexicons.ts | 6 ++ .../lexicon/types/com/atproto/admin/defs.ts | 3 + .../db/schema/moderation_subject_status.ts | 7 ++- packages/ozone/src/lexicon/index.ts | 1 + packages/ozone/src/lexicon/lexicons.ts | 6 ++ .../lexicon/types/com/atproto/admin/defs.ts | 3 + packages/ozone/src/mod-service/status.ts | 60 +++++++++++-------- .../ozone/tests/moderation-events.test.ts | 2 + packages/pds/src/lexicon/index.ts | 1 + packages/pds/src/lexicon/lexicons.ts | 6 ++ .../lexicon/types/com/atproto/admin/defs.ts | 3 + 16 files changed, 92 insertions(+), 28 deletions(-) diff --git a/lexicons/com/atproto/admin/defs.json b/lexicons/com/atproto/admin/defs.json index e1315eb7473..56a855f7427 100644 --- a/lexicons/com/atproto/admin/defs.json +++ b/lexicons/com/atproto/admin/defs.json @@ -449,7 +449,12 @@ }, "subjectReviewState": { "type": "string", - "knownValues": ["#reviewOpen", "#reviewEscalated", "#reviewClosed"] + "knownValues": [ + "#reviewOpen", + "#reviewEscalated", + "#reviewClosed", + "#reviewOptional" + ] }, "reviewOpen": { "type": "token", @@ -463,6 +468,10 @@ "type": "token", "description": "Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator" }, + "reviewOptional": { + "type": "token", + "description": "Moderator review status of a subject: Unnecessary. Indicates that the subject does not need a review at the moment but there is probably some moderation related metadata available for it" + }, "modEventTakedown": { "type": "object", "description": "Take down a subject permanently or temporarily", diff --git a/packages/api/src/client/index.ts b/packages/api/src/client/index.ts index 846c379b7a8..7f55f73390c 100644 --- a/packages/api/src/client/index.ts +++ b/packages/api/src/client/index.ts @@ -319,6 +319,7 @@ export const COM_ATPROTO_ADMIN = { DefsReviewOpen: 'com.atproto.admin.defs#reviewOpen', DefsReviewEscalated: 'com.atproto.admin.defs#reviewEscalated', DefsReviewClosed: 'com.atproto.admin.defs#reviewClosed', + DefsReviewOptional: 'com.atproto.admin.defs#reviewOptional', } export const COM_ATPROTO_MODERATION = { DefsReasonSpam: 'com.atproto.moderation.defs#reasonSpam', diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index d546b70b987..b95afb50e2a 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -747,6 +747,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#reviewOpen', 'lex:com.atproto.admin.defs#reviewEscalated', 'lex:com.atproto.admin.defs#reviewClosed', + 'lex:com.atproto.admin.defs#reviewOptional', ], }, reviewOpen: { @@ -764,6 +765,11 @@ export const schemaDict = { description: 'Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator', }, + reviewOptional: { + type: 'token', + description: + 'Moderator review status of a subject: Unnecessary. Indicates that the subject does not need a review at the moment but there is probably some moderation related metadata available for it', + }, modEventTakedown: { type: 'object', description: 'Take down a subject permanently or temporarily', diff --git a/packages/api/src/client/types/com/atproto/admin/defs.ts b/packages/api/src/client/types/com/atproto/admin/defs.ts index 4e3d35a869f..b3dbb45bc23 100644 --- a/packages/api/src/client/types/com/atproto/admin/defs.ts +++ b/packages/api/src/client/types/com/atproto/admin/defs.ts @@ -497,6 +497,7 @@ export type SubjectReviewState = | 'lex:com.atproto.admin.defs#reviewOpen' | 'lex:com.atproto.admin.defs#reviewEscalated' | 'lex:com.atproto.admin.defs#reviewClosed' + | 'lex:com.atproto.admin.defs#reviewOptional' | (string & {}) /** Moderator review status of a subject: Open. Indicates that the subject needs to be reviewed by a moderator */ @@ -505,6 +506,8 @@ export const REVIEWOPEN = 'com.atproto.admin.defs#reviewOpen' export const REVIEWESCALATED = 'com.atproto.admin.defs#reviewEscalated' /** Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator */ export const REVIEWCLOSED = 'com.atproto.admin.defs#reviewClosed' +/** Moderator review status of a subject: Unnecessary. Indicates that the subject does not need a review at the moment but there is probably some moderation related metadata available for it */ +export const REVIEWOPTIONAL = 'com.atproto.admin.defs#reviewOptional' /** Take down a subject permanently or temporarily */ export interface ModEventTakedown { diff --git a/packages/bsky/src/lexicon/index.ts b/packages/bsky/src/lexicon/index.ts index cf2c613e686..d42435e2f7c 100644 --- a/packages/bsky/src/lexicon/index.ts +++ b/packages/bsky/src/lexicon/index.ts @@ -142,6 +142,7 @@ export const COM_ATPROTO_ADMIN = { DefsReviewOpen: 'com.atproto.admin.defs#reviewOpen', DefsReviewEscalated: 'com.atproto.admin.defs#reviewEscalated', DefsReviewClosed: 'com.atproto.admin.defs#reviewClosed', + DefsReviewOptional: 'com.atproto.admin.defs#reviewOptional', } export const COM_ATPROTO_MODERATION = { DefsReasonSpam: 'com.atproto.moderation.defs#reasonSpam', diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index d546b70b987..b95afb50e2a 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -747,6 +747,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#reviewOpen', 'lex:com.atproto.admin.defs#reviewEscalated', 'lex:com.atproto.admin.defs#reviewClosed', + 'lex:com.atproto.admin.defs#reviewOptional', ], }, reviewOpen: { @@ -764,6 +765,11 @@ export const schemaDict = { description: 'Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator', }, + reviewOptional: { + type: 'token', + description: + 'Moderator review status of a subject: Unnecessary. Indicates that the subject does not need a review at the moment but there is probably some moderation related metadata available for it', + }, modEventTakedown: { type: 'object', description: 'Take down a subject permanently or temporarily', diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts index a713a635635..9e6aa7e3269 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts @@ -497,6 +497,7 @@ export type SubjectReviewState = | 'lex:com.atproto.admin.defs#reviewOpen' | 'lex:com.atproto.admin.defs#reviewEscalated' | 'lex:com.atproto.admin.defs#reviewClosed' + | 'lex:com.atproto.admin.defs#reviewOptional' | (string & {}) /** Moderator review status of a subject: Open. Indicates that the subject needs to be reviewed by a moderator */ @@ -505,6 +506,8 @@ export const REVIEWOPEN = 'com.atproto.admin.defs#reviewOpen' export const REVIEWESCALATED = 'com.atproto.admin.defs#reviewEscalated' /** Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator */ export const REVIEWCLOSED = 'com.atproto.admin.defs#reviewClosed' +/** Moderator review status of a subject: Unnecessary. Indicates that the subject does not need a review at the moment but there is probably some moderation related metadata available for it */ +export const REVIEWOPTIONAL = 'com.atproto.admin.defs#reviewOptional' /** Take down a subject permanently or temporarily */ export interface ModEventTakedown { diff --git a/packages/ozone/src/db/schema/moderation_subject_status.ts b/packages/ozone/src/db/schema/moderation_subject_status.ts index 59803133bcb..4ed2751f217 100644 --- a/packages/ozone/src/db/schema/moderation_subject_status.ts +++ b/packages/ozone/src/db/schema/moderation_subject_status.ts @@ -3,6 +3,7 @@ import { REVIEWCLOSED, REVIEWOPEN, REVIEWESCALATED, + REVIEWOPTIONAL, } from '../../lexicon/types/com/atproto/admin/defs' export const subjectStatusTableName = 'moderation_subject_status' @@ -13,7 +14,11 @@ export interface ModerationSubjectStatus { recordPath: string recordCid: string | null blobCids: string[] | null - reviewState: typeof REVIEWCLOSED | typeof REVIEWOPEN | typeof REVIEWESCALATED + reviewState: + | typeof REVIEWCLOSED + | typeof REVIEWOPEN + | typeof REVIEWESCALATED + | typeof REVIEWOPTIONAL createdAt: string updatedAt: string lastReviewedBy: string | null diff --git a/packages/ozone/src/lexicon/index.ts b/packages/ozone/src/lexicon/index.ts index cf2c613e686..d42435e2f7c 100644 --- a/packages/ozone/src/lexicon/index.ts +++ b/packages/ozone/src/lexicon/index.ts @@ -142,6 +142,7 @@ export const COM_ATPROTO_ADMIN = { DefsReviewOpen: 'com.atproto.admin.defs#reviewOpen', DefsReviewEscalated: 'com.atproto.admin.defs#reviewEscalated', DefsReviewClosed: 'com.atproto.admin.defs#reviewClosed', + DefsReviewOptional: 'com.atproto.admin.defs#reviewOptional', } export const COM_ATPROTO_MODERATION = { DefsReasonSpam: 'com.atproto.moderation.defs#reasonSpam', diff --git a/packages/ozone/src/lexicon/lexicons.ts b/packages/ozone/src/lexicon/lexicons.ts index d546b70b987..b95afb50e2a 100644 --- a/packages/ozone/src/lexicon/lexicons.ts +++ b/packages/ozone/src/lexicon/lexicons.ts @@ -747,6 +747,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#reviewOpen', 'lex:com.atproto.admin.defs#reviewEscalated', 'lex:com.atproto.admin.defs#reviewClosed', + 'lex:com.atproto.admin.defs#reviewOptional', ], }, reviewOpen: { @@ -764,6 +765,11 @@ export const schemaDict = { description: 'Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator', }, + reviewOptional: { + type: 'token', + description: + 'Moderator review status of a subject: Unnecessary. Indicates that the subject does not need a review at the moment but there is probably some moderation related metadata available for it', + }, modEventTakedown: { type: 'object', description: 'Take down a subject permanently or temporarily', diff --git a/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts b/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts index a713a635635..9e6aa7e3269 100644 --- a/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts @@ -497,6 +497,7 @@ export type SubjectReviewState = | 'lex:com.atproto.admin.defs#reviewOpen' | 'lex:com.atproto.admin.defs#reviewEscalated' | 'lex:com.atproto.admin.defs#reviewClosed' + | 'lex:com.atproto.admin.defs#reviewOptional' | (string & {}) /** Moderator review status of a subject: Open. Indicates that the subject needs to be reviewed by a moderator */ @@ -505,6 +506,8 @@ export const REVIEWOPEN = 'com.atproto.admin.defs#reviewOpen' export const REVIEWESCALATED = 'com.atproto.admin.defs#reviewEscalated' /** Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator */ export const REVIEWCLOSED = 'com.atproto.admin.defs#reviewClosed' +/** Moderator review status of a subject: Unnecessary. Indicates that the subject does not need a review at the moment but there is probably some moderation related metadata available for it */ +export const REVIEWOPTIONAL = 'com.atproto.admin.defs#reviewOptional' /** Take down a subject permanently or temporarily */ export interface ModEventTakedown { diff --git a/packages/ozone/src/mod-service/status.ts b/packages/ozone/src/mod-service/status.ts index 2edba64282f..efbdbe66290 100644 --- a/packages/ozone/src/mod-service/status.ts +++ b/packages/ozone/src/mod-service/status.ts @@ -7,6 +7,7 @@ import { REVIEWOPEN, REVIEWCLOSED, REVIEWESCALATED, + REVIEWOPTIONAL, } from '../lexicon/types/com/atproto/admin/defs' import { ModerationEventRow, ModerationSubjectStatusRow } from './types' import { HOUR } from '@atproto/common' @@ -14,16 +15,22 @@ import { REASONAPPEAL } from '../lexicon/types/com/atproto/moderation/defs' import { jsonb } from '../db/types' const getSubjectStatusForModerationEvent = ({ + currentStatus, action, createdBy, createdAt, durationInHours, }: { + currentStatus?: ModerationSubjectStatusRow action: string createdBy: string createdAt: string durationInHours: number | null -}): Partial | null => { +}): Partial => { + const defaultReviewState = currentStatus + ? currentStatus.reviewState + : REVIEWOPTIONAL + switch (action) { case 'com.atproto.admin.defs#modEventAcknowledge': return { @@ -54,7 +61,9 @@ const getSubjectStatusForModerationEvent = ({ return { lastReviewedBy: createdBy, muteUntil: null, - reviewState: REVIEWOPEN, + // It's not likely to receive an unmute event that does not already have a status row + // but if it does happen, default to unnecessary + reviewState: defaultReviewState, lastReviewedAt: createdAt, } case 'com.atproto.admin.defs#modEventTakedown': @@ -70,26 +79,29 @@ const getSubjectStatusForModerationEvent = ({ case 'com.atproto.admin.defs#modEventMute': return { lastReviewedBy: createdBy, - reviewState: REVIEWOPEN, lastReviewedAt: createdAt, // By default, mute for 24hrs muteUntil: new Date( Date.now() + (durationInHours || 24) * HOUR, ).toISOString(), + // It's not likely to receive a mute event on a subject that does not already have a status row + // but if it does happen, default to unnecessary + reviewState: defaultReviewState, } case 'com.atproto.admin.defs#modEventComment': return { lastReviewedBy: createdBy, lastReviewedAt: createdAt, + reviewState: defaultReviewState, } case 'com.atproto.admin.defs#modEventTag': - return { tags: [] } + return { tags: [], reviewState: defaultReviewState } case 'com.atproto.admin.defs#modEventResolveAppeal': return { appealed: false, } default: - return null + return {} } } @@ -114,23 +126,6 @@ export const adjustModerationSubjectStatus = async ( createdAt, } = moderationEvent - const isAppealEvent = - action === 'com.atproto.admin.defs#modEventReport' && - meta?.reportType === REASONAPPEAL - - const subjectStatus = getSubjectStatusForModerationEvent({ - action, - createdBy, - createdAt, - durationInHours: moderationEvent.durationInHours, - }) - - // If there are no subjectStatus that means there are no side-effect of the incoming event - if (!subjectStatus) { - return null - } - - const now = new Date().toISOString() // If subjectUri exists, it's not a repoRef so pass along the uri to get identifier back const identifier = getStatusIdentifierFromSubject(subjectUri || subjectDid) @@ -143,12 +138,25 @@ export const adjustModerationSubjectStatus = async ( .selectAll() .executeTakeFirst() + const isAppealEvent = + action === 'com.atproto.admin.defs#modEventReport' && + meta?.reportType === REASONAPPEAL + + const subjectStatus = getSubjectStatusForModerationEvent({ + currentStatus, + action, + createdBy, + createdAt, + durationInHours: moderationEvent.durationInHours, + }) + + const now = new Date().toISOString() if ( currentStatus?.reviewState === REVIEWESCALATED && - subjectStatus.reviewState === REVIEWOPEN + subjectStatus.reviewState !== REVIEWCLOSED ) { - // If the current status is escalated and the incoming event is to open the review - // We want to keep the status as escalated + // If the current status is escalated only allow incoming events to move the state to + // reviewClosed because escalated subjects should never move to any other state subjectStatus.reviewState = REVIEWESCALATED } @@ -158,7 +166,7 @@ export const adjustModerationSubjectStatus = async ( // Defaulting reviewState to open for any event may not be the desired behavior. // For instance, if a subject never had any event and we just want to leave a comment to keep an eye on it // that shouldn't mean we want to review the subject - reviewState: REVIEWOPEN, + reviewState: REVIEWOPTIONAL, recordCid: subjectCid || null, } const newStatus = { diff --git a/packages/ozone/tests/moderation-events.test.ts b/packages/ozone/tests/moderation-events.test.ts index 12277ea77a4..305bdd11c9e 100644 --- a/packages/ozone/tests/moderation-events.test.ts +++ b/packages/ozone/tests/moderation-events.test.ts @@ -40,6 +40,7 @@ describe('moderation-events', () => { $type: 'com.atproto.admin.defs#repoRef', did: sc.dids.alice, } + console.log({ alicesAccount }) const bobsPost = { $type: 'com.atproto.repo.strongRef', uri: sc.posts[sc.dids.bob][0].ref.uriStr, @@ -358,6 +359,7 @@ describe('moderation-events', () => { { id: 1 }, { headers: network.bsky.adminAuthHeaders('moderator') }, ) + console.log(JSON.stringify(data, null, 2)) expect(forSnapshot(data)).toMatchSnapshot() }) }) diff --git a/packages/pds/src/lexicon/index.ts b/packages/pds/src/lexicon/index.ts index cf2c613e686..d42435e2f7c 100644 --- a/packages/pds/src/lexicon/index.ts +++ b/packages/pds/src/lexicon/index.ts @@ -142,6 +142,7 @@ export const COM_ATPROTO_ADMIN = { DefsReviewOpen: 'com.atproto.admin.defs#reviewOpen', DefsReviewEscalated: 'com.atproto.admin.defs#reviewEscalated', DefsReviewClosed: 'com.atproto.admin.defs#reviewClosed', + DefsReviewOptional: 'com.atproto.admin.defs#reviewOptional', } export const COM_ATPROTO_MODERATION = { DefsReasonSpam: 'com.atproto.moderation.defs#reasonSpam', diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index d546b70b987..b95afb50e2a 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -747,6 +747,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#reviewOpen', 'lex:com.atproto.admin.defs#reviewEscalated', 'lex:com.atproto.admin.defs#reviewClosed', + 'lex:com.atproto.admin.defs#reviewOptional', ], }, reviewOpen: { @@ -764,6 +765,11 @@ export const schemaDict = { description: 'Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator', }, + reviewOptional: { + type: 'token', + description: + 'Moderator review status of a subject: Unnecessary. Indicates that the subject does not need a review at the moment but there is probably some moderation related metadata available for it', + }, modEventTakedown: { type: 'object', description: 'Take down a subject permanently or temporarily', diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts index a713a635635..9e6aa7e3269 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts @@ -497,6 +497,7 @@ export type SubjectReviewState = | 'lex:com.atproto.admin.defs#reviewOpen' | 'lex:com.atproto.admin.defs#reviewEscalated' | 'lex:com.atproto.admin.defs#reviewClosed' + | 'lex:com.atproto.admin.defs#reviewOptional' | (string & {}) /** Moderator review status of a subject: Open. Indicates that the subject needs to be reviewed by a moderator */ @@ -505,6 +506,8 @@ export const REVIEWOPEN = 'com.atproto.admin.defs#reviewOpen' export const REVIEWESCALATED = 'com.atproto.admin.defs#reviewEscalated' /** Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator */ export const REVIEWCLOSED = 'com.atproto.admin.defs#reviewClosed' +/** Moderator review status of a subject: Unnecessary. Indicates that the subject does not need a review at the moment but there is probably some moderation related metadata available for it */ +export const REVIEWOPTIONAL = 'com.atproto.admin.defs#reviewOptional' /** Take down a subject permanently or temporarily */ export interface ModEventTakedown { From ccc8ab9045f85c2613e8d1cfd0b441ecfde6dff7 Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Tue, 27 Feb 2024 20:10:13 +0100 Subject: [PATCH 02/20] :white_check_mark: Update snapshots on some tests --- .../__snapshots__/get-record.test.ts.snap | 4 +-- .../tests/__snapshots__/get-repo.test.ts.snap | 2 +- .../moderation-events.test.ts.snap | 34 ++++++++++++++----- .../moderation-statuses.test.ts.snap | 29 ++++++++++++---- .../__snapshots__/moderation.test.ts.snap | 8 ++--- .../ozone/tests/moderation-events.test.ts | 4 +-- .../ozone/tests/moderation-statuses.test.ts | 2 +- 7 files changed, 58 insertions(+), 25 deletions(-) diff --git a/packages/ozone/tests/__snapshots__/get-record.test.ts.snap b/packages/ozone/tests/__snapshots__/get-record.test.ts.snap index decfb8f4ba4..5c977ea406b 100644 --- a/packages/ozone/tests/__snapshots__/get-record.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/get-record.test.ts.snap @@ -27,7 +27,7 @@ Object { "moderation": Object { "subjectStatus": Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, + "id": 3, "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "did:example:admin", @@ -124,7 +124,7 @@ Object { "moderation": Object { "subjectStatus": Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, + "id": 3, "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "did:example:admin", diff --git a/packages/ozone/tests/__snapshots__/get-repo.test.ts.snap b/packages/ozone/tests/__snapshots__/get-repo.test.ts.snap index 67404b88362..36a2e602c5b 100644 --- a/packages/ozone/tests/__snapshots__/get-repo.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/get-repo.test.ts.snap @@ -20,7 +20,7 @@ Object { "moderation": Object { "subjectStatus": Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, + "id": 5, "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "did:example:admin", diff --git a/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap b/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap index ac48d862f58..a8b7f46d7c1 100644 --- a/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap @@ -18,7 +18,25 @@ Object { "blobCids": Array [], "cid": "cids(0)", "indexedAt": "1970-01-01T00:00:00.000Z", - "moderation": Object {}, + "moderation": Object { + "subjectStatus": Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "id": 1, + "reviewState": "com.atproto.admin.defs#reviewOptional", + "subject": Object { + "$type": "com.atproto.repo.strongRef", + "cid": "cids(0)", + "uri": "record(0)", + }, + "subjectBlobCids": Array [], + "subjectRepoHandle": "alice.test", + "tags": Array [ + "lang:und", + ], + "takendown": false, + "updatedAt": "1970-01-01T00:00:00.000Z", + }, + }, "repo": Object { "did": "user(0)", "handle": "alice.test", @@ -26,7 +44,7 @@ Object { "moderation": Object { "subjectStatus": Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, + "id": 5, "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "user(1)", @@ -101,7 +119,7 @@ Array [ "comment": "X", "reportType": "com.atproto.moderation.defs#reasonSpam", }, - "id": 13, + "id": 15, "subject": Object { "$type": "com.atproto.admin.defs#repoRef", "did": "user(0)", @@ -120,7 +138,7 @@ Array [ ], "remove": Array [], }, - "id": 8, + "id": 10, "subject": Object { "$type": "com.atproto.admin.defs#repoRef", "did": "user(0)", @@ -137,7 +155,7 @@ Array [ "comment": "X", "reportType": "com.atproto.moderation.defs#reasonSpam", }, - "id": 7, + "id": 9, "subject": Object { "$type": "com.atproto.admin.defs#repoRef", "did": "user(0)", @@ -159,7 +177,7 @@ Array [ "comment": "X", "reportType": "com.atproto.moderation.defs#reasonSpam", }, - "id": 12, + "id": 14, "subject": Object { "$type": "com.atproto.repo.strongRef", "cid": "cids(0)", @@ -178,7 +196,7 @@ Array [ ], "remove": Array [], }, - "id": 6, + "id": 8, "subject": Object { "$type": "com.atproto.repo.strongRef", "cid": "cids(0)", @@ -196,7 +214,7 @@ Array [ "comment": "X", "reportType": "com.atproto.moderation.defs#reasonSpam", }, - "id": 5, + "id": 7, "subject": Object { "$type": "com.atproto.repo.strongRef", "cid": "cids(0)", diff --git a/packages/ozone/tests/__snapshots__/moderation-statuses.test.ts.snap b/packages/ozone/tests/__snapshots__/moderation-statuses.test.ts.snap index cf361739a6c..b2e0cabee16 100644 --- a/packages/ozone/tests/__snapshots__/moderation-statuses.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/moderation-statuses.test.ts.snap @@ -4,7 +4,7 @@ exports[`moderation-statuses query statuses returns statuses filtered by subject Array [ Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 7, + "id": 9, "lastReportedAt": "1970-01-01T00:00:00.000Z", "reviewState": "com.atproto.admin.defs#reviewOpen", "subject": Object { @@ -23,7 +23,7 @@ Array [ }, Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 5, + "id": 7, "lastReportedAt": "1970-01-01T00:00:00.000Z", "reviewState": "com.atproto.admin.defs#reviewOpen", "subject": Object { @@ -46,7 +46,7 @@ exports[`moderation-statuses query statuses returns statuses for subjects that r Array [ Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 7, + "id": 9, "lastReportedAt": "1970-01-01T00:00:00.000Z", "reviewState": "com.atproto.admin.defs#reviewOpen", "subject": Object { @@ -65,7 +65,7 @@ Array [ }, Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 5, + "id": 7, "lastReportedAt": "1970-01-01T00:00:00.000Z", "reviewState": "com.atproto.admin.defs#reviewOpen", "subject": Object { @@ -83,7 +83,7 @@ Array [ }, Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 3, + "id": 5, "lastReportedAt": "1970-01-01T00:00:00.000Z", "reviewState": "com.atproto.admin.defs#reviewOpen", "subject": Object { @@ -101,7 +101,7 @@ Array [ }, Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, + "id": 3, "lastReportedAt": "1970-01-01T00:00:00.000Z", "reviewState": "com.atproto.admin.defs#reviewOpen", "subject": Object { @@ -116,5 +116,22 @@ Array [ "takendown": false, "updatedAt": "1970-01-01T00:00:00.000Z", }, + Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "id": 1, + "reviewState": "com.atproto.admin.defs#reviewOptional", + "subject": Object { + "$type": "com.atproto.repo.strongRef", + "cid": "cids(2)", + "uri": "record(2)", + }, + "subjectBlobCids": Array [], + "subjectRepoHandle": "bob.test", + "tags": Array [ + "lang:und", + ], + "takendown": false, + "updatedAt": "1970-01-01T00:00:00.000Z", + }, ] `; diff --git a/packages/ozone/tests/__snapshots__/moderation.test.ts.snap b/packages/ozone/tests/__snapshots__/moderation.test.ts.snap index 1cd4c192081..51bc3f4b0e5 100644 --- a/packages/ozone/tests/__snapshots__/moderation.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/moderation.test.ts.snap @@ -4,7 +4,7 @@ exports[`moderation reporting creates reports of a record. 1`] = ` Array [ Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 7, + "id": 9, "reasonType": "com.atproto.moderation.defs#reasonSpam", "reportedBy": "user(0)", "subject": Object { @@ -15,7 +15,7 @@ Array [ }, Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 9, + "id": 11, "reason": "defamation", "reasonType": "com.atproto.moderation.defs#reasonOther", "reportedBy": "user(1)", @@ -32,7 +32,7 @@ exports[`moderation reporting creates reports of a repo. 1`] = ` Array [ Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 3, + "id": 5, "reasonType": "com.atproto.moderation.defs#reasonSpam", "reportedBy": "user(0)", "subject": Object { @@ -42,7 +42,7 @@ Array [ }, Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 5, + "id": 7, "reason": "impersonation", "reasonType": "com.atproto.moderation.defs#reasonOther", "reportedBy": "user(2)", diff --git a/packages/ozone/tests/moderation-events.test.ts b/packages/ozone/tests/moderation-events.test.ts index 305bdd11c9e..767ca3f65b7 100644 --- a/packages/ozone/tests/moderation-events.test.ts +++ b/packages/ozone/tests/moderation-events.test.ts @@ -40,7 +40,6 @@ describe('moderation-events', () => { $type: 'com.atproto.admin.defs#repoRef', did: sc.dids.alice, } - console.log({ alicesAccount }) const bobsPost = { $type: 'com.atproto.repo.strongRef', uri: sc.posts[sc.dids.bob][0].ref.uriStr, @@ -204,7 +203,7 @@ describe('moderation-events', () => { const defaultEvents = await getPaginatedEvents() const reversedEvents = await getPaginatedEvents('asc') - expect(allEvents.data.events.length).toEqual(7) + expect(allEvents.data.events.length).toEqual(8) expect(defaultEvents.length).toEqual(allEvents.data.events.length) expect(reversedEvents.length).toEqual(allEvents.data.events.length) // First event in the reversed list is the last item in the default list @@ -359,7 +358,6 @@ describe('moderation-events', () => { { id: 1 }, { headers: network.bsky.adminAuthHeaders('moderator') }, ) - console.log(JSON.stringify(data, null, 2)) expect(forSnapshot(data)).toMatchSnapshot() }) }) diff --git a/packages/ozone/tests/moderation-statuses.test.ts b/packages/ozone/tests/moderation-statuses.test.ts index 14184454e62..38c69a5056a 100644 --- a/packages/ozone/tests/moderation-statuses.test.ts +++ b/packages/ozone/tests/moderation-statuses.test.ts @@ -136,7 +136,7 @@ describe('moderation-statuses', () => { } const list = await getPaginatedStatuses({}) - expect(list[0].id).toEqual(7) + expect(list[0].id).toEqual(11) expect(list[list.length - 1].id).toEqual(1) await emitModerationEvent({ From d7f6538003c8a55df39cd52645e70032af8497a6 Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Tue, 27 Feb 2024 20:22:18 +0100 Subject: [PATCH 03/20] :white_check_mark: Update snapshots on some tests --- .../__snapshots__/get-record.test.ts.snap | 4 +-- .../moderation-events.test.ts.snap | 2 +- .../moderation-statuses.test.ts.snap | 31 ++++++++++++++----- .../ozone/tests/moderation-events.test.ts | 4 ++- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/packages/ozone/tests/__snapshots__/get-record.test.ts.snap b/packages/ozone/tests/__snapshots__/get-record.test.ts.snap index 5c977ea406b..65e1a21bfd7 100644 --- a/packages/ozone/tests/__snapshots__/get-record.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/get-record.test.ts.snap @@ -27,7 +27,7 @@ Object { "moderation": Object { "subjectStatus": Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 3, + "id": 5, "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "did:example:admin", @@ -124,7 +124,7 @@ Object { "moderation": Object { "subjectStatus": Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 3, + "id": 5, "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "did:example:admin", diff --git a/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap b/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap index a8b7f46d7c1..1615b746242 100644 --- a/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap @@ -48,7 +48,7 @@ Object { "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "user(1)", - "reviewState": "com.atproto.admin.defs#reviewEscalated", + "reviewState": "com.atproto.admin.defs#reviewOpen", "subject": Object { "$type": "com.atproto.admin.defs#repoRef", "did": "user(0)", diff --git a/packages/ozone/tests/__snapshots__/moderation-statuses.test.ts.snap b/packages/ozone/tests/__snapshots__/moderation-statuses.test.ts.snap index b2e0cabee16..bf760973d5f 100644 --- a/packages/ozone/tests/__snapshots__/moderation-statuses.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/moderation-statuses.test.ts.snap @@ -4,7 +4,7 @@ exports[`moderation-statuses query statuses returns statuses filtered by subject Array [ Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 9, + "id": 11, "lastReportedAt": "1970-01-01T00:00:00.000Z", "reviewState": "com.atproto.admin.defs#reviewOpen", "subject": Object { @@ -23,7 +23,7 @@ Array [ }, Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 7, + "id": 9, "lastReportedAt": "1970-01-01T00:00:00.000Z", "reviewState": "com.atproto.admin.defs#reviewOpen", "subject": Object { @@ -46,7 +46,7 @@ exports[`moderation-statuses query statuses returns statuses for subjects that r Array [ Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 9, + "id": 11, "lastReportedAt": "1970-01-01T00:00:00.000Z", "reviewState": "com.atproto.admin.defs#reviewOpen", "subject": Object { @@ -65,7 +65,7 @@ Array [ }, Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 7, + "id": 9, "lastReportedAt": "1970-01-01T00:00:00.000Z", "reviewState": "com.atproto.admin.defs#reviewOpen", "subject": Object { @@ -83,7 +83,7 @@ Array [ }, Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 5, + "id": 7, "lastReportedAt": "1970-01-01T00:00:00.000Z", "reviewState": "com.atproto.admin.defs#reviewOpen", "subject": Object { @@ -101,7 +101,7 @@ Array [ }, Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 3, + "id": 5, "lastReportedAt": "1970-01-01T00:00:00.000Z", "reviewState": "com.atproto.admin.defs#reviewOpen", "subject": Object { @@ -118,7 +118,7 @@ Array [ }, Object { "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, + "id": 3, "reviewState": "com.atproto.admin.defs#reviewOptional", "subject": Object { "$type": "com.atproto.repo.strongRef", @@ -133,5 +133,22 @@ Array [ "takendown": false, "updatedAt": "1970-01-01T00:00:00.000Z", }, + Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "id": 1, + "reviewState": "com.atproto.admin.defs#reviewOptional", + "subject": Object { + "$type": "com.atproto.repo.strongRef", + "cid": "cids(3)", + "uri": "record(3)", + }, + "subjectBlobCids": Array [], + "subjectRepoHandle": "alice.test", + "tags": Array [ + "lang:und", + ], + "takendown": false, + "updatedAt": "1970-01-01T00:00:00.000Z", + }, ] `; diff --git a/packages/ozone/tests/moderation-events.test.ts b/packages/ozone/tests/moderation-events.test.ts index 767ca3f65b7..267db2d0190 100644 --- a/packages/ozone/tests/moderation-events.test.ts +++ b/packages/ozone/tests/moderation-events.test.ts @@ -207,7 +207,9 @@ describe('moderation-events', () => { expect(defaultEvents.length).toEqual(allEvents.data.events.length) expect(reversedEvents.length).toEqual(allEvents.data.events.length) // First event in the reversed list is the last item in the default list - expect(reversedEvents[0].id).toEqual(defaultEvents[6].id) + expect(reversedEvents[0].id).toEqual( + defaultEvents[defaultEvents.length - 1].id, + ) }) it('returns report events matching reportType filters', async () => { From 93ac0e4a3456d8b8ebed75b8241f56758f2767d7 Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Tue, 27 Feb 2024 21:27:34 +0100 Subject: [PATCH 04/20] :white_check_mark: Add test for reviewOptional status mutation --- packages/ozone/src/mod-service/status.ts | 6 ++ .../ozone/tests/moderation-statuses.test.ts | 88 +++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/packages/ozone/src/mod-service/status.ts b/packages/ozone/src/mod-service/status.ts index efbdbe66290..4faac7a94fe 100644 --- a/packages/ozone/src/mod-service/status.ts +++ b/packages/ozone/src/mod-service/status.ts @@ -160,6 +160,12 @@ export const adjustModerationSubjectStatus = async ( subjectStatus.reviewState = REVIEWESCALATED } + if (currentStatus && subjectStatus.reviewState === REVIEWOPTIONAL) { + // reviewOptional is ONLY allowed when there is no current status + // If there is a current status, it should not be allowed to move back to reviewOptional + subjectStatus.reviewState = currentStatus.reviewState + } + // Set these because we don't want to override them if they're already set const defaultData = { comment: null, diff --git a/packages/ozone/tests/moderation-statuses.test.ts b/packages/ozone/tests/moderation-statuses.test.ts index 38c69a5056a..b9c857a3b5d 100644 --- a/packages/ozone/tests/moderation-statuses.test.ts +++ b/packages/ozone/tests/moderation-statuses.test.ts @@ -9,6 +9,10 @@ import { REASONMISLEADING, REASONSPAM, } from '../src/lexicon/types/com/atproto/moderation/defs' +import { + REVIEWOPEN, + REVIEWOPTIONAL, +} from '../src/lexicon/types/com/atproto/admin/defs' describe('moderation-statuses', () => { let network: TestNetwork @@ -160,6 +164,90 @@ describe('moderation-statuses', () => { }) }) + describe('reviewState changes', () => { + it('only sets state to #reviewOptional on first non-impactful event', async () => { + const bobsAccount = { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.bob, + } + const alicesPost = { + $type: 'com.atproto.repo.strongRef', + uri: sc.posts[sc.dids.alice][0].ref.uriStr, + cid: sc.posts[sc.dids.alice][0].ref.cidStr, + } + const getBobsAccountStatus = async () => { + const { data } = await queryModerationStatuses({ + subject: bobsAccount.did, + }) + + return data.subjectStatuses[0] + } + // Since bob's account already had a reviewState, it won't be changed by non-impactful events + const bobsAccountStatusBeforeTag = await getBobsAccountStatus() + + await Promise.all([ + emitModerationEvent({ + subject: bobsAccount, + event: { + $type: 'com.atproto.admin.defs#modEventTag', + add: ['newTag'], + remove: [], + comment: 'X', + }, + createdBy: sc.dids.alice, + }), + emitModerationEvent({ + subject: bobsAccount, + event: { + $type: 'com.atproto.admin.defs#modEventComment', + comment: 'X', + }, + createdBy: sc.dids.alice, + }), + ]) + const bobsAccountStatusAfterTag = await getBobsAccountStatus() + + expect(bobsAccountStatusBeforeTag.reviewState).toEqual( + bobsAccountStatusAfterTag.reviewState, + ) + + // Since alice's post didn't have a reviewState it is set to reviewOptional on first non-impactful event + const getAlicesPostStatus = async () => { + const { data } = await queryModerationStatuses({ + subject: alicesPost.uri, + }) + + return data.subjectStatuses[0] + } + + const alicesPostStatusBeforeTag = await getAlicesPostStatus() + expect(alicesPostStatusBeforeTag).toBeUndefined() + + await emitModerationEvent({ + subject: alicesPost, + event: { + $type: 'com.atproto.admin.defs#modEventComment', + comment: 'X', + }, + createdBy: sc.dids.alice, + }) + const alicesPostStatusAfterTag = await getAlicesPostStatus() + expect(alicesPostStatusAfterTag.reviewState).toEqual(REVIEWOPTIONAL) + + await emitModerationEvent({ + subject: alicesPost, + event: { + $type: 'com.atproto.admin.defs#modEventReport', + reportType: REASONMISLEADING, + comment: 'X', + }, + createdBy: sc.dids.alice, + }) + const alicesPostStatusAfterReport = await getAlicesPostStatus() + expect(alicesPostStatusAfterReport.reviewState).toEqual(REVIEWOPEN) + }) + }) + describe('blobs', () => { it('are tracked on takendown subject', async () => { const post = sc.posts[sc.dids.carol][0] From ddd9cd5af3c7db7c764ab109be01417acd127bc3 Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Thu, 29 Feb 2024 01:23:50 +0100 Subject: [PATCH 05/20] :sparkles: Add divertBlobs event to send blobs to abyss --- lexicons/com/atproto/admin/defs.json | 13 +- packages/api/src/client/lexicons.ts | 12 + .../client/types/com/atproto/admin/defs.ts | 20 ++ packages/bsky/src/lexicon/lexicons.ts | 12 + .../lexicon/types/com/atproto/admin/defs.ts | 20 ++ packages/ozone/package.json | 3 +- packages/ozone/src/config/config.ts | 12 + packages/ozone/src/config/env.ts | 4 + packages/ozone/src/context.ts | 7 + packages/ozone/src/daemon/blob-diverter.ts | 206 ++++++++++++++++++ packages/ozone/src/daemon/context.ts | 13 ++ packages/ozone/src/daemon/index.ts | 4 + .../20240228T153256972Z-blob-divert-event.ts | 27 +++ packages/ozone/src/db/migrations/index.ts | 1 + .../ozone/src/db/schema/blob_divert_event.ts | 17 ++ packages/ozone/src/db/schema/index.ts | 4 +- packages/ozone/src/lexicon/lexicons.ts | 12 + .../lexicon/types/com/atproto/admin/defs.ts | 20 ++ packages/ozone/src/mod-service/index.ts | 32 +++ packages/ozone/tests/blob-divert.test.ts | 101 +++++++++ packages/pds/src/lexicon/lexicons.ts | 12 + .../lexicon/types/com/atproto/admin/defs.ts | 20 ++ pnpm-lock.yaml | 31 ++- 23 files changed, 595 insertions(+), 8 deletions(-) create mode 100644 packages/ozone/src/daemon/blob-diverter.ts create mode 100644 packages/ozone/src/db/migrations/20240228T153256972Z-blob-divert-event.ts create mode 100644 packages/ozone/src/db/schema/blob_divert_event.ts create mode 100644 packages/ozone/tests/blob-divert.test.ts diff --git a/lexicons/com/atproto/admin/defs.json b/lexicons/com/atproto/admin/defs.json index 56a855f7427..cedcf022c3a 100644 --- a/lexicons/com/atproto/admin/defs.json +++ b/lexicons/com/atproto/admin/defs.json @@ -34,7 +34,8 @@ "#modEventEscalate", "#modEventMute", "#modEventEmail", - "#modEventResolveAppeal" + "#modEventResolveAppeal", + "#modEventDivertBlobs" ] }, "subject": { @@ -72,7 +73,8 @@ "#modEventEscalate", "#modEventMute", "#modEventEmail", - "#modEventResolveAppeal" + "#modEventResolveAppeal", + "#modEventDivertBlobs" ] }, "subject": { @@ -621,6 +623,13 @@ } } }, + "modEventDivertBlobs": { + "type": "object", + "description": "Divert a record's blobs to a 3rd party service for further scanning/tagging", + "properties": { + "comment": { "type": "string" } + } + }, "communicationTemplateView": { "type": "object", "required": [ diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index b56763f9e62..22e8143fe95 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -92,6 +92,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', + 'lex:com.atproto.admin.defs#modEventDivertBlobs', ], }, subject: { @@ -150,6 +151,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', + 'lex:com.atproto.admin.defs#modEventDivertBlobs', ], }, subject: { @@ -936,6 +938,16 @@ export const schemaDict = { }, }, }, + modEventDivertBlobs: { + type: 'object', + description: + "Divert a record's blobs to a 3rd party service for further scanning/tagging", + properties: { + comment: { + type: 'string', + }, + }, + }, communicationTemplateView: { type: 'object', required: [ diff --git a/packages/api/src/client/types/com/atproto/admin/defs.ts b/packages/api/src/client/types/com/atproto/admin/defs.ts index b3dbb45bc23..13ffd8e3ab4 100644 --- a/packages/api/src/client/types/com/atproto/admin/defs.ts +++ b/packages/api/src/client/types/com/atproto/admin/defs.ts @@ -41,6 +41,7 @@ export interface ModEventView { | ModEventMute | ModEventEmail | ModEventResolveAppeal + | ModEventDivertBlobs | { $type: string; [k: string]: unknown } subject: | RepoRef @@ -79,6 +80,7 @@ export interface ModEventViewDetail { | ModEventMute | ModEventEmail | ModEventResolveAppeal + | ModEventDivertBlobs | { $type: string; [k: string]: unknown } subject: | RepoView @@ -747,6 +749,24 @@ export function validateModEventTag(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#modEventTag', v) } +/** Divert a record's blobs to a 3rd party service for further scanning/tagging */ +export interface ModEventDivertBlobs { + comment?: string + [k: string]: unknown +} + +export function isModEventDivertBlobs(v: unknown): v is ModEventDivertBlobs { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventDivertBlobs' + ) +} + +export function validateModEventDivertBlobs(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventDivertBlobs', v) +} + export interface CommunicationTemplateView { id: string /** Name of the template. */ diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index b56763f9e62..22e8143fe95 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -92,6 +92,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', + 'lex:com.atproto.admin.defs#modEventDivertBlobs', ], }, subject: { @@ -150,6 +151,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', + 'lex:com.atproto.admin.defs#modEventDivertBlobs', ], }, subject: { @@ -936,6 +938,16 @@ export const schemaDict = { }, }, }, + modEventDivertBlobs: { + type: 'object', + description: + "Divert a record's blobs to a 3rd party service for further scanning/tagging", + properties: { + comment: { + type: 'string', + }, + }, + }, communicationTemplateView: { type: 'object', required: [ diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts index 9e6aa7e3269..f99a17c343d 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts @@ -41,6 +41,7 @@ export interface ModEventView { | ModEventMute | ModEventEmail | ModEventResolveAppeal + | ModEventDivertBlobs | { $type: string; [k: string]: unknown } subject: | RepoRef @@ -79,6 +80,7 @@ export interface ModEventViewDetail { | ModEventMute | ModEventEmail | ModEventResolveAppeal + | ModEventDivertBlobs | { $type: string; [k: string]: unknown } subject: | RepoView @@ -747,6 +749,24 @@ export function validateModEventTag(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#modEventTag', v) } +/** Divert a record's blobs to a 3rd party service for further scanning/tagging */ +export interface ModEventDivertBlobs { + comment?: string + [k: string]: unknown +} + +export function isModEventDivertBlobs(v: unknown): v is ModEventDivertBlobs { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventDivertBlobs' + ) +} + +export function validateModEventDivertBlobs(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventDivertBlobs', v) +} + export interface CommunicationTemplateView { id: string /** Name of the template. */ diff --git a/packages/ozone/package.json b/packages/ozone/package.json index e76dc644f10..433f95eeb46 100644 --- a/packages/ozone/package.json +++ b/packages/ozone/package.json @@ -65,6 +65,7 @@ "@types/express-serve-static-core": "^4.17.36", "@types/pg": "^8.6.6", "@types/qs": "^6.9.7", - "axios": "^0.27.2" + "axios": "^0.27.2", + "nock": "14.0.0-beta.4" } } diff --git a/packages/ozone/src/config/config.ts b/packages/ozone/src/config/config.ts index caa799b2a90..9762f976585 100644 --- a/packages/ozone/src/config/config.ts +++ b/packages/ozone/src/config/config.ts @@ -40,12 +40,18 @@ export const envToCfg = (env: OzoneEnvironment): OzoneConfig => { plcUrl: env.didPlcUrl, } + const blobReportServiceCfg = { + url: env.blobReportServiceUrl, + authToken: env.blobReportServiceAuthToken, + } + return { service: serviceCfg, db: dbCfg, appview: appviewCfg, pds: pdsCfg, identity: identityCfg, + blobReportService: blobReportServiceCfg, } } @@ -55,6 +61,7 @@ export type OzoneConfig = { appview: AppviewConfig pds: PdsConfig | null identity: IdentityConfig + blobReportService: BlobReportServiceConfig } export type ServiceConfig = { @@ -64,6 +71,11 @@ export type ServiceConfig = { version?: string } +export type BlobReportServiceConfig = { + url?: string + authToken?: string +} + export type DatabaseConfig = { postgresUrl: string postgresSchema?: string diff --git a/packages/ozone/src/config/env.ts b/packages/ozone/src/config/env.ts index 4f96ba63d53..66022b330f2 100644 --- a/packages/ozone/src/config/env.ts +++ b/packages/ozone/src/config/env.ts @@ -18,6 +18,8 @@ export const readEnv = (): OzoneEnvironment => { moderatorPassword: envStr('OZONE_MODERATOR_PASSWORD'), triagePassword: envStr('OZONE_TRIAGE_PASSWORD'), signingKeyHex: envStr('OZONE_SIGNING_KEY_HEX'), + blobReportServiceUrl: envStr('OZONE_BLOB_REPORT_SERVICE_URL'), + blobReportServiceAuthToken: envStr('OZONE_BLOB_REPORT_SERVICE_AUTH_TOKEN'), } } @@ -38,4 +40,6 @@ export type OzoneEnvironment = { moderatorPassword?: string triagePassword?: string signingKeyHex?: string + blobReportServiceUrl?: string + blobReportServiceAuthToken?: string } diff --git a/packages/ozone/src/context.ts b/packages/ozone/src/context.ts index 1cb0ec1bd83..67907bc41ec 100644 --- a/packages/ozone/src/context.ts +++ b/packages/ozone/src/context.ts @@ -14,6 +14,7 @@ import { CommunicationTemplateService, CommunicationTemplateServiceCreator, } from './communication-service/template' +import { BlobDiverter } from './daemon/blob-diverter' export type AppContextOptions = { db: Database @@ -59,10 +60,16 @@ export class AppContext { appview: cfg.appview, pds: cfg.pds ?? undefined, }) + const blobDiverter = new BlobDiverter(db, { + serviceConfig: cfg.blobReportService, + appview: cfg.appview, + pds: cfg.pds ?? undefined, + }) const modService = ModerationService.creator( backgroundQueue, eventPusher, + blobDiverter, appviewAgent, appviewAuth, cfg.service.did, diff --git a/packages/ozone/src/daemon/blob-diverter.ts b/packages/ozone/src/daemon/blob-diverter.ts new file mode 100644 index 00000000000..a6a8b7e7de7 --- /dev/null +++ b/packages/ozone/src/daemon/blob-diverter.ts @@ -0,0 +1,206 @@ +import AtpAgent, { ComAtprotoSyncGetBlob } from '@atproto/api' +import { SECOND } from '@atproto/common' +import Database from '../db' +import { retryHttp } from '../util' +import { dbLogger } from '../logger' +import { BlobReportServiceConfig } from '../config' + +type PollState = { + timer?: NodeJS.Timer + promise: Promise +} + +type Service = { + agent: AtpAgent + did: string +} + +export class BlobDiverter { + destroyed = false + + pollState: PollState = { + promise: Promise.resolve(), + } + + appview: Service | undefined + pds: Service | undefined + serviceConfig: BlobReportServiceConfig + + constructor( + public db: Database, + services: { + serviceConfig: BlobReportServiceConfig + appview?: { + url: string + did: string + } + pds?: { + url: string + did: string + } + }, + ) { + this.serviceConfig = services.serviceConfig + if (services.appview) { + this.appview = { + agent: new AtpAgent({ service: services.appview.url }), + did: services.appview.did, + } + } + if (services.pds) { + this.pds = { + agent: new AtpAgent({ service: services.pds.url }), + did: services.pds.did, + } + } + } + + start() { + this.poll(this.pollState, () => this.divertBlob()) + } + + poll(state: PollState, fn: () => Promise) { + if (this.destroyed) return + state.promise = fn() + .catch((err) => { + dbLogger.error({ err }, 'blob divert failed') + }) + .finally(() => { + state.timer = setTimeout(() => this.poll(state, fn), 30 * SECOND) + }) + } + + async processAll() { + await Promise.all([this.divertBlob(), this.pollState.promise]) + } + + async destroy() { + this.destroyed = true + const destroyState = (state: PollState) => { + if (state.timer) { + clearTimeout(state.timer) + } + return state.promise + } + await destroyState(this.pollState) + } + + async divertBlob() { + const toPush = await this.db.db + .selectFrom('blob_divert_event') + .select('id') + .forUpdate() + .skipLocked() + .where('divertedAt', 'is', null) + .where('attempts', '<', 10) + .execute() + await Promise.all(toPush.map((evt) => this.attemptBlobDivert(evt.id))) + } + + private async getBlob(opts: { did: string; cid: string }) { + // TODO: Is this safe to do or should we be reaching out to the pds instead? + // do we need to resolve the pds url before we can call this? + return this.pds?.agent.api.com.atproto.sync.getBlob(opts) + } + + private async uploadBlob( + blobResponse: ComAtprotoSyncGetBlob.Response, + { subjectDid, subjectUri }: { subjectDid: string; subjectUri: string }, + ) { + if (!this.serviceConfig.authToken || !this.serviceConfig.url) { + return null + } + + const url = `${this.serviceConfig.url}?did=${subjectDid}&uri=${subjectUri}` + return fetch(url, { + method: 'POST', + body: blobResponse.data, + headers: { + Authorization: this.serviceConfig.authToken, + 'Content-Type': + blobResponse.headers['content-type'] || 'application/octet-stream', + }, + }) + } + + private async uploadBlobOnService({ + subjectDid, + subjectUri, + subjectBlobCid, + }: { + subjectDid: string + subjectUri: string + subjectBlobCid: string + }): Promise { + try { + if (!this.serviceConfig.authToken || !this.serviceConfig.url) { + throw new Error('Blob divert service not configured') + } + + const blobResult = await retryHttp(() => + this.getBlob({ did: subjectDid, cid: subjectBlobCid }), + ) + + if (!blobResult?.success) { + throw new Error('Failed to get blob') + } + + const uploadResult = await retryHttp(() => + this.uploadBlob(blobResult, { subjectDid, subjectUri }), + ) + + return uploadResult?.status === 200 + } catch (err) { + dbLogger.error({ err }, 'failed to upload diverted blob') + return false + } + } + + async attemptBlobDivert(id: number) { + await this.db.transaction(async (dbTxn) => { + const evt = await dbTxn.db + .selectFrom('blob_divert_event') + .selectAll() + .forUpdate() + .skipLocked() + .where('id', '=', id) + .where('divertedAt', 'is', null) + .executeTakeFirst() + if (!evt) return + + const succeeded = await this.uploadBlobOnService(evt) + await dbTxn.db + .updateTable('blob_divert_event') + .set( + succeeded + ? { divertedAt: new Date() } + : { + lastAttempted: new Date(), + attempts: (evt.attempts ?? 0) + 1, + }, + ) + .where('subjectDid', '=', evt.subjectDid) + .where('subjectBlobCid', '=', evt.subjectBlobCid) + .execute() + }) + } + + async logDivertEvent(values: { + subjectDid: string + subjectUri: string + subjectBlobCid: string + }) { + return this.db.db + .insertInto('blob_divert_event') + .values(values) + .onConflict((oc) => + oc.columns(['subjectDid', 'subjectBlobCid']).doUpdateSet({ + divertedAt: null, + attempts: 0, + lastAttempted: null, + }), + ) + .returning('id') + .execute() + } +} diff --git a/packages/ozone/src/daemon/context.ts b/packages/ozone/src/daemon/context.ts index 5af19d89bc4..80037e1edd0 100644 --- a/packages/ozone/src/daemon/context.ts +++ b/packages/ozone/src/daemon/context.ts @@ -7,6 +7,7 @@ import { EventPusher } from './event-pusher' import { EventReverser } from './event-reverser' import { ModerationService, ModerationServiceCreator } from '../mod-service' import { BackgroundQueue } from '../background' +import { BlobDiverter } from './blob-diverter' export type DaemonContextOptions = { db: Database @@ -14,6 +15,7 @@ export type DaemonContextOptions = { modService: ModerationServiceCreator signingKey: Keypair eventPusher: EventPusher + blobDiverter: BlobDiverter eventReverser: EventReverser } @@ -46,10 +48,16 @@ export class DaemonContext { appview: cfg.appview, pds: cfg.pds ?? undefined, }) + const blobDiverter = new BlobDiverter(db, { + serviceConfig: cfg.blobReportService, + appview: cfg.appview, + pds: cfg.pds ?? undefined, + }) const backgroundQueue = new BackgroundQueue(db) const modService = ModerationService.creator( backgroundQueue, eventPusher, + blobDiverter, appviewAgent, appviewAuth, cfg.service.did, @@ -62,6 +70,7 @@ export class DaemonContext { modService, signingKey, eventPusher, + blobDiverter, eventReverser, ...(overrides ?? {}), }) @@ -83,6 +92,10 @@ export class DaemonContext { return this.opts.eventPusher } + get blobDiverter(): BlobDiverter { + return this.opts.blobDiverter + } + get eventReverser(): EventReverser { return this.opts.eventReverser } diff --git a/packages/ozone/src/daemon/index.ts b/packages/ozone/src/daemon/index.ts index aa5d7b12734..8f1d67273f2 100644 --- a/packages/ozone/src/daemon/index.ts +++ b/packages/ozone/src/daemon/index.ts @@ -3,6 +3,7 @@ import DaemonContext from './context' import { AppContextOptions } from '../context' export { EventPusher } from './event-pusher' +export { BlobDiverter } from './blob-diverter' export { EventReverser } from './event-reverser' export class OzoneDaemon { @@ -19,15 +20,18 @@ export class OzoneDaemon { async start() { this.ctx.eventPusher.start() this.ctx.eventReverser.start() + this.ctx.blobDiverter.start() } async processAll() { await this.ctx.eventPusher.processAll() + await this.ctx.blobDiverter.processAll() } async destroy() { await this.ctx.eventReverser.destroy() await this.ctx.eventPusher.destroy() + await this.ctx.blobDiverter.destroy() await this.ctx.db.close() } } diff --git a/packages/ozone/src/db/migrations/20240228T153256972Z-blob-divert-event.ts b/packages/ozone/src/db/migrations/20240228T153256972Z-blob-divert-event.ts new file mode 100644 index 00000000000..933cd3df4b5 --- /dev/null +++ b/packages/ozone/src/db/migrations/20240228T153256972Z-blob-divert-event.ts @@ -0,0 +1,27 @@ +import { Kysely } from 'kysely' + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('blob_divert_event') + .addColumn('id', 'serial', (col) => col.primaryKey()) + .addColumn('subjectDid', 'varchar', (col) => col.notNull()) + .addColumn('subjectBlobCid', 'varchar', (col) => col.notNull()) + .addColumn('subjectUri', 'varchar') + .addColumn('divertedAt', 'timestamptz') + .addColumn('lastAttempted', 'timestamptz') + .addColumn('attempts', 'integer', (col) => col.notNull().defaultTo(0)) + .addUniqueConstraint('blob_divert_event_unique_evt', [ + 'subjectDid', + 'subjectBlobCid', + ]) + .execute() + await db.schema + .createIndex('blob_divert_unique_idx') + .on('blob_divert_event') + .columns(['divertedAt', 'attempts']) + .execute() +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable('blob_divert_event').execute() +} diff --git a/packages/ozone/src/db/migrations/index.ts b/packages/ozone/src/db/migrations/index.ts index 1a823f860c5..4ccaa5791f5 100644 --- a/packages/ozone/src/db/migrations/index.ts +++ b/packages/ozone/src/db/migrations/index.ts @@ -6,3 +6,4 @@ export * as _20231219T205730722Z from './20231219T205730722Z-init' export * as _20240116T085607200Z from './20240116T085607200Z-communication-template' export * as _20240201T051104136Z from './20240201T051104136Z-mod-event-blobs' export * as _20240208T213404429Z from './20240208T213404429Z-add-tags-column-to-moderation-subject' +export * as _20240228T153256972Z from './20240228T153256972Z-blob-divert-event' diff --git a/packages/ozone/src/db/schema/blob_divert_event.ts b/packages/ozone/src/db/schema/blob_divert_event.ts new file mode 100644 index 00000000000..77627f866bc --- /dev/null +++ b/packages/ozone/src/db/schema/blob_divert_event.ts @@ -0,0 +1,17 @@ +import { Generated } from 'kysely' + +export const eventTableName = 'blob_divert_event' + +export interface BlobDivertEvent { + id: Generated + subjectDid: string + subjectBlobCid: string + subjectUri: string + divertedAt: Date | null + lastAttempted: Date | null + attempts: Generated +} + +export type PartialDB = { + [eventTableName]: BlobDivertEvent +} diff --git a/packages/ozone/src/db/schema/index.ts b/packages/ozone/src/db/schema/index.ts index b522a75ef9f..a589d4d0055 100644 --- a/packages/ozone/src/db/schema/index.ts +++ b/packages/ozone/src/db/schema/index.ts @@ -6,6 +6,7 @@ import * as recordPushEvent from './record_push_event' import * as blobPushEvent from './blob_push_event' import * as label from './label' import * as communicationTemplate from './communication_template' +import * as blobDivertEvent from './blob_divert_event' export type DatabaseSchemaType = modEvent.PartialDB & modSubjectStatus.PartialDB & @@ -13,7 +14,8 @@ export type DatabaseSchemaType = modEvent.PartialDB & repoPushEvent.PartialDB & recordPushEvent.PartialDB & blobPushEvent.PartialDB & - communicationTemplate.PartialDB + communicationTemplate.PartialDB & + blobDivertEvent.PartialDB export type DatabaseSchema = Kysely diff --git a/packages/ozone/src/lexicon/lexicons.ts b/packages/ozone/src/lexicon/lexicons.ts index b56763f9e62..22e8143fe95 100644 --- a/packages/ozone/src/lexicon/lexicons.ts +++ b/packages/ozone/src/lexicon/lexicons.ts @@ -92,6 +92,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', + 'lex:com.atproto.admin.defs#modEventDivertBlobs', ], }, subject: { @@ -150,6 +151,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', + 'lex:com.atproto.admin.defs#modEventDivertBlobs', ], }, subject: { @@ -936,6 +938,16 @@ export const schemaDict = { }, }, }, + modEventDivertBlobs: { + type: 'object', + description: + "Divert a record's blobs to a 3rd party service for further scanning/tagging", + properties: { + comment: { + type: 'string', + }, + }, + }, communicationTemplateView: { type: 'object', required: [ diff --git a/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts b/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts index 9e6aa7e3269..f99a17c343d 100644 --- a/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts @@ -41,6 +41,7 @@ export interface ModEventView { | ModEventMute | ModEventEmail | ModEventResolveAppeal + | ModEventDivertBlobs | { $type: string; [k: string]: unknown } subject: | RepoRef @@ -79,6 +80,7 @@ export interface ModEventViewDetail { | ModEventMute | ModEventEmail | ModEventResolveAppeal + | ModEventDivertBlobs | { $type: string; [k: string]: unknown } subject: | RepoView @@ -747,6 +749,24 @@ export function validateModEventTag(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#modEventTag', v) } +/** Divert a record's blobs to a 3rd party service for further scanning/tagging */ +export interface ModEventDivertBlobs { + comment?: string + [k: string]: unknown +} + +export function isModEventDivertBlobs(v: unknown): v is ModEventDivertBlobs { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventDivertBlobs' + ) +} + +export function validateModEventDivertBlobs(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventDivertBlobs', v) +} + export interface CommunicationTemplateView { id: string /** Name of the template. */ diff --git a/packages/ozone/src/mod-service/index.ts b/packages/ozone/src/mod-service/index.ts index 7f06272d55d..8890630e557 100644 --- a/packages/ozone/src/mod-service/index.ts +++ b/packages/ozone/src/mod-service/index.ts @@ -15,6 +15,7 @@ import { isModEventTag, RepoRef, RepoBlobRef, + isModEventDivertBlobs, } from '../lexicon/types/com/atproto/admin/defs' import { adjustModerationSubjectStatus, @@ -43,6 +44,7 @@ import { BlobPushEvent } from '../db/schema/blob_push_event' import { BackgroundQueue } from '../background' import { EventPusher } from '../daemon' import { jsonb } from '../db/types' +import { BlobDiverter } from '../daemon/blob-diverter' export type ModerationServiceCreator = (db: Database) => ModerationService @@ -51,6 +53,7 @@ export class ModerationService { public db: Database, public backgroundQueue: BackgroundQueue, public eventPusher: EventPusher, + public blobDiverter: BlobDiverter, public appviewAgent: AtpAgent, private appviewAuth: AppviewAuth, public serverDid: string, @@ -59,6 +62,7 @@ export class ModerationService { static creator( backgroundQueue: BackgroundQueue, eventPusher: EventPusher, + blobDiverter: BlobDiverter, appviewAgent: AtpAgent, appviewAuth: AppviewAuth, serverDid: string, @@ -68,6 +72,7 @@ export class ModerationService { db, backgroundQueue, eventPusher, + blobDiverter, appviewAgent, appviewAuth, serverDid, @@ -321,6 +326,33 @@ export class ModerationService { subject.blobCids, ) + if ( + isModEventDivertBlobs(event) && + subjectInfo.subjectUri && + subjectInfo.subjectBlobCids?.length + ) { + const blobDiverts = await Promise.all( + subjectInfo.subjectBlobCids?.map((subjectBlobCid) => + this.blobDiverter.logDivertEvent({ + subjectDid: subjectInfo.subjectDid, + subjectBlobCid: subjectBlobCid, + // TODO: Check is done above already + // @ts-ignore + subjectUri: subjectInfo.subjectUri, + }), + ), + ) + this.db.onCommit(() => { + this.backgroundQueue.add(async () => { + await Promise.all( + blobDiverts.map((divert) => + this.blobDiverter.attemptBlobDivert(divert[0].id), + ), + ) + }) + }) + } + return { event: modEvent, subjectStatus } } diff --git a/packages/ozone/tests/blob-divert.test.ts b/packages/ozone/tests/blob-divert.test.ts new file mode 100644 index 00000000000..51f3d2e03f1 --- /dev/null +++ b/packages/ozone/tests/blob-divert.test.ts @@ -0,0 +1,101 @@ +import { SeedClient, TestNetwork, basicSeed } from '@atproto/dev-env' +import AtpAgent from '@atproto/api' +import nock from 'nock' + +const BLOB_DIVERT_SERVICE_HOST = 'http://blob-report.com' +const BLOB_DIVERT_SERVICE_PATH = '/xrpc/com.atproto.unspecced.reportBlob' +const BLOB_DIVERT_SERVICE_AUTH_TOKEN = 'Basic test' + +describe('blob divert', () => { + let network: TestNetwork + let agent: AtpAgent + let sc: SeedClient + + beforeAll(async () => { + network = await TestNetwork.create({ + dbPostgresSchema: 'ozone_blob_divert_test', + ozone: { + blobReportServiceUrl: `${BLOB_DIVERT_SERVICE_HOST}${BLOB_DIVERT_SERVICE_PATH}`, + blobReportServiceAuthToken: BLOB_DIVERT_SERVICE_AUTH_TOKEN, + }, + }) + agent = network.pds.getClient() + sc = network.getSeedClient() + await basicSeed(sc) + await network.processAll() + }) + + afterAll(async () => { + await network.close() + }) + + afterEach(() => { + nock.cleanAll() + }) + + const mockReportServiceResponse = ( + status: number, + data: Record, + ) => + nock(BLOB_DIVERT_SERVICE_HOST) + .persist() + .post(BLOB_DIVERT_SERVICE_PATH, () => true) + .query(true) + .reply(status, data) + + it('fails and keeps attempt count when report service fails to accept upload.', async () => { + // Simulate failure to accept upload + const reportServiceRequest = mockReportServiceResponse(401, { + success: false, + }) + + await agent.api.com.atproto.admin.emitModerationEvent( + { + subject: { + $type: 'com.atproto.repo.strongRef', + uri: sc.posts[sc.dids.carol][0].ref.uriStr, + cid: sc.posts[sc.dids.carol][0].ref.cidStr, + }, + event: { + $type: 'com.atproto.admin.defs#modEventDivertBlobs', + comment: 'Diverting for test', + }, + createdBy: sc.dids.alice, + subjectBlobCids: sc.posts[sc.dids.carol][0].images.map((img) => + img.image.ref.toString(), + ), + }, + { headers: network.pds.adminAuthHeaders(), encoding: 'application/json' }, + ) + + await network.ozone.processAll() + + const divertEvents = await network.ozone.ctx.db.db + .selectFrom('blob_divert_event') + .selectAll() + .execute() + + expect(divertEvents[0].attempts).toBeGreaterThan(0) + expect(divertEvents[1].attempts).toBeGreaterThan(0) + reportServiceRequest.done() + }) + + it('sends blobs to configured divert service and marks divert date', async () => { + // Simulate failure to accept upload + const reportServiceRequest = mockReportServiceResponse(200, { + success: true, + }) + + await network.ozone.processAll() + + const divertEvents = await network.ozone.ctx.db.db + .selectFrom('blob_divert_event') + .selectAll() + .execute() + + expect(divertEvents[0].divertedAt).toBeTruthy() + expect(divertEvents[1].divertedAt).toBeTruthy() + reportServiceRequest.done() + reportServiceRequest.persist(false) + }) +}) diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index b56763f9e62..22e8143fe95 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -92,6 +92,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', + 'lex:com.atproto.admin.defs#modEventDivertBlobs', ], }, subject: { @@ -150,6 +151,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', + 'lex:com.atproto.admin.defs#modEventDivertBlobs', ], }, subject: { @@ -936,6 +938,16 @@ export const schemaDict = { }, }, }, + modEventDivertBlobs: { + type: 'object', + description: + "Divert a record's blobs to a 3rd party service for further scanning/tagging", + properties: { + comment: { + type: 'string', + }, + }, + }, communicationTemplateView: { type: 'object', required: [ diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts index 9e6aa7e3269..f99a17c343d 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts @@ -41,6 +41,7 @@ export interface ModEventView { | ModEventMute | ModEventEmail | ModEventResolveAppeal + | ModEventDivertBlobs | { $type: string; [k: string]: unknown } subject: | RepoRef @@ -79,6 +80,7 @@ export interface ModEventViewDetail { | ModEventMute | ModEventEmail | ModEventResolveAppeal + | ModEventDivertBlobs | { $type: string; [k: string]: unknown } subject: | RepoView @@ -747,6 +749,24 @@ export function validateModEventTag(v: unknown): ValidationResult { return lexicons.validate('com.atproto.admin.defs#modEventTag', v) } +/** Divert a record's blobs to a 3rd party service for further scanning/tagging */ +export interface ModEventDivertBlobs { + comment?: string + [k: string]: unknown +} + +export function isModEventDivertBlobs(v: unknown): v is ModEventDivertBlobs { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'com.atproto.admin.defs#modEventDivertBlobs' + ) +} + +export function validateModEventDivertBlobs(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventDivertBlobs', v) +} + export interface CommunicationTemplateView { id: string /** Name of the template. */ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8ec8d78ae3b..a65b2c8f94b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,9 +1,5 @@ lockfileVersion: '6.0' -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - importers: .: @@ -660,6 +656,9 @@ importers: axios: specifier: ^0.27.2 version: 0.27.2 + nock: + specifier: 14.0.0-beta.4 + version: 14.0.0-beta.4 packages/pds: dependencies: @@ -9430,6 +9429,10 @@ packages: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} dev: true + /json-stringify-safe@5.0.1: + resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} + dev: true + /json5@2.2.3: resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} engines: {node: '>=6'} @@ -9924,6 +9927,17 @@ packages: /neo-async@2.6.2: resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==} + /nock@14.0.0-beta.4: + resolution: {integrity: sha512-N9GIOnNFas/TtdCQpavpi6A6SyVVInkD/vrUCF2u51vlE2wSnqfPifVli6xSX8l6Lz/3sdSwPusE9n3KPDDh0g==} + engines: {node: '>= 18'} + dependencies: + debug: 4.3.4 + json-stringify-safe: 5.0.1 + propagate: 2.0.1 + transitivePeerDependencies: + - supports-color + dev: true + /node-abi@3.47.0: resolution: {integrity: sha512-2s6B2CWZM//kPgwnuI0KrYwNjfdByE25zvAaEpq9IH4zcNsarH8Ihu/UuX6XMPEogDAxkuUFeZn60pXNHAqn3A==} engines: {node: '>=10'} @@ -10540,6 +10554,11 @@ packages: sisteransi: 1.0.5 dev: true + /propagate@2.0.1: + resolution: {integrity: sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==} + engines: {node: '>= 8'} + dev: true + /protobufjs@7.2.5: resolution: {integrity: sha512-gGXRSXvxQ7UiPgfw8gevrfRWcTlSbOFg+p/N+JVJEK5VhueL2miT6qTymqAmjr1Q5WbOCyJbyrk6JfWKwlFn6A==} engines: {node: '>=12.0.0'} @@ -12049,3 +12068,7 @@ packages: /zod@3.21.4: resolution: {integrity: sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==} + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false From c5f2c10b9ebb582b816c61727a76842903535046 Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Thu, 29 Feb 2024 01:52:03 +0100 Subject: [PATCH 06/20] :recycle: Rename reviewOptional -> reviewNone --- lexicons/com/atproto/admin/defs.json | 4 ++-- packages/api/src/client/index.ts | 2 +- packages/api/src/client/lexicons.ts | 4 ++-- .../api/src/client/types/com/atproto/admin/defs.ts | 4 ++-- packages/bsky/src/lexicon/index.ts | 2 +- packages/bsky/src/lexicon/lexicons.ts | 4 ++-- .../bsky/src/lexicon/types/com/atproto/admin/defs.ts | 4 ++-- .../ozone/src/db/schema/moderation_subject_status.ts | 4 ++-- packages/ozone/src/lexicon/index.ts | 2 +- packages/ozone/src/lexicon/lexicons.ts | 4 ++-- .../src/lexicon/types/com/atproto/admin/defs.ts | 4 ++-- packages/ozone/src/mod-service/status.ts | 12 ++++++------ packages/ozone/tests/moderation-statuses.test.ts | 8 ++++---- packages/pds/src/lexicon/index.ts | 2 +- packages/pds/src/lexicon/lexicons.ts | 4 ++-- .../pds/src/lexicon/types/com/atproto/admin/defs.ts | 4 ++-- 16 files changed, 34 insertions(+), 34 deletions(-) diff --git a/lexicons/com/atproto/admin/defs.json b/lexicons/com/atproto/admin/defs.json index 56a855f7427..c73700474fe 100644 --- a/lexicons/com/atproto/admin/defs.json +++ b/lexicons/com/atproto/admin/defs.json @@ -453,7 +453,7 @@ "#reviewOpen", "#reviewEscalated", "#reviewClosed", - "#reviewOptional" + "#reviewNone" ] }, "reviewOpen": { @@ -468,7 +468,7 @@ "type": "token", "description": "Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator" }, - "reviewOptional": { + "reviewNone": { "type": "token", "description": "Moderator review status of a subject: Unnecessary. Indicates that the subject does not need a review at the moment but there is probably some moderation related metadata available for it" }, diff --git a/packages/api/src/client/index.ts b/packages/api/src/client/index.ts index 7f55f73390c..205a986e666 100644 --- a/packages/api/src/client/index.ts +++ b/packages/api/src/client/index.ts @@ -319,7 +319,7 @@ export const COM_ATPROTO_ADMIN = { DefsReviewOpen: 'com.atproto.admin.defs#reviewOpen', DefsReviewEscalated: 'com.atproto.admin.defs#reviewEscalated', DefsReviewClosed: 'com.atproto.admin.defs#reviewClosed', - DefsReviewOptional: 'com.atproto.admin.defs#reviewOptional', + DefsReviewNone: 'com.atproto.admin.defs#reviewNone', } export const COM_ATPROTO_MODERATION = { DefsReasonSpam: 'com.atproto.moderation.defs#reasonSpam', diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index b56763f9e62..a675a0b0201 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -747,7 +747,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#reviewOpen', 'lex:com.atproto.admin.defs#reviewEscalated', 'lex:com.atproto.admin.defs#reviewClosed', - 'lex:com.atproto.admin.defs#reviewOptional', + 'lex:com.atproto.admin.defs#reviewNone', ], }, reviewOpen: { @@ -765,7 +765,7 @@ export const schemaDict = { description: 'Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator', }, - reviewOptional: { + reviewNone: { type: 'token', description: 'Moderator review status of a subject: Unnecessary. Indicates that the subject does not need a review at the moment but there is probably some moderation related metadata available for it', diff --git a/packages/api/src/client/types/com/atproto/admin/defs.ts b/packages/api/src/client/types/com/atproto/admin/defs.ts index b3dbb45bc23..082282505b6 100644 --- a/packages/api/src/client/types/com/atproto/admin/defs.ts +++ b/packages/api/src/client/types/com/atproto/admin/defs.ts @@ -497,7 +497,7 @@ export type SubjectReviewState = | 'lex:com.atproto.admin.defs#reviewOpen' | 'lex:com.atproto.admin.defs#reviewEscalated' | 'lex:com.atproto.admin.defs#reviewClosed' - | 'lex:com.atproto.admin.defs#reviewOptional' + | 'lex:com.atproto.admin.defs#reviewNone' | (string & {}) /** Moderator review status of a subject: Open. Indicates that the subject needs to be reviewed by a moderator */ @@ -507,7 +507,7 @@ export const REVIEWESCALATED = 'com.atproto.admin.defs#reviewEscalated' /** Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator */ export const REVIEWCLOSED = 'com.atproto.admin.defs#reviewClosed' /** Moderator review status of a subject: Unnecessary. Indicates that the subject does not need a review at the moment but there is probably some moderation related metadata available for it */ -export const REVIEWOPTIONAL = 'com.atproto.admin.defs#reviewOptional' +export const REVIEWNONE = 'com.atproto.admin.defs#reviewNone' /** Take down a subject permanently or temporarily */ export interface ModEventTakedown { diff --git a/packages/bsky/src/lexicon/index.ts b/packages/bsky/src/lexicon/index.ts index d42435e2f7c..4ace1ffbc86 100644 --- a/packages/bsky/src/lexicon/index.ts +++ b/packages/bsky/src/lexicon/index.ts @@ -142,7 +142,7 @@ export const COM_ATPROTO_ADMIN = { DefsReviewOpen: 'com.atproto.admin.defs#reviewOpen', DefsReviewEscalated: 'com.atproto.admin.defs#reviewEscalated', DefsReviewClosed: 'com.atproto.admin.defs#reviewClosed', - DefsReviewOptional: 'com.atproto.admin.defs#reviewOptional', + DefsReviewNone: 'com.atproto.admin.defs#reviewNone', } export const COM_ATPROTO_MODERATION = { DefsReasonSpam: 'com.atproto.moderation.defs#reasonSpam', diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index b56763f9e62..a675a0b0201 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -747,7 +747,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#reviewOpen', 'lex:com.atproto.admin.defs#reviewEscalated', 'lex:com.atproto.admin.defs#reviewClosed', - 'lex:com.atproto.admin.defs#reviewOptional', + 'lex:com.atproto.admin.defs#reviewNone', ], }, reviewOpen: { @@ -765,7 +765,7 @@ export const schemaDict = { description: 'Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator', }, - reviewOptional: { + reviewNone: { type: 'token', description: 'Moderator review status of a subject: Unnecessary. Indicates that the subject does not need a review at the moment but there is probably some moderation related metadata available for it', diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts index 9e6aa7e3269..e18381d7b58 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts @@ -497,7 +497,7 @@ export type SubjectReviewState = | 'lex:com.atproto.admin.defs#reviewOpen' | 'lex:com.atproto.admin.defs#reviewEscalated' | 'lex:com.atproto.admin.defs#reviewClosed' - | 'lex:com.atproto.admin.defs#reviewOptional' + | 'lex:com.atproto.admin.defs#reviewNone' | (string & {}) /** Moderator review status of a subject: Open. Indicates that the subject needs to be reviewed by a moderator */ @@ -507,7 +507,7 @@ export const REVIEWESCALATED = 'com.atproto.admin.defs#reviewEscalated' /** Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator */ export const REVIEWCLOSED = 'com.atproto.admin.defs#reviewClosed' /** Moderator review status of a subject: Unnecessary. Indicates that the subject does not need a review at the moment but there is probably some moderation related metadata available for it */ -export const REVIEWOPTIONAL = 'com.atproto.admin.defs#reviewOptional' +export const REVIEWNONE = 'com.atproto.admin.defs#reviewNone' /** Take down a subject permanently or temporarily */ export interface ModEventTakedown { diff --git a/packages/ozone/src/db/schema/moderation_subject_status.ts b/packages/ozone/src/db/schema/moderation_subject_status.ts index 4ed2751f217..45dc6df9d89 100644 --- a/packages/ozone/src/db/schema/moderation_subject_status.ts +++ b/packages/ozone/src/db/schema/moderation_subject_status.ts @@ -3,7 +3,7 @@ import { REVIEWCLOSED, REVIEWOPEN, REVIEWESCALATED, - REVIEWOPTIONAL, + REVIEWNONE, } from '../../lexicon/types/com/atproto/admin/defs' export const subjectStatusTableName = 'moderation_subject_status' @@ -18,7 +18,7 @@ export interface ModerationSubjectStatus { | typeof REVIEWCLOSED | typeof REVIEWOPEN | typeof REVIEWESCALATED - | typeof REVIEWOPTIONAL + | typeof REVIEWNONE createdAt: string updatedAt: string lastReviewedBy: string | null diff --git a/packages/ozone/src/lexicon/index.ts b/packages/ozone/src/lexicon/index.ts index d42435e2f7c..4ace1ffbc86 100644 --- a/packages/ozone/src/lexicon/index.ts +++ b/packages/ozone/src/lexicon/index.ts @@ -142,7 +142,7 @@ export const COM_ATPROTO_ADMIN = { DefsReviewOpen: 'com.atproto.admin.defs#reviewOpen', DefsReviewEscalated: 'com.atproto.admin.defs#reviewEscalated', DefsReviewClosed: 'com.atproto.admin.defs#reviewClosed', - DefsReviewOptional: 'com.atproto.admin.defs#reviewOptional', + DefsReviewNone: 'com.atproto.admin.defs#reviewNone', } export const COM_ATPROTO_MODERATION = { DefsReasonSpam: 'com.atproto.moderation.defs#reasonSpam', diff --git a/packages/ozone/src/lexicon/lexicons.ts b/packages/ozone/src/lexicon/lexicons.ts index b56763f9e62..a675a0b0201 100644 --- a/packages/ozone/src/lexicon/lexicons.ts +++ b/packages/ozone/src/lexicon/lexicons.ts @@ -747,7 +747,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#reviewOpen', 'lex:com.atproto.admin.defs#reviewEscalated', 'lex:com.atproto.admin.defs#reviewClosed', - 'lex:com.atproto.admin.defs#reviewOptional', + 'lex:com.atproto.admin.defs#reviewNone', ], }, reviewOpen: { @@ -765,7 +765,7 @@ export const schemaDict = { description: 'Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator', }, - reviewOptional: { + reviewNone: { type: 'token', description: 'Moderator review status of a subject: Unnecessary. Indicates that the subject does not need a review at the moment but there is probably some moderation related metadata available for it', diff --git a/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts b/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts index 9e6aa7e3269..e18381d7b58 100644 --- a/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts @@ -497,7 +497,7 @@ export type SubjectReviewState = | 'lex:com.atproto.admin.defs#reviewOpen' | 'lex:com.atproto.admin.defs#reviewEscalated' | 'lex:com.atproto.admin.defs#reviewClosed' - | 'lex:com.atproto.admin.defs#reviewOptional' + | 'lex:com.atproto.admin.defs#reviewNone' | (string & {}) /** Moderator review status of a subject: Open. Indicates that the subject needs to be reviewed by a moderator */ @@ -507,7 +507,7 @@ export const REVIEWESCALATED = 'com.atproto.admin.defs#reviewEscalated' /** Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator */ export const REVIEWCLOSED = 'com.atproto.admin.defs#reviewClosed' /** Moderator review status of a subject: Unnecessary. Indicates that the subject does not need a review at the moment but there is probably some moderation related metadata available for it */ -export const REVIEWOPTIONAL = 'com.atproto.admin.defs#reviewOptional' +export const REVIEWNONE = 'com.atproto.admin.defs#reviewNone' /** Take down a subject permanently or temporarily */ export interface ModEventTakedown { diff --git a/packages/ozone/src/mod-service/status.ts b/packages/ozone/src/mod-service/status.ts index 96d8c016d44..38f6253d766 100644 --- a/packages/ozone/src/mod-service/status.ts +++ b/packages/ozone/src/mod-service/status.ts @@ -7,7 +7,7 @@ import { REVIEWOPEN, REVIEWCLOSED, REVIEWESCALATED, - REVIEWOPTIONAL, + REVIEWNONE, } from '../lexicon/types/com/atproto/admin/defs' import { ModerationEventRow, ModerationSubjectStatusRow } from './types' import { HOUR } from '@atproto/common' @@ -29,7 +29,7 @@ const getSubjectStatusForModerationEvent = ({ }): Partial => { const defaultReviewState = currentStatus ? currentStatus.reviewState - : REVIEWOPTIONAL + : REVIEWNONE switch (action) { case 'com.atproto.admin.defs#modEventAcknowledge': @@ -160,9 +160,9 @@ export const adjustModerationSubjectStatus = async ( subjectStatus.reviewState = REVIEWESCALATED } - if (currentStatus && subjectStatus.reviewState === REVIEWOPTIONAL) { - // reviewOptional is ONLY allowed when there is no current status - // If there is a current status, it should not be allowed to move back to reviewOptional + if (currentStatus && subjectStatus.reviewState === REVIEWNONE) { + // reviewNone is ONLY allowed when there is no current status + // If there is a current status, it should not be allowed to move back to reviewNone subjectStatus.reviewState = currentStatus.reviewState } @@ -172,7 +172,7 @@ export const adjustModerationSubjectStatus = async ( // Defaulting reviewState to open for any event may not be the desired behavior. // For instance, if a subject never had any event and we just want to leave a comment to keep an eye on it // that shouldn't mean we want to review the subject - reviewState: REVIEWOPTIONAL, + reviewState: REVIEWNONE, recordCid: subjectCid || null, } const newStatus = { diff --git a/packages/ozone/tests/moderation-statuses.test.ts b/packages/ozone/tests/moderation-statuses.test.ts index 1df3684f1d2..d1f47c880fc 100644 --- a/packages/ozone/tests/moderation-statuses.test.ts +++ b/packages/ozone/tests/moderation-statuses.test.ts @@ -11,7 +11,7 @@ import { } from '../src/lexicon/types/com/atproto/moderation/defs' import { REVIEWOPEN, - REVIEWOPTIONAL, + REVIEWNONE, } from '../src/lexicon/types/com/atproto/admin/defs' describe('moderation-statuses', () => { @@ -165,7 +165,7 @@ describe('moderation-statuses', () => { }) describe('reviewState changes', () => { - it('only sets state to #reviewOptional on first non-impactful event', async () => { + it('only sets state to #reviewNone on first non-impactful event', async () => { const bobsAccount = { $type: 'com.atproto.admin.defs#repoRef', did: sc.dids.bob, @@ -211,7 +211,7 @@ describe('moderation-statuses', () => { bobsAccountStatusAfterTag.reviewState, ) - // Since alice's post didn't have a reviewState it is set to reviewOptional on first non-impactful event + // Since alice's post didn't have a reviewState it is set to reviewNone on first non-impactful event const getAlicesPostStatus = async () => { const { data } = await queryModerationStatuses({ subject: alicesPost.uri, @@ -232,7 +232,7 @@ describe('moderation-statuses', () => { createdBy: sc.dids.alice, }) const alicesPostStatusAfterTag = await getAlicesPostStatus() - expect(alicesPostStatusAfterTag.reviewState).toEqual(REVIEWOPTIONAL) + expect(alicesPostStatusAfterTag.reviewState).toEqual(REVIEWNONE) await emitModerationEvent({ subject: alicesPost, diff --git a/packages/pds/src/lexicon/index.ts b/packages/pds/src/lexicon/index.ts index d42435e2f7c..4ace1ffbc86 100644 --- a/packages/pds/src/lexicon/index.ts +++ b/packages/pds/src/lexicon/index.ts @@ -142,7 +142,7 @@ export const COM_ATPROTO_ADMIN = { DefsReviewOpen: 'com.atproto.admin.defs#reviewOpen', DefsReviewEscalated: 'com.atproto.admin.defs#reviewEscalated', DefsReviewClosed: 'com.atproto.admin.defs#reviewClosed', - DefsReviewOptional: 'com.atproto.admin.defs#reviewOptional', + DefsReviewNone: 'com.atproto.admin.defs#reviewNone', } export const COM_ATPROTO_MODERATION = { DefsReasonSpam: 'com.atproto.moderation.defs#reasonSpam', diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index b56763f9e62..a675a0b0201 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -747,7 +747,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#reviewOpen', 'lex:com.atproto.admin.defs#reviewEscalated', 'lex:com.atproto.admin.defs#reviewClosed', - 'lex:com.atproto.admin.defs#reviewOptional', + 'lex:com.atproto.admin.defs#reviewNone', ], }, reviewOpen: { @@ -765,7 +765,7 @@ export const schemaDict = { description: 'Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator', }, - reviewOptional: { + reviewNone: { type: 'token', description: 'Moderator review status of a subject: Unnecessary. Indicates that the subject does not need a review at the moment but there is probably some moderation related metadata available for it', diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts index 9e6aa7e3269..e18381d7b58 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts @@ -497,7 +497,7 @@ export type SubjectReviewState = | 'lex:com.atproto.admin.defs#reviewOpen' | 'lex:com.atproto.admin.defs#reviewEscalated' | 'lex:com.atproto.admin.defs#reviewClosed' - | 'lex:com.atproto.admin.defs#reviewOptional' + | 'lex:com.atproto.admin.defs#reviewNone' | (string & {}) /** Moderator review status of a subject: Open. Indicates that the subject needs to be reviewed by a moderator */ @@ -507,7 +507,7 @@ export const REVIEWESCALATED = 'com.atproto.admin.defs#reviewEscalated' /** Moderator review status of a subject: Closed. Indicates that the subject was already reviewed and resolved by a moderator */ export const REVIEWCLOSED = 'com.atproto.admin.defs#reviewClosed' /** Moderator review status of a subject: Unnecessary. Indicates that the subject does not need a review at the moment but there is probably some moderation related metadata available for it */ -export const REVIEWOPTIONAL = 'com.atproto.admin.defs#reviewOptional' +export const REVIEWNONE = 'com.atproto.admin.defs#reviewNone' /** Take down a subject permanently or temporarily */ export interface ModEventTakedown { From 4b546e960c27963f45162c55e779817f7c8cc690 Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Thu, 29 Feb 2024 13:16:29 +0100 Subject: [PATCH 07/20] :recycle: Rename modEventDivertBlobs -> modEventDivert --- lexicons/com/atproto/admin/defs.json | 6 +++--- packages/api/src/client/lexicons.ts | 6 +++--- .../api/src/client/types/com/atproto/admin/defs.ts | 14 +++++++------- packages/bsky/src/lexicon/lexicons.ts | 6 +++--- .../src/lexicon/types/com/atproto/admin/defs.ts | 14 +++++++------- packages/ozone/src/lexicon/lexicons.ts | 6 +++--- .../src/lexicon/types/com/atproto/admin/defs.ts | 14 +++++++------- packages/ozone/tests/blob-divert.test.ts | 2 +- packages/pds/src/lexicon/lexicons.ts | 6 +++--- .../src/lexicon/types/com/atproto/admin/defs.ts | 14 +++++++------- 10 files changed, 44 insertions(+), 44 deletions(-) diff --git a/lexicons/com/atproto/admin/defs.json b/lexicons/com/atproto/admin/defs.json index cedcf022c3a..b84d6a4b272 100644 --- a/lexicons/com/atproto/admin/defs.json +++ b/lexicons/com/atproto/admin/defs.json @@ -35,7 +35,7 @@ "#modEventMute", "#modEventEmail", "#modEventResolveAppeal", - "#modEventDivertBlobs" + "#modEventDivert" ] }, "subject": { @@ -74,7 +74,7 @@ "#modEventMute", "#modEventEmail", "#modEventResolveAppeal", - "#modEventDivertBlobs" + "#modEventDivert" ] }, "subject": { @@ -623,7 +623,7 @@ } } }, - "modEventDivertBlobs": { + "modEventDivert": { "type": "object", "description": "Divert a record's blobs to a 3rd party service for further scanning/tagging", "properties": { diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index 22e8143fe95..e0a254ead71 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -92,7 +92,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', - 'lex:com.atproto.admin.defs#modEventDivertBlobs', + 'lex:com.atproto.admin.defs#modEventDivert', ], }, subject: { @@ -151,7 +151,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', - 'lex:com.atproto.admin.defs#modEventDivertBlobs', + 'lex:com.atproto.admin.defs#modEventDivert', ], }, subject: { @@ -938,7 +938,7 @@ export const schemaDict = { }, }, }, - modEventDivertBlobs: { + modEventDivert: { type: 'object', description: "Divert a record's blobs to a 3rd party service for further scanning/tagging", diff --git a/packages/api/src/client/types/com/atproto/admin/defs.ts b/packages/api/src/client/types/com/atproto/admin/defs.ts index 13ffd8e3ab4..4475473355a 100644 --- a/packages/api/src/client/types/com/atproto/admin/defs.ts +++ b/packages/api/src/client/types/com/atproto/admin/defs.ts @@ -41,7 +41,7 @@ export interface ModEventView { | ModEventMute | ModEventEmail | ModEventResolveAppeal - | ModEventDivertBlobs + | ModEventDivert | { $type: string; [k: string]: unknown } subject: | RepoRef @@ -80,7 +80,7 @@ export interface ModEventViewDetail { | ModEventMute | ModEventEmail | ModEventResolveAppeal - | ModEventDivertBlobs + | ModEventDivert | { $type: string; [k: string]: unknown } subject: | RepoView @@ -750,21 +750,21 @@ export function validateModEventTag(v: unknown): ValidationResult { } /** Divert a record's blobs to a 3rd party service for further scanning/tagging */ -export interface ModEventDivertBlobs { +export interface ModEventDivert { comment?: string [k: string]: unknown } -export function isModEventDivertBlobs(v: unknown): v is ModEventDivertBlobs { +export function isModEventDivert(v: unknown): v is ModEventDivert { return ( isObj(v) && hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#modEventDivertBlobs' + v.$type === 'com.atproto.admin.defs#modEventDivert' ) } -export function validateModEventDivertBlobs(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#modEventDivertBlobs', v) +export function validateModEventDivert(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventDivert', v) } export interface CommunicationTemplateView { diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index 22e8143fe95..e0a254ead71 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -92,7 +92,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', - 'lex:com.atproto.admin.defs#modEventDivertBlobs', + 'lex:com.atproto.admin.defs#modEventDivert', ], }, subject: { @@ -151,7 +151,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', - 'lex:com.atproto.admin.defs#modEventDivertBlobs', + 'lex:com.atproto.admin.defs#modEventDivert', ], }, subject: { @@ -938,7 +938,7 @@ export const schemaDict = { }, }, }, - modEventDivertBlobs: { + modEventDivert: { type: 'object', description: "Divert a record's blobs to a 3rd party service for further scanning/tagging", diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts index f99a17c343d..dc3a92a713f 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts @@ -41,7 +41,7 @@ export interface ModEventView { | ModEventMute | ModEventEmail | ModEventResolveAppeal - | ModEventDivertBlobs + | ModEventDivert | { $type: string; [k: string]: unknown } subject: | RepoRef @@ -80,7 +80,7 @@ export interface ModEventViewDetail { | ModEventMute | ModEventEmail | ModEventResolveAppeal - | ModEventDivertBlobs + | ModEventDivert | { $type: string; [k: string]: unknown } subject: | RepoView @@ -750,21 +750,21 @@ export function validateModEventTag(v: unknown): ValidationResult { } /** Divert a record's blobs to a 3rd party service for further scanning/tagging */ -export interface ModEventDivertBlobs { +export interface ModEventDivert { comment?: string [k: string]: unknown } -export function isModEventDivertBlobs(v: unknown): v is ModEventDivertBlobs { +export function isModEventDivert(v: unknown): v is ModEventDivert { return ( isObj(v) && hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#modEventDivertBlobs' + v.$type === 'com.atproto.admin.defs#modEventDivert' ) } -export function validateModEventDivertBlobs(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#modEventDivertBlobs', v) +export function validateModEventDivert(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventDivert', v) } export interface CommunicationTemplateView { diff --git a/packages/ozone/src/lexicon/lexicons.ts b/packages/ozone/src/lexicon/lexicons.ts index 22e8143fe95..e0a254ead71 100644 --- a/packages/ozone/src/lexicon/lexicons.ts +++ b/packages/ozone/src/lexicon/lexicons.ts @@ -92,7 +92,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', - 'lex:com.atproto.admin.defs#modEventDivertBlobs', + 'lex:com.atproto.admin.defs#modEventDivert', ], }, subject: { @@ -151,7 +151,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', - 'lex:com.atproto.admin.defs#modEventDivertBlobs', + 'lex:com.atproto.admin.defs#modEventDivert', ], }, subject: { @@ -938,7 +938,7 @@ export const schemaDict = { }, }, }, - modEventDivertBlobs: { + modEventDivert: { type: 'object', description: "Divert a record's blobs to a 3rd party service for further scanning/tagging", diff --git a/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts b/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts index f99a17c343d..dc3a92a713f 100644 --- a/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts @@ -41,7 +41,7 @@ export interface ModEventView { | ModEventMute | ModEventEmail | ModEventResolveAppeal - | ModEventDivertBlobs + | ModEventDivert | { $type: string; [k: string]: unknown } subject: | RepoRef @@ -80,7 +80,7 @@ export interface ModEventViewDetail { | ModEventMute | ModEventEmail | ModEventResolveAppeal - | ModEventDivertBlobs + | ModEventDivert | { $type: string; [k: string]: unknown } subject: | RepoView @@ -750,21 +750,21 @@ export function validateModEventTag(v: unknown): ValidationResult { } /** Divert a record's blobs to a 3rd party service for further scanning/tagging */ -export interface ModEventDivertBlobs { +export interface ModEventDivert { comment?: string [k: string]: unknown } -export function isModEventDivertBlobs(v: unknown): v is ModEventDivertBlobs { +export function isModEventDivert(v: unknown): v is ModEventDivert { return ( isObj(v) && hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#modEventDivertBlobs' + v.$type === 'com.atproto.admin.defs#modEventDivert' ) } -export function validateModEventDivertBlobs(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#modEventDivertBlobs', v) +export function validateModEventDivert(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventDivert', v) } export interface CommunicationTemplateView { diff --git a/packages/ozone/tests/blob-divert.test.ts b/packages/ozone/tests/blob-divert.test.ts index 51f3d2e03f1..193e9518b21 100644 --- a/packages/ozone/tests/blob-divert.test.ts +++ b/packages/ozone/tests/blob-divert.test.ts @@ -57,7 +57,7 @@ describe('blob divert', () => { cid: sc.posts[sc.dids.carol][0].ref.cidStr, }, event: { - $type: 'com.atproto.admin.defs#modEventDivertBlobs', + $type: 'com.atproto.admin.defs#modEventDivert', comment: 'Diverting for test', }, createdBy: sc.dids.alice, diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index 22e8143fe95..e0a254ead71 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -92,7 +92,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', - 'lex:com.atproto.admin.defs#modEventDivertBlobs', + 'lex:com.atproto.admin.defs#modEventDivert', ], }, subject: { @@ -151,7 +151,7 @@ export const schemaDict = { 'lex:com.atproto.admin.defs#modEventMute', 'lex:com.atproto.admin.defs#modEventEmail', 'lex:com.atproto.admin.defs#modEventResolveAppeal', - 'lex:com.atproto.admin.defs#modEventDivertBlobs', + 'lex:com.atproto.admin.defs#modEventDivert', ], }, subject: { @@ -938,7 +938,7 @@ export const schemaDict = { }, }, }, - modEventDivertBlobs: { + modEventDivert: { type: 'object', description: "Divert a record's blobs to a 3rd party service for further scanning/tagging", diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts index f99a17c343d..dc3a92a713f 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts @@ -41,7 +41,7 @@ export interface ModEventView { | ModEventMute | ModEventEmail | ModEventResolveAppeal - | ModEventDivertBlobs + | ModEventDivert | { $type: string; [k: string]: unknown } subject: | RepoRef @@ -80,7 +80,7 @@ export interface ModEventViewDetail { | ModEventMute | ModEventEmail | ModEventResolveAppeal - | ModEventDivertBlobs + | ModEventDivert | { $type: string; [k: string]: unknown } subject: | RepoView @@ -750,21 +750,21 @@ export function validateModEventTag(v: unknown): ValidationResult { } /** Divert a record's blobs to a 3rd party service for further scanning/tagging */ -export interface ModEventDivertBlobs { +export interface ModEventDivert { comment?: string [k: string]: unknown } -export function isModEventDivertBlobs(v: unknown): v is ModEventDivertBlobs { +export function isModEventDivert(v: unknown): v is ModEventDivert { return ( isObj(v) && hasProp(v, '$type') && - v.$type === 'com.atproto.admin.defs#modEventDivertBlobs' + v.$type === 'com.atproto.admin.defs#modEventDivert' ) } -export function validateModEventDivertBlobs(v: unknown): ValidationResult { - return lexicons.validate('com.atproto.admin.defs#modEventDivertBlobs', v) +export function validateModEventDivert(v: unknown): ValidationResult { + return lexicons.validate('com.atproto.admin.defs#modEventDivert', v) } export interface CommunicationTemplateView { From 64dd7ac1153fef7ebd1ec2a35dabe658099b492b Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Thu, 29 Feb 2024 13:21:00 +0100 Subject: [PATCH 08/20] :bug: Rename event type checker --- packages/ozone/src/mod-service/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ozone/src/mod-service/index.ts b/packages/ozone/src/mod-service/index.ts index 8890630e557..eccca3ac10e 100644 --- a/packages/ozone/src/mod-service/index.ts +++ b/packages/ozone/src/mod-service/index.ts @@ -15,7 +15,7 @@ import { isModEventTag, RepoRef, RepoBlobRef, - isModEventDivertBlobs, + isModEventDivert, } from '../lexicon/types/com/atproto/admin/defs' import { adjustModerationSubjectStatus, @@ -327,7 +327,7 @@ export class ModerationService { ) if ( - isModEventDivertBlobs(event) && + isModEventDivert(event) && subjectInfo.subjectUri && subjectInfo.subjectBlobCids?.length ) { From 071712cb1829e73e59879ab860ca7652950e8ed5 Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Thu, 29 Feb 2024 14:56:42 +0100 Subject: [PATCH 09/20] :sparkles: Use pds resolver to get blob straight from pds --- packages/ozone/src/context.ts | 11 +- packages/ozone/src/daemon/blob-diverter.ts | 115 ++++++++++++--------- packages/ozone/src/daemon/context.ts | 8 +- 3 files changed, 80 insertions(+), 54 deletions(-) diff --git a/packages/ozone/src/context.ts b/packages/ozone/src/context.ts index 298f1ee54c2..9018935d222 100644 --- a/packages/ozone/src/context.ts +++ b/packages/ozone/src/context.ts @@ -51,6 +51,10 @@ export class AppContext { ? new AtpAgent({ service: cfg.pds.url }) : undefined + const idResolver = new IdResolver({ + plcUrl: cfg.identity.plcUrl, + }) + const createAuthHeaders = (aud: string) => createServiceAuthHeaders({ iss: cfg.service.did, @@ -66,9 +70,8 @@ export class AppContext { pds: cfg.pds ?? undefined, }) const blobDiverter = new BlobDiverter(db, { + idResolver, serviceConfig: cfg.blobReportService, - appview: cfg.appview, - pds: cfg.pds ?? undefined, }) const modService = ModerationService.creator( @@ -82,10 +85,6 @@ export class AppContext { const communicationTemplateService = CommunicationTemplateService.creator() - const idResolver = new IdResolver({ - plcUrl: cfg.identity.plcUrl, - }) - const sequencer = new Sequencer(db) return new AppContext( diff --git a/packages/ozone/src/daemon/blob-diverter.ts b/packages/ozone/src/daemon/blob-diverter.ts index a6a8b7e7de7..d8be3fcddf3 100644 --- a/packages/ozone/src/daemon/blob-diverter.ts +++ b/packages/ozone/src/daemon/blob-diverter.ts @@ -1,5 +1,14 @@ -import AtpAgent, { ComAtprotoSyncGetBlob } from '@atproto/api' -import { SECOND } from '@atproto/common' +import { + SECOND, + VerifyCidTransform, + forwardStreamErrors, + getPdsEndpoint, +} from '@atproto/common' +import { IdResolver } from '@atproto/identity' +import axios from 'axios' +import { Readable } from 'stream' +import { CID } from 'multiformats/cid' + import Database from '../db' import { retryHttp } from '../util' import { dbLogger } from '../logger' @@ -10,11 +19,6 @@ type PollState = { promise: Promise } -type Service = { - agent: AtpAgent - did: string -} - export class BlobDiverter { destroyed = false @@ -22,37 +26,18 @@ export class BlobDiverter { promise: Promise.resolve(), } - appview: Service | undefined - pds: Service | undefined serviceConfig: BlobReportServiceConfig + idResolver: IdResolver constructor( public db: Database, services: { + idResolver: IdResolver serviceConfig: BlobReportServiceConfig - appview?: { - url: string - did: string - } - pds?: { - url: string - did: string - } }, ) { this.serviceConfig = services.serviceConfig - if (services.appview) { - this.appview = { - agent: new AtpAgent({ service: services.appview.url }), - did: services.appview.did, - } - } - if (services.pds) { - this.pds = { - agent: new AtpAgent({ service: services.pds.url }), - did: services.pds.did, - } - } + this.idResolver = services.idResolver } start() { @@ -97,30 +82,57 @@ export class BlobDiverter { await Promise.all(toPush.map((evt) => this.attemptBlobDivert(evt.id))) } - private async getBlob(opts: { did: string; cid: string }) { - // TODO: Is this safe to do or should we be reaching out to the pds instead? - // do we need to resolve the pds url before we can call this? - return this.pds?.agent.api.com.atproto.sync.getBlob(opts) + private async getBlob({ + pds, + did, + cid, + }: { + pds: string + did: string + cid: string + }) { + const blobResponse = await axios.get( + `${pds}/xrpc/com.atproto.sync.getBlob`, + { + params: { did, cid }, + decompress: true, + responseType: 'stream', + timeout: 5000, // 5sec of inactivity on the connection + }, + ) + const imageStream: Readable = blobResponse.data + const verifyCid = new VerifyCidTransform(CID.parse(cid)) + forwardStreamErrors(imageStream, verifyCid) + + return { + contentType: + blobResponse.headers['content-type'] || 'application/octet-stream', + imageStream: imageStream.pipe(verifyCid), + } } private async uploadBlob( - blobResponse: ComAtprotoSyncGetBlob.Response, + { + imageStream, + contentType, + }: { imageStream: Readable; contentType: string }, { subjectDid, subjectUri }: { subjectDid: string; subjectUri: string }, ) { if (!this.serviceConfig.authToken || !this.serviceConfig.url) { - return null + return false } const url = `${this.serviceConfig.url}?did=${subjectDid}&uri=${subjectUri}` - return fetch(url, { + const result = await axios(url, { method: 'POST', - body: blobResponse.data, + data: imageStream, headers: { Authorization: this.serviceConfig.authToken, - 'Content-Type': - blobResponse.headers['content-type'] || 'application/octet-stream', + 'Content-Type': contentType, }, }) + + return result.status === 200 } private async uploadBlobOnService({ @@ -137,19 +149,30 @@ export class BlobDiverter { throw new Error('Blob divert service not configured') } - const blobResult = await retryHttp(() => - this.getBlob({ did: subjectDid, cid: subjectBlobCid }), - ) + const didDoc = await this.idResolver.did.resolve(subjectDid) - if (!blobResult?.success) { - throw new Error('Failed to get blob') + if (!didDoc) { + throw new Error('Error resolving DID') } + const pds = getPdsEndpoint(didDoc) + + if (!pds) { + throw new Error('Error resolving PDS') + } + + const { imageStream, contentType } = await retryHttp(() => + this.getBlob({ pds, did: subjectDid, cid: subjectBlobCid }), + ) + const uploadResult = await retryHttp(() => - this.uploadBlob(blobResult, { subjectDid, subjectUri }), + this.uploadBlob( + { imageStream, contentType }, + { subjectDid, subjectUri }, + ), ) - return uploadResult?.status === 200 + return uploadResult } catch (err) { dbLogger.error({ err }, 'failed to upload diverted blob') return false diff --git a/packages/ozone/src/daemon/context.ts b/packages/ozone/src/daemon/context.ts index 80037e1edd0..4ad53812388 100644 --- a/packages/ozone/src/daemon/context.ts +++ b/packages/ozone/src/daemon/context.ts @@ -1,5 +1,6 @@ import { Keypair, Secp256k1Keypair } from '@atproto/crypto' import { createServiceAuthHeaders } from '@atproto/xrpc-server' +import { IdResolver } from '@atproto/identity' import AtpAgent from '@atproto/api' import { OzoneConfig, OzoneSecrets } from '../config' import { Database } from '../db' @@ -33,6 +34,10 @@ export class DaemonContext { }) const signingKey = await Secp256k1Keypair.import(secrets.signingKeyHex) + const idResolver = new IdResolver({ + plcUrl: cfg.identity.plcUrl, + }) + const appviewAgent = new AtpAgent({ service: cfg.appview.url }) const createAuthHeaders = (aud: string) => createServiceAuthHeaders({ @@ -49,9 +54,8 @@ export class DaemonContext { pds: cfg.pds ?? undefined, }) const blobDiverter = new BlobDiverter(db, { + idResolver, serviceConfig: cfg.blobReportService, - appview: cfg.appview, - pds: cfg.pds ?? undefined, }) const backgroundQueue = new BackgroundQueue(db) const modService = ModerationService.creator( From 5c0e20a22e63663fcb9472de4cc2ee1ce9e38d22 Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Mon, 4 Mar 2024 23:07:27 +0000 Subject: [PATCH 10/20] :white_check_mark: Use FOR UPDATE to respect db transactions --- packages/ozone/src/mod-service/status.ts | 2 ++ .../ozone/tests/__snapshots__/moderation-events.test.ts.snap | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/ozone/src/mod-service/status.ts b/packages/ozone/src/mod-service/status.ts index 38f6253d766..81cebe2dddb 100644 --- a/packages/ozone/src/mod-service/status.ts +++ b/packages/ozone/src/mod-service/status.ts @@ -135,6 +135,8 @@ export const adjustModerationSubjectStatus = async ( .selectFrom('moderation_subject_status') .where('did', '=', identifier.did) .where('recordPath', '=', identifier.recordPath) + // Make sure we respect other updates that may be happening at the same time + .forUpdate() .selectAll() .executeTakeFirst() diff --git a/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap b/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap index 216eb970ca5..fcb73413622 100644 --- a/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap @@ -22,7 +22,7 @@ Object { "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "user(1)", - "reviewState": "com.atproto.admin.defs#reviewOpen", + "reviewState": "com.atproto.admin.defs#reviewEscalated", "subject": Object { "$type": "com.atproto.admin.defs#repoRef", "did": "user(0)", From be01cf0291121d6445789b0e8cbb29d236b14c5d Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Wed, 6 Mar 2024 00:13:54 +0000 Subject: [PATCH 11/20] :recycle: Refactor to use event_pusher table instead of new table --- packages/ozone/package.json | 3 +- .../src/api/admin/emitModerationEvent.ts | 5 + packages/ozone/src/config/config.ts | 17 +- packages/ozone/src/context.ts | 13 +- packages/ozone/src/daemon/blob-diverter.ts | 171 +++++------------- packages/ozone/src/daemon/context.ts | 16 +- packages/ozone/src/daemon/event-pusher.ts | 91 +++++++--- packages/ozone/src/daemon/index.ts | 3 - .../20240228T153256972Z-blob-divert-event.ts | 27 --- packages/ozone/src/db/migrations/index.ts | 1 - .../ozone/src/db/schema/blob_push_event.ts | 5 +- packages/ozone/src/mod-service/index.ts | 77 +++----- packages/ozone/tests/blob-divert.test.ts | 58 +++--- 13 files changed, 198 insertions(+), 289 deletions(-) delete mode 100644 packages/ozone/src/db/migrations/20240228T153256972Z-blob-divert-event.ts diff --git a/packages/ozone/package.json b/packages/ozone/package.json index 433f95eeb46..e76dc644f10 100644 --- a/packages/ozone/package.json +++ b/packages/ozone/package.json @@ -65,7 +65,6 @@ "@types/express-serve-static-core": "^4.17.36", "@types/pg": "^8.6.6", "@types/qs": "^6.9.7", - "axios": "^0.27.2", - "nock": "14.0.0-beta.4" + "axios": "^0.27.2" } } diff --git a/packages/ozone/src/api/admin/emitModerationEvent.ts b/packages/ozone/src/api/admin/emitModerationEvent.ts index ef4c5fd2822..748707eb47c 100644 --- a/packages/ozone/src/api/admin/emitModerationEvent.ts +++ b/packages/ozone/src/api/admin/emitModerationEvent.ts @@ -2,6 +2,7 @@ import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../lexicon' import AppContext from '../../context' import { + isModEventDivert, isModEventLabel, isModEventReverseTakedown, isModEventTakedown, @@ -91,6 +92,10 @@ export default function (server: Server, ctx: AppContext) { subjectStatus: result.subjectStatus, }) + if (isModEventDivert(event) && subject.isRecord()) { + await moderationTxn.divertBlobs(subject) + } + if (subject.isRepo()) { if (isTakedownEvent) { const isSuspend = !!result.event.durationInHours diff --git a/packages/ozone/src/config/config.ts b/packages/ozone/src/config/config.ts index 60f103fb6d7..490ce398b8b 100644 --- a/packages/ozone/src/config/config.ts +++ b/packages/ozone/src/config/config.ts @@ -43,10 +43,13 @@ export const envToCfg = (env: OzoneEnvironment): OzoneConfig => { plcUrl: env.didPlcUrl, } - const blobReportServiceCfg = { - url: env.blobReportServiceUrl, - authToken: env.blobReportServiceAuthToken, - } + const blobReportServiceCfg = + env.blobReportServiceUrl && env.blobReportServiceAuthToken + ? { + url: env.blobReportServiceUrl, + authToken: env.blobReportServiceAuthToken, + } + : undefined return { service: serviceCfg, @@ -64,7 +67,7 @@ export type OzoneConfig = { appview: AppviewConfig pds: PdsConfig | null identity: IdentityConfig - blobReportService: BlobReportServiceConfig + blobReportService?: BlobReportServiceConfig } export type ServiceConfig = { @@ -75,8 +78,8 @@ export type ServiceConfig = { } export type BlobReportServiceConfig = { - url?: string - authToken?: string + url: string + authToken: string } export type DatabaseConfig = { diff --git a/packages/ozone/src/context.ts b/packages/ozone/src/context.ts index 9018935d222..2d451a8a2fa 100644 --- a/packages/ozone/src/context.ts +++ b/packages/ozone/src/context.ts @@ -65,19 +65,20 @@ export class AppContext { cfg.appview.did ? createAuthHeaders(cfg.appview.did) : undefined const backgroundQueue = new BackgroundQueue(db) + const blobDiverter = cfg.blobReportService + ? new BlobDiverter(db, { + idResolver, + serviceConfig: cfg.blobReportService, + }) + : undefined const eventPusher = new EventPusher(db, createAuthHeaders, { appview: cfg.appview, pds: cfg.pds ?? undefined, + blobDiverter, }) - const blobDiverter = new BlobDiverter(db, { - idResolver, - serviceConfig: cfg.blobReportService, - }) - const modService = ModerationService.creator( backgroundQueue, eventPusher, - blobDiverter, appviewAgent, appviewAuth, cfg.service.did, diff --git a/packages/ozone/src/daemon/blob-diverter.ts b/packages/ozone/src/daemon/blob-diverter.ts index d8be3fcddf3..b49c1fe1796 100644 --- a/packages/ozone/src/daemon/blob-diverter.ts +++ b/packages/ozone/src/daemon/blob-diverter.ts @@ -1,5 +1,4 @@ import { - SECOND, VerifyCidTransform, forwardStreamErrors, getPdsEndpoint, @@ -14,18 +13,7 @@ import { retryHttp } from '../util' import { dbLogger } from '../logger' import { BlobReportServiceConfig } from '../config' -type PollState = { - timer?: NodeJS.Timer - promise: Promise -} - export class BlobDiverter { - destroyed = false - - pollState: PollState = { - promise: Promise.resolve(), - } - serviceConfig: BlobReportServiceConfig idResolver: IdResolver @@ -40,48 +28,6 @@ export class BlobDiverter { this.idResolver = services.idResolver } - start() { - this.poll(this.pollState, () => this.divertBlob()) - } - - poll(state: PollState, fn: () => Promise) { - if (this.destroyed) return - state.promise = fn() - .catch((err) => { - dbLogger.error({ err }, 'blob divert failed') - }) - .finally(() => { - state.timer = setTimeout(() => this.poll(state, fn), 30 * SECOND) - }) - } - - async processAll() { - await Promise.all([this.divertBlob(), this.pollState.promise]) - } - - async destroy() { - this.destroyed = true - const destroyState = (state: PollState) => { - if (state.timer) { - clearTimeout(state.timer) - } - return state.promise - } - await destroyState(this.pollState) - } - - async divertBlob() { - const toPush = await this.db.db - .selectFrom('blob_divert_event') - .select('id') - .forUpdate() - .skipLocked() - .where('divertedAt', 'is', null) - .where('attempts', '<', 10) - .execute() - await Promise.all(toPush.map((evt) => this.attemptBlobDivert(evt.id))) - } - private async getBlob({ pds, did, @@ -111,18 +57,15 @@ export class BlobDiverter { } } - private async uploadBlob( - { - imageStream, - contentType, - }: { imageStream: Readable; contentType: string }, - { subjectDid, subjectUri }: { subjectDid: string; subjectUri: string }, - ) { - if (!this.serviceConfig.authToken || !this.serviceConfig.url) { - return false - } - - const url = `${this.serviceConfig.url}?did=${subjectDid}&uri=${subjectUri}` + async sendImage({ + url, + imageStream, + contentType, + }: { + url: string + imageStream: Readable + contentType: string + }) { const result = await axios(url, { method: 'POST', data: imageStream, @@ -135,20 +78,38 @@ export class BlobDiverter { return result.status === 200 } - private async uploadBlobOnService({ + private async uploadBlob( + { + imageStream, + contentType, + }: { imageStream: Readable; contentType: string }, + { + subjectDid, + subjectUri, + }: { subjectDid: string; subjectUri: string | null }, + ) { + const url = new URL(this.serviceConfig.url) + url.searchParams.set('did', subjectDid) + if (subjectUri) url.searchParams.set('uri', subjectUri) + const result = await this.sendImage({ + url: url.toString(), + imageStream, + contentType, + }) + + return result + } + + async uploadBlobOnService({ subjectDid, subjectUri, subjectBlobCid, }: { subjectDid: string - subjectUri: string + subjectUri: string | null subjectBlobCid: string }): Promise { try { - if (!this.serviceConfig.authToken || !this.serviceConfig.url) { - throw new Error('Blob divert service not configured') - } - const didDoc = await this.idResolver.did.resolve(subjectDid) if (!didDoc) { @@ -161,16 +122,18 @@ export class BlobDiverter { throw new Error('Error resolving PDS') } - const { imageStream, contentType } = await retryHttp(() => - this.getBlob({ pds, did: subjectDid, cid: subjectBlobCid }), - ) - - const uploadResult = await retryHttp(() => - this.uploadBlob( + // attempt to download and upload within the same retry block since the imageStream is not reusable + const uploadResult = await retryHttp(async () => { + const { imageStream, contentType } = await this.getBlob({ + pds, + did: subjectDid, + cid: subjectBlobCid, + }) + return this.uploadBlob( { imageStream, contentType }, { subjectDid, subjectUri }, - ), - ) + ) + }) return uploadResult } catch (err) { @@ -178,52 +141,4 @@ export class BlobDiverter { return false } } - - async attemptBlobDivert(id: number) { - await this.db.transaction(async (dbTxn) => { - const evt = await dbTxn.db - .selectFrom('blob_divert_event') - .selectAll() - .forUpdate() - .skipLocked() - .where('id', '=', id) - .where('divertedAt', 'is', null) - .executeTakeFirst() - if (!evt) return - - const succeeded = await this.uploadBlobOnService(evt) - await dbTxn.db - .updateTable('blob_divert_event') - .set( - succeeded - ? { divertedAt: new Date() } - : { - lastAttempted: new Date(), - attempts: (evt.attempts ?? 0) + 1, - }, - ) - .where('subjectDid', '=', evt.subjectDid) - .where('subjectBlobCid', '=', evt.subjectBlobCid) - .execute() - }) - } - - async logDivertEvent(values: { - subjectDid: string - subjectUri: string - subjectBlobCid: string - }) { - return this.db.db - .insertInto('blob_divert_event') - .values(values) - .onConflict((oc) => - oc.columns(['subjectDid', 'subjectBlobCid']).doUpdateSet({ - divertedAt: null, - attempts: 0, - lastAttempted: null, - }), - ) - .returning('id') - .execute() - } } diff --git a/packages/ozone/src/daemon/context.ts b/packages/ozone/src/daemon/context.ts index 4ad53812388..266e7d4143a 100644 --- a/packages/ozone/src/daemon/context.ts +++ b/packages/ozone/src/daemon/context.ts @@ -16,7 +16,7 @@ export type DaemonContextOptions = { modService: ModerationServiceCreator signingKey: Keypair eventPusher: EventPusher - blobDiverter: BlobDiverter + blobDiverter?: BlobDiverter eventReverser: EventReverser } @@ -49,19 +49,21 @@ export class DaemonContext { const appviewAuth = async () => cfg.appview.did ? createAuthHeaders(cfg.appview.did) : undefined + const blobDiverter = cfg.blobReportService + ? new BlobDiverter(db, { + idResolver, + serviceConfig: cfg.blobReportService, + }) + : undefined const eventPusher = new EventPusher(db, createAuthHeaders, { appview: cfg.appview, pds: cfg.pds ?? undefined, - }) - const blobDiverter = new BlobDiverter(db, { - idResolver, - serviceConfig: cfg.blobReportService, + blobDiverter, }) const backgroundQueue = new BackgroundQueue(db) const modService = ModerationService.creator( backgroundQueue, eventPusher, - blobDiverter, appviewAgent, appviewAuth, cfg.service.did, @@ -96,7 +98,7 @@ export class DaemonContext { return this.opts.eventPusher } - get blobDiverter(): BlobDiverter { + get blobDiverter(): BlobDiverter | undefined { return this.opts.blobDiverter } diff --git a/packages/ozone/src/daemon/event-pusher.ts b/packages/ozone/src/daemon/event-pusher.ts index faaee4529ed..c09b9be1e0f 100644 --- a/packages/ozone/src/daemon/event-pusher.ts +++ b/packages/ozone/src/daemon/event-pusher.ts @@ -5,6 +5,9 @@ import { retryHttp } from '../util' import { dbLogger } from '../logger' import { InputSchema } from '../lexicon/types/com/atproto/admin/updateSubjectStatus' import assert from 'assert' +import { BlobPushEvent } from '../db/schema/blob_push_event' +import { Insertable, Selectable } from 'kysely' +import { BlobDiverter } from './blob-diverter' type EventSubject = InputSchema['subject'] @@ -39,6 +42,7 @@ export class EventPusher { appview: Service | undefined pds: Service | undefined + blobDiverter: BlobDiverter | undefined constructor( public db: Database, @@ -52,8 +56,10 @@ export class EventPusher { url: string did: string } + blobDiverter }, ) { + this.blobDiverter = services.blobDiverter if (services.appview) { this.appview = { agent: new AtpAgent({ service: services.appview.url }), @@ -265,32 +271,67 @@ export class EventPusher { .executeTakeFirst() if (!evt) return - const service = evt.eventType === 'pds_takedown' ? this.pds : this.appview - assert(service) - const subject = { - $type: 'com.atproto.admin.defs#repoBlobRef', - did: evt.subjectDid, - cid: evt.subjectBlobCid, - } - const succeeded = await this.updateSubjectOnService( - service, - subject, - evt.takedownRef, - ) - await dbTxn.db - .updateTable('blob_push_event') - .set( - succeeded - ? { confirmedAt: new Date() } - : { - lastAttempted: new Date(), - attempts: evt.attempts ?? 0 + 1, - }, + let succeeded = false + if (evt.eventType === 'blob_divert') { + succeeded = await (this.blobDiverter + ? this.blobDiverter.uploadBlobOnService(evt) + : Promise.resolve(false)) + } else { + const service = + evt.eventType === 'pds_takedown' ? this.pds : this.appview + assert(service) + const subject = { + $type: 'com.atproto.admin.defs#repoBlobRef', + did: evt.subjectDid, + cid: evt.subjectBlobCid, + } + succeeded = await this.updateSubjectOnService( + service, + subject, + evt.takedownRef, ) - .where('subjectDid', '=', evt.subjectDid) - .where('subjectBlobCid', '=', evt.subjectBlobCid) - .where('eventType', '=', evt.eventType) - .execute() + } + await this.markBlobEventAttempt(dbTxn, evt, succeeded) }) } + + async markBlobEventAttempt( + dbTxn: Database, + event: Selectable, + succeeded: boolean, + ) { + await dbTxn.db + .updateTable('blob_push_event') + .set( + succeeded + ? { confirmedAt: new Date() } + : { + lastAttempted: new Date(), + attempts: (event.attempts ?? 0) + 1, + }, + ) + .where('subjectDid', '=', event.subjectDid) + .where('subjectBlobCid', '=', event.subjectBlobCid) + .where('eventType', '=', event.eventType) + .execute() + } + + async logBlobPushEvent( + blobValues: Insertable[], + takedownRef?: string | null, + ) { + return this.db.db + .insertInto('blob_push_event') + .values(blobValues) + .onConflict((oc) => + oc.columns(['subjectDid', 'subjectBlobCid', 'eventType']).doUpdateSet({ + takedownRef, + confirmedAt: null, + attempts: 0, + lastAttempted: null, + }), + ) + .returning('id') + .execute() + } } diff --git a/packages/ozone/src/daemon/index.ts b/packages/ozone/src/daemon/index.ts index 8f1d67273f2..501b8caad5c 100644 --- a/packages/ozone/src/daemon/index.ts +++ b/packages/ozone/src/daemon/index.ts @@ -20,18 +20,15 @@ export class OzoneDaemon { async start() { this.ctx.eventPusher.start() this.ctx.eventReverser.start() - this.ctx.blobDiverter.start() } async processAll() { await this.ctx.eventPusher.processAll() - await this.ctx.blobDiverter.processAll() } async destroy() { await this.ctx.eventReverser.destroy() await this.ctx.eventPusher.destroy() - await this.ctx.blobDiverter.destroy() await this.ctx.db.close() } } diff --git a/packages/ozone/src/db/migrations/20240228T153256972Z-blob-divert-event.ts b/packages/ozone/src/db/migrations/20240228T153256972Z-blob-divert-event.ts deleted file mode 100644 index 933cd3df4b5..00000000000 --- a/packages/ozone/src/db/migrations/20240228T153256972Z-blob-divert-event.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Kysely } from 'kysely' - -export async function up(db: Kysely): Promise { - await db.schema - .createTable('blob_divert_event') - .addColumn('id', 'serial', (col) => col.primaryKey()) - .addColumn('subjectDid', 'varchar', (col) => col.notNull()) - .addColumn('subjectBlobCid', 'varchar', (col) => col.notNull()) - .addColumn('subjectUri', 'varchar') - .addColumn('divertedAt', 'timestamptz') - .addColumn('lastAttempted', 'timestamptz') - .addColumn('attempts', 'integer', (col) => col.notNull().defaultTo(0)) - .addUniqueConstraint('blob_divert_event_unique_evt', [ - 'subjectDid', - 'subjectBlobCid', - ]) - .execute() - await db.schema - .createIndex('blob_divert_unique_idx') - .on('blob_divert_event') - .columns(['divertedAt', 'attempts']) - .execute() -} - -export async function down(db: Kysely): Promise { - await db.schema.dropTable('blob_divert_event').execute() -} diff --git a/packages/ozone/src/db/migrations/index.ts b/packages/ozone/src/db/migrations/index.ts index 4ccaa5791f5..1a823f860c5 100644 --- a/packages/ozone/src/db/migrations/index.ts +++ b/packages/ozone/src/db/migrations/index.ts @@ -6,4 +6,3 @@ export * as _20231219T205730722Z from './20231219T205730722Z-init' export * as _20240116T085607200Z from './20240116T085607200Z-communication-template' export * as _20240201T051104136Z from './20240201T051104136Z-mod-event-blobs' export * as _20240208T213404429Z from './20240208T213404429Z-add-tags-column-to-moderation-subject' -export * as _20240228T153256972Z from './20240228T153256972Z-blob-divert-event' diff --git a/packages/ozone/src/db/schema/blob_push_event.ts b/packages/ozone/src/db/schema/blob_push_event.ts index f38649e675c..d4cd8e8b482 100644 --- a/packages/ozone/src/db/schema/blob_push_event.ts +++ b/packages/ozone/src/db/schema/blob_push_event.ts @@ -2,7 +2,10 @@ import { Generated } from 'kysely' export const eventTableName = 'blob_push_event' -export type BlobPushEventType = 'pds_takedown' | 'appview_takedown' +export type BlobPushEventType = + | 'pds_takedown' + | 'appview_takedown' + | 'blob_divert' export interface BlobPushEvent { id: Generated diff --git a/packages/ozone/src/mod-service/index.ts b/packages/ozone/src/mod-service/index.ts index 988703f70e3..e31f29e10a8 100644 --- a/packages/ozone/src/mod-service/index.ts +++ b/packages/ozone/src/mod-service/index.ts @@ -15,7 +15,6 @@ import { isModEventTag, RepoRef, RepoBlobRef, - isModEventDivert, } from '../lexicon/types/com/atproto/admin/defs' import { adjustModerationSubjectStatus, @@ -44,7 +43,6 @@ import { BlobPushEvent } from '../db/schema/blob_push_event' import { BackgroundQueue } from '../background' import { EventPusher } from '../daemon' import { jsonb } from '../db/types' -import { BlobDiverter } from '../daemon/blob-diverter' import { LabelChannel } from '../db/schema/label' export type ModerationServiceCreator = (db: Database) => ModerationService @@ -54,7 +52,6 @@ export class ModerationService { public db: Database, public backgroundQueue: BackgroundQueue, public eventPusher: EventPusher, - public blobDiverter: BlobDiverter, public appviewAgent: AtpAgent, private appviewAuth: AppviewAuth, public serverDid: string, @@ -63,7 +60,6 @@ export class ModerationService { static creator( backgroundQueue: BackgroundQueue, eventPusher: EventPusher, - blobDiverter: BlobDiverter, appviewAgent: AtpAgent, appviewAuth: AppviewAuth, serverDid: string, @@ -73,7 +69,6 @@ export class ModerationService { db, backgroundQueue, eventPusher, - blobDiverter, appviewAgent, appviewAuth, serverDid, @@ -327,33 +322,6 @@ export class ModerationService { subject.blobCids, ) - if ( - isModEventDivert(event) && - subjectInfo.subjectUri && - subjectInfo.subjectBlobCids?.length - ) { - const blobDiverts = await Promise.all( - subjectInfo.subjectBlobCids?.map((subjectBlobCid) => - this.blobDiverter.logDivertEvent({ - subjectDid: subjectInfo.subjectDid, - subjectBlobCid: subjectBlobCid, - // TODO: Check is done above already - // @ts-ignore - subjectUri: subjectInfo.subjectUri, - }), - ), - ) - this.db.onCommit(() => { - this.backgroundQueue.add(async () => { - await Promise.all( - blobDiverts.map((divert) => - this.blobDiverter.attemptBlobDivert(divert[0].id), - ), - ) - }) - }) - } - return { event: modEvent, subjectStatus } } @@ -569,27 +537,17 @@ export class ModerationService { for (const cid of blobCids) { blobValues.push({ eventType, + takedownRef, subjectDid: subject.did, + subjectUri: subject.uri || null, subjectBlobCid: cid.toString(), - takedownRef, }) } } - const blobEvts = await this.db.db - .insertInto('blob_push_event') - .values(blobValues) - .onConflict((oc) => - oc - .columns(['subjectDid', 'subjectBlobCid', 'eventType']) - .doUpdateSet({ - takedownRef, - confirmedAt: null, - attempts: 0, - lastAttempted: null, - }), - ) - .returning('id') - .execute() + const blobEvts = await this.eventPusher.logBlobPushEvent( + blobValues, + takedownRef, + ) this.db.onCommit(() => { this.backgroundQueue.add(async () => { @@ -601,6 +559,29 @@ export class ModerationService { } } + async divertBlobs(subject: RecordSubject) { + const subjectInfo = subject.info() + + const blobDiverts = await this.eventPusher.logBlobPushEvent( + subjectInfo.subjectBlobCids.map((subjectBlobCid) => ({ + subjectDid: subjectInfo.subjectDid, + subjectBlobCid: subjectBlobCid, + subjectUri: subjectInfo.subjectUri, + eventType: 'blob_divert', + })), + ) + + this.db.onCommit(() => { + this.backgroundQueue.add(async () => { + await Promise.all( + blobDiverts.map((divert) => + this.eventPusher.attemptBlobEvent(divert[0].id), + ), + ) + }) + }) + } + async reverseTakedownRecord(subject: RecordSubject) { this.db.assertTransaction() const labels: string[] = [UNSPECCED_TAKEDOWN_LABEL] diff --git a/packages/ozone/tests/blob-divert.test.ts b/packages/ozone/tests/blob-divert.test.ts index 193e9518b21..6d57e6d0473 100644 --- a/packages/ozone/tests/blob-divert.test.ts +++ b/packages/ozone/tests/blob-divert.test.ts @@ -1,10 +1,6 @@ import { SeedClient, TestNetwork, basicSeed } from '@atproto/dev-env' import AtpAgent from '@atproto/api' -import nock from 'nock' - -const BLOB_DIVERT_SERVICE_HOST = 'http://blob-report.com' -const BLOB_DIVERT_SERVICE_PATH = '/xrpc/com.atproto.unspecced.reportBlob' -const BLOB_DIVERT_SERVICE_AUTH_TOKEN = 'Basic test' +import { BlobDiverter } from '../src/daemon' describe('blob divert', () => { let network: TestNetwork @@ -15,8 +11,8 @@ describe('blob divert', () => { network = await TestNetwork.create({ dbPostgresSchema: 'ozone_blob_divert_test', ozone: { - blobReportServiceUrl: `${BLOB_DIVERT_SERVICE_HOST}${BLOB_DIVERT_SERVICE_PATH}`, - blobReportServiceAuthToken: BLOB_DIVERT_SERVICE_AUTH_TOKEN, + blobReportServiceUrl: `https://blob-report.com`, + blobReportServiceAuthToken: 'test-auth-token', }, }) agent = network.pds.getClient() @@ -29,25 +25,22 @@ describe('blob divert', () => { await network.close() }) - afterEach(() => { - nock.cleanAll() - }) - - const mockReportServiceResponse = ( - status: number, - data: Record, - ) => - nock(BLOB_DIVERT_SERVICE_HOST) - .persist() - .post(BLOB_DIVERT_SERVICE_PATH, () => true) - .query(true) - .reply(status, data) + const mockReportServiceResponse = (result: boolean) => { + return jest + .spyOn(BlobDiverter.prototype, 'sendImage') + .mockImplementation(async () => { + return result + }) + // nock(BLOB_DIVERT_SERVICE_HOST) + // .persist() + // .post(BLOB_DIVERT_SERVICE_PATH, () => true) + // .query(true) + // .reply(status, data) + } it('fails and keeps attempt count when report service fails to accept upload.', async () => { - // Simulate failure to accept upload - const reportServiceRequest = mockReportServiceResponse(401, { - success: false, - }) + // Simulate failure to fail upload + const reportServiceRequest = mockReportServiceResponse(false) await agent.api.com.atproto.admin.emitModerationEvent( { @@ -71,31 +64,28 @@ describe('blob divert', () => { await network.ozone.processAll() const divertEvents = await network.ozone.ctx.db.db - .selectFrom('blob_divert_event') + .selectFrom('blob_push_event') .selectAll() .execute() expect(divertEvents[0].attempts).toBeGreaterThan(0) expect(divertEvents[1].attempts).toBeGreaterThan(0) - reportServiceRequest.done() + expect(reportServiceRequest).toHaveBeenCalled() }) it('sends blobs to configured divert service and marks divert date', async () => { // Simulate failure to accept upload - const reportServiceRequest = mockReportServiceResponse(200, { - success: true, - }) + const reportServiceRequest = mockReportServiceResponse(true) await network.ozone.processAll() const divertEvents = await network.ozone.ctx.db.db - .selectFrom('blob_divert_event') + .selectFrom('blob_push_event') .selectAll() .execute() - expect(divertEvents[0].divertedAt).toBeTruthy() - expect(divertEvents[1].divertedAt).toBeTruthy() - reportServiceRequest.done() - reportServiceRequest.persist(false) + expect(divertEvents[0].confirmedAt).toBeTruthy() + expect(divertEvents[1].confirmedAt).toBeTruthy() + expect(reportServiceRequest).toHaveBeenCalled() }) }) From 54183d8c696cfa7a25151083a9e4087580788280 Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Wed, 6 Mar 2024 00:30:21 +0000 Subject: [PATCH 12/20] :sparkles: Bring back missing lines in pnpm-lock --- pnpm-lock.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a94b5cd5c58..973f8935e50 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,5 +1,9 @@ lockfileVersion: '6.0' +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + importers: .: From 09f8137443620a3e161c3199421821c61e07bbd4 Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Wed, 6 Mar 2024 00:32:48 +0000 Subject: [PATCH 13/20] :hammer: Rebuild? --- pnpm-lock.yaml | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 973f8935e50..df7dcbcd2f9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,9 +1,5 @@ lockfileVersion: '6.0' -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - importers: .: @@ -660,9 +656,6 @@ importers: axios: specifier: ^0.27.2 version: 0.27.2 - nock: - specifier: 14.0.0-beta.4 - version: 14.0.0-beta.4 packages/pds: dependencies: @@ -9436,10 +9429,6 @@ packages: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} dev: true - /json-stringify-safe@5.0.1: - resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} - dev: true - /json5@2.2.3: resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} engines: {node: '>=6'} @@ -9934,17 +9923,6 @@ packages: /neo-async@2.6.2: resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==} - /nock@14.0.0-beta.4: - resolution: {integrity: sha512-N9GIOnNFas/TtdCQpavpi6A6SyVVInkD/vrUCF2u51vlE2wSnqfPifVli6xSX8l6Lz/3sdSwPusE9n3KPDDh0g==} - engines: {node: '>= 18'} - dependencies: - debug: 4.3.4 - json-stringify-safe: 5.0.1 - propagate: 2.0.1 - transitivePeerDependencies: - - supports-color - dev: true - /node-abi@3.47.0: resolution: {integrity: sha512-2s6B2CWZM//kPgwnuI0KrYwNjfdByE25zvAaEpq9IH4zcNsarH8Ihu/UuX6XMPEogDAxkuUFeZn60pXNHAqn3A==} engines: {node: '>=10'} @@ -10561,11 +10539,6 @@ packages: sisteransi: 1.0.5 dev: true - /propagate@2.0.1: - resolution: {integrity: sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==} - engines: {node: '>= 8'} - dev: true - /protobufjs@7.2.5: resolution: {integrity: sha512-gGXRSXvxQ7UiPgfw8gevrfRWcTlSbOFg+p/N+JVJEK5VhueL2miT6qTymqAmjr1Q5WbOCyJbyrk6JfWKwlFn6A==} engines: {node: '>=12.0.0'} From 2bd994747d7c50864099187a094681229528a1ea Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Wed, 6 Mar 2024 00:34:26 +0000 Subject: [PATCH 14/20] :rotating_light: Formatting --- packages/ozone/src/daemon/event-pusher.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ozone/src/daemon/event-pusher.ts b/packages/ozone/src/daemon/event-pusher.ts index 0848a381978..f82a8329f1f 100644 --- a/packages/ozone/src/daemon/event-pusher.ts +++ b/packages/ozone/src/daemon/event-pusher.ts @@ -331,7 +331,13 @@ export class EventPusher { lastAttempted: null, }), ) - .returning(['id', 'subjectDid', 'subjectUri', 'subjectBlobCid', 'eventType']) + .returning([ + 'id', + 'subjectDid', + 'subjectUri', + 'subjectBlobCid', + 'eventType', + ]) .execute() } } From 96e2870e926baaf9ecf9e787b450f686a620426e Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Thu, 7 Mar 2024 00:03:22 +0000 Subject: [PATCH 15/20] :recycle: Refactor to divert blob sync --- .../src/api/admin/emitModerationEvent.ts | 281 ++++++++++-------- packages/ozone/src/auth-verifier.ts | 4 +- packages/ozone/src/context.ts | 7 +- packages/ozone/src/daemon/blob-diverter.ts | 69 +++-- packages/ozone/src/daemon/context.ts | 14 - packages/ozone/src/daemon/event-pusher.ts | 34 +-- packages/ozone/src/mod-service/index.ts | 23 -- .../__snapshots__/blob-divert.test.ts.snap | 22 ++ packages/ozone/tests/blob-divert.test.ts | 62 ++-- .../pds/src/api/com/atproto/sync/getBlob.ts | 4 + 10 files changed, 275 insertions(+), 245 deletions(-) create mode 100644 packages/ozone/tests/__snapshots__/blob-divert.test.ts.snap diff --git a/packages/ozone/src/api/admin/emitModerationEvent.ts b/packages/ozone/src/api/admin/emitModerationEvent.ts index 15021deaf0d..89de1d66569 100644 --- a/packages/ozone/src/api/admin/emitModerationEvent.ts +++ b/packages/ozone/src/api/admin/emitModerationEvent.ts @@ -8,155 +8,198 @@ import { isModEventReverseTakedown, isModEventTakedown, } from '../../lexicon/types/com/atproto/admin/defs' +import { HandlerInput } from '../../lexicon/types/com/atproto/admin/emitModerationEvent' import { subjectFromInput } from '../../mod-service/subject' import { ModerationLangService } from '../../mod-service/lang' import { retryHttp } from '../../util' +import { ModeratorOutput, RoleOutput } from '../../auth-verifier' + +const handleModerationEvent = async ({ + ctx, + input, + auth, +}: { + ctx: AppContext + input: HandlerInput + auth: ModeratorOutput | RoleOutput +}) => { + { + const access = auth.credentials + const db = ctx.db + const moderationService = ctx.modService(db) + const { createdBy, event } = input.body + const isTakedownEvent = isModEventTakedown(event) + const isReverseTakedownEvent = isModEventReverseTakedown(event) + const isLabelEvent = isModEventLabel(event) + const subject = subjectFromInput( + input.body.subject, + input.body.subjectBlobCids, + ) + + // apply access rules + + // if less than moderator access then can only take ack and escalation actions + if (isTakedownEvent || isReverseTakedownEvent) { + if (!access.isModerator) { + throw new AuthRequiredError( + 'Must be a full moderator to take this type of action', + ) + } -export default function (server: Server, ctx: AppContext) { - server.com.atproto.admin.emitModerationEvent({ - auth: ctx.authVerifier.modOrRole, - handler: async ({ input, auth }) => { - const access = auth.credentials - const db = ctx.db - const moderationService = ctx.modService(db) - const { createdBy, event } = input.body - const isTakedownEvent = isModEventTakedown(event) - const isReverseTakedownEvent = isModEventReverseTakedown(event) - const isLabelEvent = isModEventLabel(event) - const subject = subjectFromInput( - input.body.subject, - input.body.subjectBlobCids, - ) + // Non admins should not be able to take down feed generators + if ( + !access.isAdmin && + subject.recordPath?.includes('app.bsky.feed.generator/') + ) { + throw new AuthRequiredError( + 'Must be a full admin to take this type of action on feed generators', + ) + } + } + // if less than moderator access then can not apply labels + if (!access.isModerator && isLabelEvent) { + throw new AuthRequiredError('Must be a full moderator to label content') + } - // apply access rules + if (isLabelEvent) { + validateLabels([ + ...(event.createLabelVals ?? []), + ...(event.negateLabelVals ?? []), + ]) + } - // if less than moderator access then can only take ack and escalation actions - if (isTakedownEvent || isReverseTakedownEvent) { - if (!access.isModerator) { - throw new AuthRequiredError( - 'Must be a full moderator to take this type of action', - ) - } + if (isTakedownEvent || isReverseTakedownEvent) { + const status = await moderationService.getStatus(subject) - // Non admins should not be able to take down feed generators - if ( - !access.isAdmin && - subject.recordPath?.includes('app.bsky.feed.generator/') - ) { - throw new AuthRequiredError( - 'Must be a full admin to take this type of action on feed generators', - ) - } - } - // if less than moderator access then can not apply labels - if (!access.isModerator && isLabelEvent) { - throw new AuthRequiredError('Must be a full moderator to label content') + if (status?.takendown && isTakedownEvent) { + throw new InvalidRequestError(`Subject is already taken down`) } - if (isLabelEvent) { - validateLabels([ - ...(event.createLabelVals ?? []), - ...(event.negateLabelVals ?? []), - ]) + if (!status?.takendown && isReverseTakedownEvent) { + throw new InvalidRequestError(`Subject is not taken down`) } - if (isTakedownEvent || isReverseTakedownEvent) { - const status = await moderationService.getStatus(subject) - - if (status?.takendown && isTakedownEvent) { - throw new InvalidRequestError(`Subject is already taken down`) - } - - if (!status?.takendown && isReverseTakedownEvent) { - throw new InvalidRequestError(`Subject is not taken down`) - } + if (status?.takendown && isReverseTakedownEvent && subject.isRecord()) { + // due to the way blob status is modeled, we should reverse takedown on all + // blobs for the record being restored, which aren't taken down on another record. + subject.blobCids = status.blobCids ?? [] + } + } - if (status?.takendown && isReverseTakedownEvent && subject.isRecord()) { - // due to the way blob status is modeled, we should reverse takedown on all - // blobs for the record being restored, which aren't taken down on another record. - subject.blobCids = status.blobCids ?? [] - } + if (isModEventEmail(event) && event.content) { + // sending email prior to logging the event to avoid a long transaction below + if (!subject.isRepo()) { + throw new InvalidRequestError( + 'Email can only be sent to a repo subject', + ) } + const { content, subjectLine } = event + await retryHttp(() => + ctx.modService(db).sendEmail({ + subject: subjectLine, + content, + recipientDid: subject.did, + }), + ) + } - if (isModEventEmail(event) && event.content) { - // sending email prior to logging the event to avoid a long transaction below - if (!subject.isRepo()) { - throw new InvalidRequestError( - 'Email can only be sent to a repo subject', - ) - } - const { content, subjectLine } = event - await retryHttp(() => - ctx.modService(db).sendEmail({ - subject: subjectLine, - content, - recipientDid: subject.did, - }), + if (isModEventDivert(event) && subject.isRecord()) { + if (!ctx.blobDiverter) { + throw new InvalidRequestError( + 'BlobDiverter not configured for this service', ) } + await ctx.blobDiverter.uploadBlobOnService(subject.info()) + } - const moderationEvent = await db.transaction(async (dbTxn) => { - const moderationTxn = ctx.modService(dbTxn) + const moderationEvent = await db.transaction(async (dbTxn) => { + const moderationTxn = ctx.modService(dbTxn) - const result = await moderationTxn.logEvent({ - event, - subject, - createdBy, - }) + const result = await moderationTxn.logEvent({ + event, + subject, + createdBy, + }) - const moderationLangService = new ModerationLangService(moderationTxn) - await moderationLangService.tagSubjectWithLang({ - subject, - createdBy: ctx.cfg.service.did, - subjectStatus: result.subjectStatus, - }) + const moderationLangService = new ModerationLangService(moderationTxn) + await moderationLangService.tagSubjectWithLang({ + subject, + createdBy: ctx.cfg.service.did, + subjectStatus: result.subjectStatus, + }) - if (isModEventDivert(event) && subject.isRecord()) { - await moderationTxn.divertBlobs(subject) + if (subject.isRepo()) { + if (isTakedownEvent) { + const isSuspend = !!result.event.durationInHours + await moderationTxn.takedownRepo(subject, result.event.id, isSuspend) + } else if (isReverseTakedownEvent) { + await moderationTxn.reverseTakedownRepo(subject) } + } - if (subject.isRepo()) { - if (isTakedownEvent) { - const isSuspend = !!result.event.durationInHours - await moderationTxn.takedownRepo( - subject, - result.event.id, - isSuspend, - ) - } else if (isReverseTakedownEvent) { - await moderationTxn.reverseTakedownRepo(subject) - } + if (subject.isRecord()) { + if (isTakedownEvent) { + await moderationTxn.takedownRecord(subject, result.event.id) + } else if (isReverseTakedownEvent) { + await moderationTxn.reverseTakedownRecord(subject) } + } - if (subject.isRecord()) { - if (isTakedownEvent) { - await moderationTxn.takedownRecord(subject, result.event.id) - } else if (isReverseTakedownEvent) { - await moderationTxn.reverseTakedownRecord(subject) - } - } + if (isLabelEvent) { + await moderationTxn.formatAndCreateLabels( + result.event.subjectUri ?? result.event.subjectDid, + result.event.subjectCid, + { + create: result.event.createLabelVals?.length + ? result.event.createLabelVals.split(' ') + : undefined, + negate: result.event.negateLabelVals?.length + ? result.event.negateLabelVals.split(' ') + : undefined, + }, + ) + } - if (isLabelEvent) { - await moderationTxn.formatAndCreateLabels( - result.event.subjectUri ?? result.event.subjectDid, - result.event.subjectCid, - { - create: result.event.createLabelVals?.length - ? result.event.createLabelVals.split(' ') - : undefined, - negate: result.event.negateLabelVals?.length - ? result.event.negateLabelVals.split(' ') - : undefined, - }, - ) - } + return result.event + }) + + return moderationService.views.formatEvent(moderationEvent) + } +} - return result.event +export default function (server: Server, ctx: AppContext) { + server.com.atproto.admin.emitModerationEvent({ + auth: ctx.authVerifier.modOrRole, + handler: async ({ input, auth }) => { + const moderationEvent = await handleModerationEvent({ + input, + auth, + ctx, }) + // On divert events, we need to automatically take down the blobs + if (isModEventDivert(input.body.event)) { + await handleModerationEvent({ + auth, + ctx, + input: { + ...input, + body: { + ...input.body, + event: { + ...input.body.event, + $type: 'com.atproto.admin.defs#modEventTakedown', + comment: + '[DIVERT_SIDE_EFFECT]: Automatically taking down after divert event', + }, + }, + }, + }) + } + return { encoding: 'application/json', - body: moderationService.views.formatEvent(moderationEvent), + body: moderationEvent, } }, }) diff --git a/packages/ozone/src/auth-verifier.ts b/packages/ozone/src/auth-verifier.ts index 48ca241e6ef..560f0192693 100644 --- a/packages/ozone/src/auth-verifier.ts +++ b/packages/ozone/src/auth-verifier.ts @@ -7,7 +7,7 @@ type ReqCtx = { req: express.Request } -type RoleOutput = { +export type RoleOutput = { credentials: { type: 'role' isAdmin: boolean @@ -16,7 +16,7 @@ type RoleOutput = { } } -type ModeratorOutput = { +export type ModeratorOutput = { credentials: { type: 'moderator' aud: string diff --git a/packages/ozone/src/context.ts b/packages/ozone/src/context.ts index 8450d30c6cf..71f1280a5e5 100644 --- a/packages/ozone/src/context.ts +++ b/packages/ozone/src/context.ts @@ -25,6 +25,7 @@ export type AppContextOptions = { communicationTemplateService: CommunicationTemplateServiceCreator appviewAgent: AtpAgent pdsAgent: AtpAgent | undefined + blobDiverter?: BlobDiverter signingKey: Keypair idResolver: IdResolver imgInvalidator?: ImageInvalidator @@ -75,7 +76,6 @@ export class AppContext { const eventPusher = new EventPusher(db, createAuthHeaders, { appview: cfg.appview, pds: cfg.pds ?? undefined, - blobDiverter, }) const modService = ModerationService.creator( cfg, @@ -116,6 +116,7 @@ export class AppContext { backgroundQueue, sequencer, authVerifier, + blobDiverter, ...(overrides ?? {}), }, secrets, @@ -142,6 +143,10 @@ export class AppContext { return this.opts.modService } + get blobDiverter(): BlobDiverter | undefined { + return this.opts.blobDiverter + } + get communicationTemplateService(): CommunicationTemplateServiceCreator { return this.opts.communicationTemplateService } diff --git a/packages/ozone/src/daemon/blob-diverter.ts b/packages/ozone/src/daemon/blob-diverter.ts index b49c1fe1796..7501ded1b45 100644 --- a/packages/ozone/src/daemon/blob-diverter.ts +++ b/packages/ozone/src/daemon/blob-diverter.ts @@ -103,42 +103,45 @@ export class BlobDiverter { async uploadBlobOnService({ subjectDid, subjectUri, - subjectBlobCid, + subjectBlobCids, }: { subjectDid: string - subjectUri: string | null - subjectBlobCid: string + subjectUri: string + subjectBlobCids: string[] }): Promise { - try { - const didDoc = await this.idResolver.did.resolve(subjectDid) - - if (!didDoc) { - throw new Error('Error resolving DID') - } - - const pds = getPdsEndpoint(didDoc) - - if (!pds) { - throw new Error('Error resolving PDS') - } - - // attempt to download and upload within the same retry block since the imageStream is not reusable - const uploadResult = await retryHttp(async () => { - const { imageStream, contentType } = await this.getBlob({ - pds, - did: subjectDid, - cid: subjectBlobCid, - }) - return this.uploadBlob( - { imageStream, contentType }, - { subjectDid, subjectUri }, - ) - }) - - return uploadResult - } catch (err) { - dbLogger.error({ err }, 'failed to upload diverted blob') - return false + const didDoc = await this.idResolver.did.resolve(subjectDid) + + if (!didDoc) { + throw new Error('Error resolving DID') } + + const pds = getPdsEndpoint(didDoc) + + if (!pds) { + throw new Error('Error resolving PDS') + } + + // attempt to download and upload within the same retry block since the imageStream is not reusable + const uploadResult = await Promise.all( + subjectBlobCids.map((cid) => + retryHttp(async () => { + const { imageStream, contentType } = await this.getBlob({ + pds, + cid, + did: subjectDid, + }) + return this.uploadBlob( + { imageStream, contentType }, + { subjectDid, subjectUri }, + ) + }), + ), + ) + + if (uploadResult.includes(false)) { + throw new Error(`Error uploading blob ${subjectUri}`) + } + + return true } } diff --git a/packages/ozone/src/daemon/context.ts b/packages/ozone/src/daemon/context.ts index 42f005f6b86..85b6a45884a 100644 --- a/packages/ozone/src/daemon/context.ts +++ b/packages/ozone/src/daemon/context.ts @@ -8,7 +8,6 @@ import { EventPusher } from './event-pusher' import { EventReverser } from './event-reverser' import { ModerationService, ModerationServiceCreator } from '../mod-service' import { BackgroundQueue } from '../background' -import { BlobDiverter } from './blob-diverter' export type DaemonContextOptions = { db: Database @@ -16,7 +15,6 @@ export type DaemonContextOptions = { modService: ModerationServiceCreator signingKey: Keypair eventPusher: EventPusher - blobDiverter?: BlobDiverter eventReverser: EventReverser } @@ -46,16 +44,9 @@ export class DaemonContext { keypair: signingKey, }) - const blobDiverter = cfg.blobReportService - ? new BlobDiverter(db, { - idResolver, - serviceConfig: cfg.blobReportService, - }) - : undefined const eventPusher = new EventPusher(db, createAuthHeaders, { appview: cfg.appview, pds: cfg.pds ?? undefined, - blobDiverter, }) const backgroundQueue = new BackgroundQueue(db) @@ -78,7 +69,6 @@ export class DaemonContext { modService, signingKey, eventPusher, - blobDiverter, eventReverser, ...(overrides ?? {}), }) @@ -100,10 +90,6 @@ export class DaemonContext { return this.opts.eventPusher } - get blobDiverter(): BlobDiverter | undefined { - return this.opts.blobDiverter - } - get eventReverser(): EventReverser { return this.opts.eventReverser } diff --git a/packages/ozone/src/daemon/event-pusher.ts b/packages/ozone/src/daemon/event-pusher.ts index f82a8329f1f..eee1f7a436b 100644 --- a/packages/ozone/src/daemon/event-pusher.ts +++ b/packages/ozone/src/daemon/event-pusher.ts @@ -7,7 +7,6 @@ import { InputSchema } from '../lexicon/types/com/atproto/admin/updateSubjectSta import assert from 'assert' import { BlobPushEvent } from '../db/schema/blob_push_event' import { Insertable, Selectable } from 'kysely' -import { BlobDiverter } from './blob-diverter' type EventSubject = InputSchema['subject'] @@ -42,7 +41,6 @@ export class EventPusher { appview: Service | undefined pds: Service | undefined - blobDiverter: BlobDiverter | undefined constructor( public db: Database, @@ -56,10 +54,8 @@ export class EventPusher { url: string did: string } - blobDiverter }, ) { - this.blobDiverter = services.blobDiverter if (services.appview) { this.appview = { agent: new AtpAgent({ service: services.appview.url }), @@ -271,26 +267,18 @@ export class EventPusher { .executeTakeFirst() if (!evt) return - let succeeded = false - if (evt.eventType === 'blob_divert') { - succeeded = await (this.blobDiverter - ? this.blobDiverter.uploadBlobOnService(evt) - : Promise.resolve(false)) - } else { - const service = - evt.eventType === 'pds_takedown' ? this.pds : this.appview - assert(service) - const subject = { - $type: 'com.atproto.admin.defs#repoBlobRef', - did: evt.subjectDid, - cid: evt.subjectBlobCid, - } - succeeded = await this.updateSubjectOnService( - service, - subject, - evt.takedownRef, - ) + const service = evt.eventType === 'pds_takedown' ? this.pds : this.appview + assert(service) + const subject = { + $type: 'com.atproto.admin.defs#repoBlobRef', + did: evt.subjectDid, + cid: evt.subjectBlobCid, } + const succeeded = await this.updateSubjectOnService( + service, + subject, + evt.takedownRef, + ) await this.markBlobEventAttempt(dbTxn, evt, succeeded) }) } diff --git a/packages/ozone/src/mod-service/index.ts b/packages/ozone/src/mod-service/index.ts index 7e61491ab24..7dff532b507 100644 --- a/packages/ozone/src/mod-service/index.ts +++ b/packages/ozone/src/mod-service/index.ts @@ -605,29 +605,6 @@ export class ModerationService { } } - async divertBlobs(subject: RecordSubject) { - const subjectInfo = subject.info() - - const blobDiverts = await this.eventPusher.logBlobPushEvent( - subjectInfo.subjectBlobCids.map((subjectBlobCid) => ({ - subjectDid: subjectInfo.subjectDid, - subjectBlobCid: subjectBlobCid, - subjectUri: subjectInfo.subjectUri, - eventType: 'blob_divert', - })), - ) - - this.db.onCommit(() => { - this.backgroundQueue.add(async () => { - await Promise.all( - blobDiverts.map((divert) => - this.eventPusher.attemptBlobEvent(divert[0].id), - ), - ) - }) - }) - } - async reverseTakedownRecord(subject: RecordSubject) { this.db.assertTransaction() const labels: string[] = [UNSPECCED_TAKEDOWN_LABEL] diff --git a/packages/ozone/tests/__snapshots__/blob-divert.test.ts.snap b/packages/ozone/tests/__snapshots__/blob-divert.test.ts.snap new file mode 100644 index 00000000000..30477a59748 --- /dev/null +++ b/packages/ozone/tests/__snapshots__/blob-divert.test.ts.snap @@ -0,0 +1,22 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`blob divert sends blobs to configured divert service and marks divert date 1`] = ` +Object { + "createdAt": "1970-01-01T00:00:00.000Z", + "createdBy": "user(0)", + "event": Object { + "$type": "com.atproto.admin.defs#modEventDivert", + "comment": "Diverting for test", + }, + "id": 1, + "subject": Object { + "$type": "com.atproto.repo.strongRef", + "cid": "cids(0)", + "uri": "record(0)", + }, + "subjectBlobCids": Array [ + "cids(1)", + "cids(2)", + ], +} +`; diff --git a/packages/ozone/tests/blob-divert.test.ts b/packages/ozone/tests/blob-divert.test.ts index 6d57e6d0473..1dcaf8349dd 100644 --- a/packages/ozone/tests/blob-divert.test.ts +++ b/packages/ozone/tests/blob-divert.test.ts @@ -1,6 +1,7 @@ import { SeedClient, TestNetwork, basicSeed } from '@atproto/dev-env' import AtpAgent from '@atproto/api' import { BlobDiverter } from '../src/daemon' +import { forSnapshot } from './_util' describe('blob divert', () => { let network: TestNetwork @@ -31,24 +32,18 @@ describe('blob divert', () => { .mockImplementation(async () => { return result }) - // nock(BLOB_DIVERT_SERVICE_HOST) - // .persist() - // .post(BLOB_DIVERT_SERVICE_PATH, () => true) - // .query(true) - // .reply(status, data) } - it('fails and keeps attempt count when report service fails to accept upload.', async () => { - // Simulate failure to fail upload - const reportServiceRequest = mockReportServiceResponse(false) + const getSubject = () => ({ + $type: 'com.atproto.repo.strongRef', + uri: sc.posts[sc.dids.carol][0].ref.uriStr, + cid: sc.posts[sc.dids.carol][0].ref.cidStr, + }) - await agent.api.com.atproto.admin.emitModerationEvent( + const emitDivertEvent = async () => + agent.api.com.atproto.admin.emitModerationEvent( { - subject: { - $type: 'com.atproto.repo.strongRef', - uri: sc.posts[sc.dids.carol][0].ref.uriStr, - cid: sc.posts[sc.dids.carol][0].ref.cidStr, - }, + subject: getSubject(), event: { $type: 'com.atproto.admin.defs#modEventDivert', comment: 'Diverting for test', @@ -58,18 +53,18 @@ describe('blob divert', () => { img.image.ref.toString(), ), }, - { headers: network.pds.adminAuthHeaders(), encoding: 'application/json' }, + { + headers: network.pds.adminAuthHeaders(), + encoding: 'application/json', + }, ) - await network.ozone.processAll() + it('fails and keeps attempt count when report service fails to accept upload.', async () => { + // Simulate failure to fail upload + const reportServiceRequest = mockReportServiceResponse(false) - const divertEvents = await network.ozone.ctx.db.db - .selectFrom('blob_push_event') - .selectAll() - .execute() + await expect(emitDivertEvent()).rejects.toThrow() - expect(divertEvents[0].attempts).toBeGreaterThan(0) - expect(divertEvents[1].attempts).toBeGreaterThan(0) expect(reportServiceRequest).toHaveBeenCalled() }) @@ -77,15 +72,22 @@ describe('blob divert', () => { // Simulate failure to accept upload const reportServiceRequest = mockReportServiceResponse(true) - await network.ozone.processAll() - - const divertEvents = await network.ozone.ctx.db.db - .selectFrom('blob_push_event') - .selectAll() - .execute() + const { data: divertEvent } = await emitDivertEvent() - expect(divertEvents[0].confirmedAt).toBeTruthy() - expect(divertEvents[1].confirmedAt).toBeTruthy() expect(reportServiceRequest).toHaveBeenCalled() + expect(forSnapshot(divertEvent)).toMatchSnapshot() + + const { + data: { subjectStatuses }, + } = await agent.api.com.atproto.admin.queryModerationStatuses( + { + subject: getSubject().uri, + }, + { + headers: network.pds.adminAuthHeaders(), + }, + ) + + expect(subjectStatuses[0].takendown).toBe(true) }) }) diff --git a/packages/pds/src/api/com/atproto/sync/getBlob.ts b/packages/pds/src/api/com/atproto/sync/getBlob.ts index 3b3d1d2f65a..f553d8eb2c6 100644 --- a/packages/pds/src/api/com/atproto/sync/getBlob.ts +++ b/packages/pds/src/api/com/atproto/sync/getBlob.ts @@ -11,6 +11,7 @@ export default function (server: Server, ctx: AppContext) { if (!ctx.authVerifier.isUserOrAdmin(auth, params.did)) { const available = await ctx.accountManager.isRepoAvailable(params.did) if (!available) { + console.log('!available blob not found') throw new InvalidRequestError('Blob not found') } } @@ -20,13 +21,16 @@ export default function (server: Server, ctx: AppContext) { return await store.repo.blob.getBlob(cid) } catch (err) { if (err instanceof BlobNotFoundError) { + console.log('actorStore! blob not found') throw new InvalidRequestError('Blob not found') } else { + console.log('actorStore! outside blob not found', err) throw err } } }) if (!found) { + console.log('!found blob not found') throw new InvalidRequestError('Blob not found') } res.setHeader('content-length', found.size) From 93fdbc0fbd6c0aed358321e9ceb24f7185221f3a Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Fri, 8 Mar 2024 11:53:04 +0000 Subject: [PATCH 16/20] :broom: Cleanup --- packages/ozone/src/config/config.ts | 12 ++++++------ packages/ozone/src/config/env.ts | 8 ++++---- packages/ozone/src/context.ts | 4 ++-- packages/ozone/src/daemon/blob-diverter.ts | 1 - .../ozone/src/db/schema/blob_divert_event.ts | 17 ----------------- packages/ozone/src/db/schema/blob_push_event.ts | 5 +---- packages/ozone/src/db/schema/index.ts | 4 +--- packages/ozone/tests/blob-divert.test.ts | 4 ++-- .../pds/src/api/com/atproto/sync/getBlob.ts | 4 ---- 9 files changed, 16 insertions(+), 43 deletions(-) delete mode 100644 packages/ozone/src/db/schema/blob_divert_event.ts diff --git a/packages/ozone/src/config/config.ts b/packages/ozone/src/config/config.ts index a74ad4aafc6..fbab8fdd11f 100644 --- a/packages/ozone/src/config/config.ts +++ b/packages/ozone/src/config/config.ts @@ -48,11 +48,11 @@ export const envToCfg = (env: OzoneEnvironment): OzoneConfig => { plcUrl: env.didPlcUrl, } - const blobReportServiceCfg = - env.blobReportServiceUrl && env.blobReportServiceAuthToken + const blobDivertServiceCfg = + env.blobDivertServiceUrl && env.blobDivertServiceAuthToken ? { - url: env.blobReportServiceUrl, - authToken: env.blobReportServiceAuthToken, + url: env.blobDivertServiceUrl, + authToken: env.blobDivertServiceAuthToken, } : undefined const accessCfg: OzoneConfig['access'] = { @@ -68,7 +68,7 @@ export const envToCfg = (env: OzoneEnvironment): OzoneConfig => { pds: pdsCfg, cdn: cdnCfg, identity: identityCfg, - blobReportService: blobReportServiceCfg, + blobDivertService: blobDivertServiceCfg, access: accessCfg, } } @@ -80,7 +80,7 @@ export type OzoneConfig = { pds: PdsConfig | null cdn: CdnConfig identity: IdentityConfig - blobReportService?: BlobReportServiceConfig + blobDivertService?: BlobReportServiceConfig access: AccessConfig } diff --git a/packages/ozone/src/config/env.ts b/packages/ozone/src/config/env.ts index 009880f8b20..8f1d15f5989 100644 --- a/packages/ozone/src/config/env.ts +++ b/packages/ozone/src/config/env.ts @@ -26,8 +26,8 @@ export const readEnv = (): OzoneEnvironment => { moderatorPassword: envStr('OZONE_MODERATOR_PASSWORD'), triagePassword: envStr('OZONE_TRIAGE_PASSWORD'), signingKeyHex: envStr('OZONE_SIGNING_KEY_HEX'), - blobReportServiceUrl: envStr('OZONE_BLOB_REPORT_SERVICE_URL'), - blobReportServiceAuthToken: envStr('OZONE_BLOB_REPORT_SERVICE_AUTH_TOKEN'), + blobDivertServiceUrl: envStr('OZONE_BLOB_DIVERT_SERVICE_URL'), + blobDivertServiceAuthToken: envStr('OZONE_BLOB_DIVERT_SERVICE_AUTH_TOKEN'), } } @@ -56,6 +56,6 @@ export type OzoneEnvironment = { moderatorPassword?: string triagePassword?: string signingKeyHex?: string - blobReportServiceUrl?: string - blobReportServiceAuthToken?: string + blobDivertServiceUrl?: string + blobDivertServiceAuthToken?: string } diff --git a/packages/ozone/src/context.ts b/packages/ozone/src/context.ts index 71f1280a5e5..ff63c6ccbe5 100644 --- a/packages/ozone/src/context.ts +++ b/packages/ozone/src/context.ts @@ -67,10 +67,10 @@ export class AppContext { }) const backgroundQueue = new BackgroundQueue(db) - const blobDiverter = cfg.blobReportService + const blobDiverter = cfg.blobDivertService ? new BlobDiverter(db, { idResolver, - serviceConfig: cfg.blobReportService, + serviceConfig: cfg.blobDivertService, }) : undefined const eventPusher = new EventPusher(db, createAuthHeaders, { diff --git a/packages/ozone/src/daemon/blob-diverter.ts b/packages/ozone/src/daemon/blob-diverter.ts index 7501ded1b45..407a0fdbd27 100644 --- a/packages/ozone/src/daemon/blob-diverter.ts +++ b/packages/ozone/src/daemon/blob-diverter.ts @@ -10,7 +10,6 @@ import { CID } from 'multiformats/cid' import Database from '../db' import { retryHttp } from '../util' -import { dbLogger } from '../logger' import { BlobReportServiceConfig } from '../config' export class BlobDiverter { diff --git a/packages/ozone/src/db/schema/blob_divert_event.ts b/packages/ozone/src/db/schema/blob_divert_event.ts deleted file mode 100644 index 77627f866bc..00000000000 --- a/packages/ozone/src/db/schema/blob_divert_event.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Generated } from 'kysely' - -export const eventTableName = 'blob_divert_event' - -export interface BlobDivertEvent { - id: Generated - subjectDid: string - subjectBlobCid: string - subjectUri: string - divertedAt: Date | null - lastAttempted: Date | null - attempts: Generated -} - -export type PartialDB = { - [eventTableName]: BlobDivertEvent -} diff --git a/packages/ozone/src/db/schema/blob_push_event.ts b/packages/ozone/src/db/schema/blob_push_event.ts index d4cd8e8b482..f38649e675c 100644 --- a/packages/ozone/src/db/schema/blob_push_event.ts +++ b/packages/ozone/src/db/schema/blob_push_event.ts @@ -2,10 +2,7 @@ import { Generated } from 'kysely' export const eventTableName = 'blob_push_event' -export type BlobPushEventType = - | 'pds_takedown' - | 'appview_takedown' - | 'blob_divert' +export type BlobPushEventType = 'pds_takedown' | 'appview_takedown' export interface BlobPushEvent { id: Generated diff --git a/packages/ozone/src/db/schema/index.ts b/packages/ozone/src/db/schema/index.ts index a589d4d0055..b522a75ef9f 100644 --- a/packages/ozone/src/db/schema/index.ts +++ b/packages/ozone/src/db/schema/index.ts @@ -6,7 +6,6 @@ import * as recordPushEvent from './record_push_event' import * as blobPushEvent from './blob_push_event' import * as label from './label' import * as communicationTemplate from './communication_template' -import * as blobDivertEvent from './blob_divert_event' export type DatabaseSchemaType = modEvent.PartialDB & modSubjectStatus.PartialDB & @@ -14,8 +13,7 @@ export type DatabaseSchemaType = modEvent.PartialDB & repoPushEvent.PartialDB & recordPushEvent.PartialDB & blobPushEvent.PartialDB & - communicationTemplate.PartialDB & - blobDivertEvent.PartialDB + communicationTemplate.PartialDB export type DatabaseSchema = Kysely diff --git a/packages/ozone/tests/blob-divert.test.ts b/packages/ozone/tests/blob-divert.test.ts index 1dcaf8349dd..186689c5bf5 100644 --- a/packages/ozone/tests/blob-divert.test.ts +++ b/packages/ozone/tests/blob-divert.test.ts @@ -12,8 +12,8 @@ describe('blob divert', () => { network = await TestNetwork.create({ dbPostgresSchema: 'ozone_blob_divert_test', ozone: { - blobReportServiceUrl: `https://blob-report.com`, - blobReportServiceAuthToken: 'test-auth-token', + blobDivertServiceUrl: `https://blob-report.com`, + blobDivertServiceAuthToken: 'test-auth-token', }, }) agent = network.pds.getClient() diff --git a/packages/pds/src/api/com/atproto/sync/getBlob.ts b/packages/pds/src/api/com/atproto/sync/getBlob.ts index f553d8eb2c6..3b3d1d2f65a 100644 --- a/packages/pds/src/api/com/atproto/sync/getBlob.ts +++ b/packages/pds/src/api/com/atproto/sync/getBlob.ts @@ -11,7 +11,6 @@ export default function (server: Server, ctx: AppContext) { if (!ctx.authVerifier.isUserOrAdmin(auth, params.did)) { const available = await ctx.accountManager.isRepoAvailable(params.did) if (!available) { - console.log('!available blob not found') throw new InvalidRequestError('Blob not found') } } @@ -21,16 +20,13 @@ export default function (server: Server, ctx: AppContext) { return await store.repo.blob.getBlob(cid) } catch (err) { if (err instanceof BlobNotFoundError) { - console.log('actorStore! blob not found') throw new InvalidRequestError('Blob not found') } else { - console.log('actorStore! outside blob not found', err) throw err } } }) if (!found) { - console.log('!found blob not found') throw new InvalidRequestError('Blob not found') } res.setHeader('content-length', found.size) From c7cff5237d42deb3715f070ceb08efba352de2c1 Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Fri, 8 Mar 2024 12:21:44 +0000 Subject: [PATCH 17/20] :white_check_mark: Use modClient seed client in blob-divert test --- packages/ozone/tests/blob-divert.test.ts | 31 +++++++++++------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/packages/ozone/tests/blob-divert.test.ts b/packages/ozone/tests/blob-divert.test.ts index 186689c5bf5..977ee326cb0 100644 --- a/packages/ozone/tests/blob-divert.test.ts +++ b/packages/ozone/tests/blob-divert.test.ts @@ -1,4 +1,9 @@ -import { SeedClient, TestNetwork, basicSeed } from '@atproto/dev-env' +import { + ModeratorClient, + SeedClient, + TestNetwork, + basicSeed, +} from '@atproto/dev-env' import AtpAgent from '@atproto/api' import { BlobDiverter } from '../src/daemon' import { forSnapshot } from './_util' @@ -7,6 +12,7 @@ describe('blob divert', () => { let network: TestNetwork let agent: AtpAgent let sc: SeedClient + let modClient: ModeratorClient beforeAll(async () => { network = await TestNetwork.create({ @@ -18,6 +24,7 @@ describe('blob divert', () => { }) agent = network.pds.getClient() sc = network.getSeedClient() + modClient = network.ozone.getModClient() await basicSeed(sc) await network.processAll() }) @@ -41,7 +48,7 @@ describe('blob divert', () => { }) const emitDivertEvent = async () => - agent.api.com.atproto.admin.emitModerationEvent( + modClient.emitModerationEvent( { subject: getSubject(), event: { @@ -53,10 +60,7 @@ describe('blob divert', () => { img.image.ref.toString(), ), }, - { - headers: network.pds.adminAuthHeaders(), - encoding: 'application/json', - }, + 'moderator', ) it('fails and keeps attempt count when report service fails to accept upload.', async () => { @@ -72,21 +76,14 @@ describe('blob divert', () => { // Simulate failure to accept upload const reportServiceRequest = mockReportServiceResponse(true) - const { data: divertEvent } = await emitDivertEvent() + const divertEvent = await emitDivertEvent() expect(reportServiceRequest).toHaveBeenCalled() expect(forSnapshot(divertEvent)).toMatchSnapshot() - const { - data: { subjectStatuses }, - } = await agent.api.com.atproto.admin.queryModerationStatuses( - { - subject: getSubject().uri, - }, - { - headers: network.pds.adminAuthHeaders(), - }, - ) + const { subjectStatuses } = await modClient.queryModerationStatuses({ + subject: getSubject().uri, + }) expect(subjectStatuses[0].takendown).toBe(true) }) From f2ac002965019b41e362ed34dd5d92f09a63e8e6 Mon Sep 17 00:00:00 2001 From: Devin Ivy Date: Mon, 11 Mar 2024 19:10:22 -0400 Subject: [PATCH 18/20] update divert blob config to use basic admin auth --- packages/ozone/src/config/config.ts | 16 ++++++++-------- packages/ozone/src/config/env.ts | 8 ++++---- packages/ozone/src/context.ts | 4 ++-- packages/ozone/src/daemon/blob-diverter.ts | 12 ++++++++---- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/packages/ozone/src/config/config.ts b/packages/ozone/src/config/config.ts index d7337cd98d2..b3fb1c8dbe9 100644 --- a/packages/ozone/src/config/config.ts +++ b/packages/ozone/src/config/config.ts @@ -51,12 +51,12 @@ export const envToCfg = (env: OzoneEnvironment): OzoneConfig => { } const blobDivertServiceCfg = - env.blobDivertServiceUrl && env.blobDivertServiceAuthToken + env.blobDivertUrl && env.blobDivertAdminPassword ? { - url: env.blobDivertServiceUrl, - authToken: env.blobDivertServiceAuthToken, + url: env.blobDivertUrl, + adminPassword: env.blobDivertAdminPassword, } - : undefined + : null const accessCfg: OzoneConfig['access'] = { admins: env.adminDids, moderators: env.moderatorDids, @@ -70,7 +70,7 @@ export const envToCfg = (env: OzoneEnvironment): OzoneConfig => { pds: pdsCfg, cdn: cdnCfg, identity: identityCfg, - blobDivertService: blobDivertServiceCfg, + blobDivert: blobDivertServiceCfg, access: accessCfg, } } @@ -82,7 +82,7 @@ export type OzoneConfig = { pds: PdsConfig | null cdn: CdnConfig identity: IdentityConfig - blobDivertService?: BlobReportServiceConfig + blobDivert: BlobDivertConfig | null access: AccessConfig } @@ -94,9 +94,9 @@ export type ServiceConfig = { devMode?: boolean } -export type BlobReportServiceConfig = { +export type BlobDivertConfig = { url: string - authToken: string + adminPassword: string } export type DatabaseConfig = { diff --git a/packages/ozone/src/config/env.ts b/packages/ozone/src/config/env.ts index 57ad90638e1..a879339aacd 100644 --- a/packages/ozone/src/config/env.ts +++ b/packages/ozone/src/config/env.ts @@ -25,8 +25,8 @@ export const readEnv = (): OzoneEnvironment => { triageDids: envList('OZONE_TRIAGE_DIDS'), adminPassword: envStr('OZONE_ADMIN_PASSWORD'), signingKeyHex: envStr('OZONE_SIGNING_KEY_HEX'), - blobDivertServiceUrl: envStr('OZONE_BLOB_DIVERT_SERVICE_URL'), - blobDivertServiceAuthToken: envStr('OZONE_BLOB_DIVERT_SERVICE_AUTH_TOKEN'), + blobDivertUrl: envStr('OZONE_BLOB_DIVERT_URL'), + blobDivertAdminPassword: envStr('OZONE_BLOB_DIVERT_ADMIN_PASSWORD'), } } @@ -54,6 +54,6 @@ export type OzoneEnvironment = { triageDids: string[] adminPassword?: string signingKeyHex?: string - blobDivertServiceUrl?: string - blobDivertServiceAuthToken?: string + blobDivertUrl?: string + blobDivertAdminPassword?: string } diff --git a/packages/ozone/src/context.ts b/packages/ozone/src/context.ts index 4d6378cf804..d0cbd9ae347 100644 --- a/packages/ozone/src/context.ts +++ b/packages/ozone/src/context.ts @@ -70,10 +70,10 @@ export class AppContext { }) const backgroundQueue = new BackgroundQueue(db) - const blobDiverter = cfg.blobDivertService + const blobDiverter = cfg.blobDivert ? new BlobDiverter(db, { idResolver, - serviceConfig: cfg.blobDivertService, + serviceConfig: cfg.blobDivert, }) : undefined const eventPusher = new EventPusher(db, createAuthHeaders, { diff --git a/packages/ozone/src/daemon/blob-diverter.ts b/packages/ozone/src/daemon/blob-diverter.ts index 407a0fdbd27..386c3a090cb 100644 --- a/packages/ozone/src/daemon/blob-diverter.ts +++ b/packages/ozone/src/daemon/blob-diverter.ts @@ -10,17 +10,17 @@ import { CID } from 'multiformats/cid' import Database from '../db' import { retryHttp } from '../util' -import { BlobReportServiceConfig } from '../config' +import { BlobDivertConfig } from '../config' export class BlobDiverter { - serviceConfig: BlobReportServiceConfig + serviceConfig: BlobDivertConfig idResolver: IdResolver constructor( public db: Database, services: { idResolver: IdResolver - serviceConfig: BlobReportServiceConfig + serviceConfig: BlobDivertConfig }, ) { this.serviceConfig = services.serviceConfig @@ -69,7 +69,7 @@ export class BlobDiverter { method: 'POST', data: imageStream, headers: { - Authorization: this.serviceConfig.authToken, + Authorization: basicAuth('admin', this.serviceConfig.adminPassword), 'Content-Type': contentType, }, }) @@ -144,3 +144,7 @@ export class BlobDiverter { return true } } + +const basicAuth = (username: string, password: string) => { + return 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64') +} From 0e4c8c8f35b58ce5d261f32a7e66273653a9cc6c Mon Sep 17 00:00:00 2001 From: Devin Ivy Date: Mon, 11 Mar 2024 19:30:46 -0400 Subject: [PATCH 19/20] fix --- packages/ozone/tests/blob-divert.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ozone/tests/blob-divert.test.ts b/packages/ozone/tests/blob-divert.test.ts index 977ee326cb0..0890ac8a136 100644 --- a/packages/ozone/tests/blob-divert.test.ts +++ b/packages/ozone/tests/blob-divert.test.ts @@ -18,8 +18,8 @@ describe('blob divert', () => { network = await TestNetwork.create({ dbPostgresSchema: 'ozone_blob_divert_test', ozone: { - blobDivertServiceUrl: `https://blob-report.com`, - blobDivertServiceAuthToken: 'test-auth-token', + blobDivertUrl: `https://blob-report.com`, + blobDivertAdminPassword: 'test-auth-token', }, }) agent = network.pds.getClient() From fefd23eb93883f5dd603056c0a2c50ac9bd706de Mon Sep 17 00:00:00 2001 From: Devin Ivy Date: Mon, 11 Mar 2024 20:53:56 -0400 Subject: [PATCH 20/20] build --- .github/workflows/build-and-push-ozone-aws.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-push-ozone-aws.yaml b/.github/workflows/build-and-push-ozone-aws.yaml index 53f95c5b731..ff8162bb941 100644 --- a/.github/workflows/build-and-push-ozone-aws.yaml +++ b/.github/workflows/build-and-push-ozone-aws.yaml @@ -3,6 +3,7 @@ on: push: branches: - main + - divert-blobs env: REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }} USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }}