auto-moderator tweaks: pass along record URI, create report for taked…

…own action (#1643) * auto-moderator: include record URI in abyss requests * auto-moderator: log attempt at hard takedown; create report as well The motivation is to flag the event to mod team, and to make it easier to confirm that takedown took place. * auto-mod: typo fix * auto-mod: bugfixes * bsky: always create auto-mod report locally, not pushAgent (if possible) * bsky: fix auto-mod build * bsky: URL-encode scanBlob call
bluesky-social · Sep 26, 2023 · 8a62d6e · 8a62d6e
1 parent 607d94f
commit 8a62d6e
Show file tree

Hide file tree

Showing 3 changed files with 62 additions and 18 deletions.
diff --git a/packages/bsky/src/auto-moderator/abyss.ts b/packages/bsky/src/auto-moderator/abyss.ts
@@ -1,5 +1,6 @@
 import axios from 'axios'
 import { CID } from 'multiformats/cid'
+import { AtUri } from '@atproto/syntax'
 import * as ui8 from 'uint8arrays'
 import { resolveBlob } from '../api/blob-resolver'
 import { retryHttp } from '../util/retry'
@@ -8,7 +9,7 @@ import { IdResolver } from '@atproto/identity'
 import { labelerLogger as log } from '../logger'
 
 export interface ImageFlagger {
-  scanImage(did: string, cid: CID): Promise<string[]>
+  scanImage(did: string, cid: CID, uri: AtUri): Promise<string[]>
 }
 
 export class Abyss implements ImageFlagger {
@@ -22,11 +23,11 @@ export class Abyss implements ImageFlagger {
     this.auth = basicAuth(this.password)
   }
 
-  async scanImage(did: string, cid: CID): Promise<string[]> {
+  async scanImage(did: string, cid: CID, uri: AtUri): Promise<string[]> {
     const start = Date.now()
     const res = await retryHttp(async () => {
       try {
-        return await this.makeReq(did, cid)
+        return await this.makeReq(did, cid, uri)
       } catch (err) {
         log.warn({ err, did, cid: cid.toString() }, 'abyss request failed')
         throw err
@@ -39,20 +40,24 @@ export class Abyss implements ImageFlagger {
     return this.parseRes(res)
   }
 
-  async makeReq(did: string, cid: CID): Promise<ScannerResp> {
+  async makeReq(did: string, cid: CID, uri: AtUri): Promise<ScannerResp> {
     const { stream, contentType } = await resolveBlob(
       did,
       cid,
       this.ctx.db,
       this.ctx.idResolver,
     )
-    const { data } = await axios.post(this.getReqUrl({ did }), stream, {
-      headers: {
-        'Content-Type': contentType,
-        authorization: this.auth,
+    const { data } = await axios.post(
+      this.getReqUrl({ did, uri: uri.toString() }),
+      stream,
+      {
+        headers: {
+          'Content-Type': contentType,
+          authorization: this.auth,
+        },
+        timeout: 10000,
       },
-      timeout: 10000,
-    })
+    )
     return data
   }
 
@@ -69,8 +74,11 @@ export class Abyss implements ImageFlagger {
     return labels
   }
 
-  getReqUrl(params: { did: string }) {
-    return `${this.endpoint}/xrpc/com.atproto.unspecced.scanBlob?did=${params.did}`
+  getReqUrl(params: { did: string; uri: string }) {
+    const search = new URLSearchParams(params)
+    return `${
+      this.endpoint
+    }/xrpc/com.atproto.unspecced.scanBlob?${search.toString()}`
   }
 }
 

diff --git a/packages/bsky/src/auto-moderator/index.ts b/packages/bsky/src/auto-moderator/index.ts
@@ -18,7 +18,10 @@ import { ImageUriBuilder } from '../image/uri'
 import { ImageInvalidator } from '../image/invalidator'
 import { Abyss } from './abyss'
 import { FuzzyMatcher, TextFlagger } from './fuzzy-matcher'
-import { REASONOTHER } from '../lexicon/types/com/atproto/moderation/defs'
+import {
+  REASONOTHER,
+  REASONVIOLATION,
+} from '../lexicon/types/com/atproto/moderation/defs'
 
 export class AutoModerator {
   public pushAgent?: AtpAgent
@@ -172,7 +175,7 @@ export class AutoModerator {
   async checkImgForTakedown(uri: AtUri, recordCid: CID, imgCids: CID[]) {
     if (imgCids.length < 0) return
     const results = await Promise.all(
-      imgCids.map((cid) => this.imageFlagger?.scanImage(uri.host, cid)),
+      imgCids.map((cid) => this.imageFlagger?.scanImage(uri.host, cid, uri)),
     )
     const takedownCids: CID[] = []
     for (let i = 0; i < results.length; i++) {
@@ -207,7 +210,39 @@ export class AutoModerator {
     takedownCids: CID[],
     labels: string[],
   ) {
-    const reason = `automated takedown for labels: ${labels.join(', ')}`
+    const reportReason = `automated takedown (${labels.join(
+      ', ',
+    )}). account needs review and possibly additional action`
+    const takedownReason = `automated takedown for labels: ${labels.join(', ')}`
+    log.warn(
+      {
+        uri: uri.toString(),
+        blobCids: takedownCids,
+        labels,
+      },
+      'hard takedown of record (and blobs) based on auto-matching',
+    )
+
+    if (this.services.moderation) {
+      await this.ctx.db.transaction(async (dbTxn) => {
+        // directly/locally create report, even if we use pushAgent for the takedown. don't have acctual account credentials for pushAgent, only admin auth
+        if (!this.services.moderation) {
+          // checked above, outside the transaction
+          return
+        }
+        const modSrvc = this.services.moderation(dbTxn)
+        await modSrvc.report({
+          reportedBy: this.ctx.cfg.labelerDid,
+          reasonType: REASONVIOLATION,
+          subject: {
+            uri: uri,
+            cid: recordCid,
+          },
+          reason: reportReason,
+        })
+      })
+    }
+
     if (this.pushAgent) {
       await this.pushAgent.com.atproto.admin.takeModerationAction({
         action: 'com.atproto.admin.defs#takedown',
@@ -217,7 +252,7 @@ export class AutoModerator {
           cid: recordCid.toString(),
         },
         subjectBlobCids: takedownCids.map((c) => c.toString()),
-        reason,
+        reason: takedownReason,
         createdBy: this.ctx.cfg.labelerDid,
       })
     } else {
@@ -230,7 +265,7 @@ export class AutoModerator {
           action: 'com.atproto.admin.defs#takedown',
           subject: { uri, cid: recordCid },
           subjectBlobCids: takedownCids,
-          reason,
+          reason: takedownReason,
           createdBy: this.ctx.cfg.labelerDid,
         })
         await modSrvc.takedownRecord({

diff --git a/packages/bsky/tests/auto-moderator/takedowns.test.ts b/packages/bsky/tests/auto-moderator/takedowns.test.ts
@@ -7,6 +7,7 @@ import { TestNetwork } from '@atproto/dev-env'
 import { ImageRef, SeedClient } from '../seeds/client'
 import usersSeed from '../seeds/users'
 import { CID } from 'multiformats/cid'
+import { AtUri } from '@atproto/syntax'
 import { ImageFlagger } from '../../src/auto-moderator/abyss'
 import { ImageInvalidator } from '../../src/image/invalidator'
 import { sha256 } from '@atproto/crypto'
@@ -157,7 +158,7 @@ class TestInvalidator implements ImageInvalidator {
 }
 
 class TestFlagger implements ImageFlagger {
-  async scanImage(_did: string, cid: CID): Promise<string[]> {
+  async scanImage(_did: string, cid: CID, _uri: AtUri): Promise<string[]> {
     if (cid.equals(badCid1)) {
       return ['kill-it']
     } else if (cid.equals(badCid2)) {