Merge branch 'main' into disable-pds-appview-indexing

Makefile

@@ -37,6 +37,8 @@ codegen: ## Re-generate packages from lexicon/ files
 	cd packages/api; pnpm run codegen
 	cd packages/pds; pnpm run codegen
 	cd packages/bsky; pnpm run codegen
+	# clean up codegen output
+	pnpm format
 
 .PHONY: lint
 lint: ## Run style checks and verify syntax

lexicons/app/bsky/feed/post.json

@@ -38,6 +38,12 @@
             "type": "union",
             "refs": ["com.atproto.label.defs#selfLabels"]
           },
+          "tags": {
+            "type": "array",
+            "maxLength": 8,
+            "items": { "type": "string", "maxLength": 640, "maxGraphemes": 64 },
+            "description": "Additional non-inline tags describing this post."
+          },
           "createdAt": { "type": "string", "format": "datetime" }
         }
       }

lexicons/app/bsky/richtext/facet.json

@@ -9,7 +9,7 @@
         "index": { "type": "ref", "ref": "#byteSlice" },
         "features": {
           "type": "array",
-          "items": { "type": "union", "refs": ["#mention", "#link"] }
+          "items": { "type": "union", "refs": ["#mention", "#link", "#tag"] }
         }
       }
     },
@@ -29,6 +29,14 @@
         "uri": { "type": "string", "format": "uri" }
       }
     },
+    "tag": {
+      "type": "object",
+      "description": "A hashtag.",
+      "required": ["tag"],
+      "properties": {
+        "tag": { "type": "string", "maxLength": 640, "maxGraphemes": 64 }
+      }
+    },
     "byteSlice": {
       "type": "object",
       "description": "A text segment. Start is inclusive, end is exclusive. Indices are for utf8-encoded strings.",

packages/api/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @atproto/api
 
+## 0.6.17
+
+### Patch Changes
+
+- [#1637](https://github.com/bluesky-social/atproto/pull/1637) [`d96f7d9b`](https://github.com/bluesky-social/atproto/commit/d96f7d9b84c6fbab9711059c8584a77d892dcedd) Thanks [@estrattonbailey](https://github.com/estrattonbailey)! - Introduce general support for tags on posts
+
 ## 0.6.16
 
 ### Patch Changes

packages/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atproto/api",
-  "version": "0.6.16",
+  "version": "0.6.17",
   "license": "MIT",
   "description": "Client library for atproto and Bluesky",
   "keywords": [

packages/api/src/client/lexicons.ts

@@ -5622,6 +5622,16 @@ export const schemaDict = {
               type: 'union',
               refs: ['lex:com.atproto.label.defs#selfLabels'],
             },
+            tags: {
+              type: 'array',
+              maxLength: 8,
+              items: {
+                type: 'string',
+                maxLength: 640,
+                maxGraphemes: 64,
+              },
+              description: 'Additional non-inline tags describing this post.',
+            },
             createdAt: {
               type: 'string',
               format: 'datetime',
@@ -6761,6 +6771,7 @@ export const schemaDict = {
               refs: [
                 'lex:app.bsky.richtext.facet#mention',
                 'lex:app.bsky.richtext.facet#link',
+                'lex:app.bsky.richtext.facet#tag',
               ],
             },
           },
@@ -6788,6 +6799,18 @@ export const schemaDict = {
           },
         },
       },
+      tag: {
+        type: 'object',
+        description: 'A hashtag.',
+        required: ['tag'],
+        properties: {
+          tag: {
+            type: 'string',
+            maxLength: 640,
+            maxGraphemes: 64,
+          },
+        },
+      },
       byteSlice: {
         type: 'object',
         description:

packages/api/src/client/types/app/bsky/feed/post.ts

@@ -29,6 +29,8 @@ export interface Record {
   labels?:
     | ComAtprotoLabelDefs.SelfLabels
     | { $type: string; [k: string]: unknown }
+  /** Additional non-inline tags describing this post. */
+  tags?: string[]
   createdAt: string
   [k: string]: unknown
 }

packages/api/src/client/types/app/bsky/richtext/facet.ts

@@ -8,7 +8,7 @@ import { CID } from 'multiformats/cid'
 
 export interface Main {
   index: ByteSlice
-  features: (Mention | Link | { $type: string; [k: string]: unknown })[]
+  features: (Mention | Link | Tag | { $type: string; [k: string]: unknown })[]
   [k: string]: unknown
 }
 
@@ -61,6 +61,22 @@ export function validateLink(v: unknown): ValidationResult {
   return lexicons.validate('app.bsky.richtext.facet#link', v)
 }
 
+/** A hashtag. */
+export interface Tag {
+  tag: string
+  [k: string]: unknown
+}
+
+export function isTag(v: unknown): v is Tag {
+  return (
+    isObj(v) && hasProp(v, '$type') && v.$type === 'app.bsky.richtext.facet#tag'
+  )
+}
+
+export function validateTag(v: unknown): ValidationResult {
+  return lexicons.validate('app.bsky.richtext.facet#tag', v)
+}
+
 /** A text segment. Start is inclusive, end is exclusive. Indices are for utf8-encoded strings. */
 export interface ByteSlice {
   byteStart: number

packages/bsky/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @atproto/bsky
 
+## 0.0.8
+
+### Patch Changes
+
+- [#1637](https://github.com/bluesky-social/atproto/pull/1637) [`d96f7d9b`](https://github.com/bluesky-social/atproto/commit/d96f7d9b84c6fbab9711059c8584a77d892dcedd) Thanks [@estrattonbailey](https://github.com/estrattonbailey)! - Introduce general support for tags on posts
+
+- Updated dependencies [[`d96f7d9b`](https://github.com/bluesky-social/atproto/commit/d96f7d9b84c6fbab9711059c8584a77d892dcedd)]:
+  - @atproto/api@0.6.17
+
 ## 0.0.7
 
 ### Patch Changes

packages/bsky/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atproto/bsky",
-  "version": "0.0.7",
+  "version": "0.0.8",
   "license": "MIT",
   "description": "Reference implementation of app.bsky App View (Bluesky API)",
   "keywords": [

packages/bsky/src/auto-moderator/abyss.ts

@@ -1,5 +1,6 @@
 import axios from 'axios'
 import { CID } from 'multiformats/cid'
+import { AtUri } from '@atproto/syntax'
 import * as ui8 from 'uint8arrays'
 import { resolveBlob } from '../api/blob-resolver'
 import { retryHttp } from '../util/retry'
@@ -8,7 +9,7 @@ import { IdResolver } from '@atproto/identity'
 import { labelerLogger as log } from '../logger'
 
 export interface ImageFlagger {
-  scanImage(did: string, cid: CID): Promise<string[]>
+  scanImage(did: string, cid: CID, uri: AtUri): Promise<string[]>
 }
 
 export class Abyss implements ImageFlagger {
@@ -22,11 +23,11 @@ export class Abyss implements ImageFlagger {
     this.auth = basicAuth(this.password)
   }
 
-  async scanImage(did: string, cid: CID): Promise<string[]> {
+  async scanImage(did: string, cid: CID, uri: AtUri): Promise<string[]> {
     const start = Date.now()
     const res = await retryHttp(async () => {
       try {
-        return await this.makeReq(did, cid)
+        return await this.makeReq(did, cid, uri)
       } catch (err) {
         log.warn({ err, did, cid: cid.toString() }, 'abyss request failed')
         throw err
@@ -39,20 +40,24 @@ export class Abyss implements ImageFlagger {
     return this.parseRes(res)
   }
 
-  async makeReq(did: string, cid: CID): Promise<ScannerResp> {
+  async makeReq(did: string, cid: CID, uri: AtUri): Promise<ScannerResp> {
     const { stream, contentType } = await resolveBlob(
       did,
       cid,
       this.ctx.db,
       this.ctx.idResolver,
     )
-    const { data } = await axios.post(this.getReqUrl({ did }), stream, {
-      headers: {
-        'Content-Type': contentType,
-        authorization: this.auth,
+    const { data } = await axios.post(
+      this.getReqUrl({ did, uri: uri.toString() }),
+      stream,
+      {
+        headers: {
+          'Content-Type': contentType,
+          authorization: this.auth,
+        },
+        timeout: 10000,
       },
-      timeout: 10000,
-    })
+    )
     return data
   }
 
@@ -69,8 +74,11 @@ export class Abyss implements ImageFlagger {
     return labels
   }
 
-  getReqUrl(params: { did: string }) {
-    return `${this.endpoint}/xrpc/com.atproto.unspecced.scanBlob?did=${params.did}`
+  getReqUrl(params: { did: string; uri: string }) {
+    const search = new URLSearchParams(params)
+    return `${
+      this.endpoint
+    }/xrpc/com.atproto.unspecced.scanBlob?${search.toString()}`
   }
 }
 

packages/bsky/src/auto-moderator/index.ts

@@ -18,7 +18,10 @@ import { ImageUriBuilder } from '../image/uri'
 import { ImageInvalidator } from '../image/invalidator'
 import { Abyss } from './abyss'
 import { FuzzyMatcher, TextFlagger } from './fuzzy-matcher'
-import { REASONOTHER } from '../lexicon/types/com/atproto/moderation/defs'
+import {
+  REASONOTHER,
+  REASONVIOLATION,
+} from '../lexicon/types/com/atproto/moderation/defs'
 
 export class AutoModerator {
   public pushAgent?: AtpAgent
@@ -172,7 +175,7 @@ export class AutoModerator {
   async checkImgForTakedown(uri: AtUri, recordCid: CID, imgCids: CID[]) {
     if (imgCids.length < 0) return
     const results = await Promise.all(
-      imgCids.map((cid) => this.imageFlagger?.scanImage(uri.host, cid)),
+      imgCids.map((cid) => this.imageFlagger?.scanImage(uri.host, cid, uri)),
     )
     const takedownCids: CID[] = []
     for (let i = 0; i < results.length; i++) {
@@ -207,7 +210,39 @@ export class AutoModerator {
     takedownCids: CID[],
     labels: string[],
   ) {
-    const reason = `automated takedown for labels: ${labels.join(', ')}`
+    const reportReason = `automated takedown (${labels.join(
+      ', ',
+    )}). account needs review and possibly additional action`
+    const takedownReason = `automated takedown for labels: ${labels.join(', ')}`
+    log.warn(
+      {
+        uri: uri.toString(),
+        blobCids: takedownCids,
+        labels,
+      },
+      'hard takedown of record (and blobs) based on auto-matching',
+    )
+
+    if (this.services.moderation) {
+      await this.ctx.db.transaction(async (dbTxn) => {
+        // directly/locally create report, even if we use pushAgent for the takedown. don't have acctual account credentials for pushAgent, only admin auth
+        if (!this.services.moderation) {
+          // checked above, outside the transaction
+          return
+        }
+        const modSrvc = this.services.moderation(dbTxn)
+        await modSrvc.report({
+          reportedBy: this.ctx.cfg.labelerDid,
+          reasonType: REASONVIOLATION,
+          subject: {
+            uri: uri,
+            cid: recordCid,
+          },
+          reason: reportReason,
+        })
+      })
+    }
+
     if (this.pushAgent) {
       await this.pushAgent.com.atproto.admin.takeModerationAction({
         action: 'com.atproto.admin.defs#takedown',
@@ -217,7 +252,7 @@ export class AutoModerator {
           cid: recordCid.toString(),
         },
         subjectBlobCids: takedownCids.map((c) => c.toString()),
-        reason,
+        reason: takedownReason,
         createdBy: this.ctx.cfg.labelerDid,
       })
     } else {
@@ -230,7 +265,7 @@ export class AutoModerator {
           action: 'com.atproto.admin.defs#takedown',
           subject: { uri, cid: recordCid },
           subjectBlobCids: takedownCids,
-          reason,
+          reason: takedownReason,
           createdBy: this.ctx.cfg.labelerDid,
         })
         await modSrvc.takedownRecord({

packages/bsky/src/db/migrations/20230920T213858047Z-add-tags-to-post.ts

@@ -0,0 +1,9 @@
+import { Kysely } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema.alterTable('post').addColumn('tags', 'jsonb').execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.alterTable('post').dropColumn('tags').execute()
+}

packages/bsky/src/db/migrations/index.ts

@@ -28,3 +28,4 @@ export * as _20230817T195936007Z from './20230817T195936007Z-native-notification
 export * as _20230830T205507322Z from './20230830T205507322Z-suggested-feeds'
 export * as _20230904T211011773Z from './20230904T211011773Z-block-lists'
 export * as _20230906T222220386Z from './20230906T222220386Z-thread-gating'
+export * as _20230920T213858047Z from './20230920T213858047Z-add-tags-to-post'

packages/bsky/src/db/tables/post.ts

@@ -12,6 +12,7 @@ export interface Post {
   replyParent: string | null
   replyParentCid: string | null
   langs: string[] | null
+  tags: string[] | null
   invalidReplyRoot: boolean | null
   violatesThreadGate: boolean | null
   createdAt: string

packages/bsky/src/lexicon/lexicons.ts

@@ -5622,6 +5622,16 @@ export const schemaDict = {
               type: 'union',
               refs: ['lex:com.atproto.label.defs#selfLabels'],
             },
+            tags: {
+              type: 'array',
+              maxLength: 8,
+              items: {
+                type: 'string',
+                maxLength: 640,
+                maxGraphemes: 64,
+              },
+              description: 'Additional non-inline tags describing this post.',
+            },
             createdAt: {
               type: 'string',
               format: 'datetime',
@@ -6761,6 +6771,7 @@ export const schemaDict = {
               refs: [
                 'lex:app.bsky.richtext.facet#mention',
                 'lex:app.bsky.richtext.facet#link',
+                'lex:app.bsky.richtext.facet#tag',
               ],
             },
           },
@@ -6788,6 +6799,18 @@ export const schemaDict = {
           },
         },
       },
+      tag: {
+        type: 'object',
+        description: 'A hashtag.',
+        required: ['tag'],
+        properties: {
+          tag: {
+            type: 'string',
+            maxLength: 640,
+            maxGraphemes: 64,
+          },
+        },
+      },
       byteSlice: {
         type: 'object',
         description: