add tags to posts (#1637)

* add tags to post lex * kiss * add richtext facet and validation attrs * add tag validation attrs to post * codegen * add maxLength for tags, add description * validate post tags on write * add test * handle tags in indexer * add tags to postView, codegen * return tags on post thread view * format * revert formatting change to docs * use establish validation pattern * add changeset (cherry picked from commit fcb6fe7) * remove tags from postView, codegen * remove tags from thread view * revert unused changes
bluesky-social · Sep 25, 2023 · d96f7d9 · d96f7d9
1 parent d703bc3
commit d96f7d9
Show file tree

Hide file tree

Showing 20 changed files with 236 additions and 6 deletions.
diff --git a/.changeset/three-snakes-turn.md b/.changeset/three-snakes-turn.md
@@ -0,0 +1,7 @@
+---
+'@atproto/bsky': patch
+'@atproto/api': patch
+'@atproto/pds': patch
+---
+
+Introduce general support for tags on posts
diff --git a/lexicons/app/bsky/feed/post.json b/lexicons/app/bsky/feed/post.json
@@ -38,6 +38,12 @@
             "type": "union",
             "refs": ["com.atproto.label.defs#selfLabels"]
           },
+          "tags": {
+            "type": "array",
+            "maxLength": 8,
+            "items": { "type": "string", "maxLength": 640, "maxGraphemes": 64 },
+            "description": "Additional non-inline tags describing this post."
+          },
           "createdAt": { "type": "string", "format": "datetime" }
         }
       }

diff --git a/lexicons/app/bsky/richtext/facet.json b/lexicons/app/bsky/richtext/facet.json
@@ -9,7 +9,7 @@
         "index": { "type": "ref", "ref": "#byteSlice" },
         "features": {
           "type": "array",
-          "items": { "type": "union", "refs": ["#mention", "#link"] }
+          "items": { "type": "union", "refs": ["#mention", "#link", "#tag"] }
         }
       }
     },
@@ -29,6 +29,14 @@
         "uri": { "type": "string", "format": "uri" }
       }
     },
+    "tag": {
+      "type": "object",
+      "description": "A hashtag.",
+      "required": ["tag"],
+      "properties": {
+        "tag": { "type": "string", "maxLength": 640, "maxGraphemes": 64 }
+      }
+    },
     "byteSlice": {
       "type": "object",
       "description": "A text segment. Start is inclusive, end is exclusive. Indices are for utf8-encoded strings.",

diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts
@@ -5622,6 +5622,16 @@ export const schemaDict = {
               type: 'union',
               refs: ['lex:com.atproto.label.defs#selfLabels'],
             },
+            tags: {
+              type: 'array',
+              maxLength: 8,
+              items: {
+                type: 'string',
+                maxLength: 640,
+                maxGraphemes: 64,
+              },
+              description: 'Additional non-inline tags describing this post.',
+            },
             createdAt: {
               type: 'string',
               format: 'datetime',
@@ -6761,6 +6771,7 @@ export const schemaDict = {
               refs: [
                 'lex:app.bsky.richtext.facet#mention',
                 'lex:app.bsky.richtext.facet#link',
+                'lex:app.bsky.richtext.facet#tag',
               ],
             },
           },
@@ -6788,6 +6799,18 @@ export const schemaDict = {
           },
         },
       },
+      tag: {
+        type: 'object',
+        description: 'A hashtag.',
+        required: ['tag'],
+        properties: {
+          tag: {
+            type: 'string',
+            maxLength: 640,
+            maxGraphemes: 64,
+          },
+        },
+      },
       byteSlice: {
         type: 'object',
         description:

diff --git a/packages/api/src/client/types/app/bsky/feed/post.ts b/packages/api/src/client/types/app/bsky/feed/post.ts
@@ -29,6 +29,8 @@ export interface Record {
   labels?:
     | ComAtprotoLabelDefs.SelfLabels
     | { $type: string; [k: string]: unknown }
+  /** Additional non-inline tags describing this post. */
+  tags?: string[]
   createdAt: string
   [k: string]: unknown
 }

diff --git a/packages/api/src/client/types/app/bsky/richtext/facet.ts b/packages/api/src/client/types/app/bsky/richtext/facet.ts
@@ -8,7 +8,7 @@ import { CID } from 'multiformats/cid'
 
 export interface Main {
   index: ByteSlice
-  features: (Mention | Link | { $type: string; [k: string]: unknown })[]
+  features: (Mention | Link | Tag | { $type: string; [k: string]: unknown })[]
   [k: string]: unknown
 }
 
@@ -61,6 +61,22 @@ export function validateLink(v: unknown): ValidationResult {
   return lexicons.validate('app.bsky.richtext.facet#link', v)
 }
 
+/** A hashtag. */
+export interface Tag {
+  tag: string
+  [k: string]: unknown
+}
+
+export function isTag(v: unknown): v is Tag {
+  return (
+    isObj(v) && hasProp(v, '$type') && v.$type === 'app.bsky.richtext.facet#tag'
+  )
+}
+
+export function validateTag(v: unknown): ValidationResult {
+  return lexicons.validate('app.bsky.richtext.facet#tag', v)
+}
+
 /** A text segment. Start is inclusive, end is exclusive. Indices are for utf8-encoded strings. */
 export interface ByteSlice {
   byteStart: number

diff --git a/packages/bsky/src/db/migrations/20230920T213858047Z-add-tags-to-post.ts b/packages/bsky/src/db/migrations/20230920T213858047Z-add-tags-to-post.ts
@@ -0,0 +1,9 @@
+import { Kysely } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema.alterTable('post').addColumn('tags', 'jsonb').execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.alterTable('post').dropColumn('tags').execute()
+}
diff --git a/packages/bsky/src/db/migrations/index.ts b/packages/bsky/src/db/migrations/index.ts
@@ -28,3 +28,4 @@ export * as _20230817T195936007Z from './20230817T195936007Z-native-notification
 export * as _20230830T205507322Z from './20230830T205507322Z-suggested-feeds'
 export * as _20230904T211011773Z from './20230904T211011773Z-block-lists'
 export * as _20230906T222220386Z from './20230906T222220386Z-thread-gating'
+export * as _20230920T213858047Z from './20230920T213858047Z-add-tags-to-post'
diff --git a/packages/bsky/src/db/tables/post.ts b/packages/bsky/src/db/tables/post.ts
@@ -12,6 +12,7 @@ export interface Post {
   replyParent: string | null
   replyParentCid: string | null
   langs: string[] | null
+  tags: string[] | null
   invalidReplyRoot: boolean | null
   violatesThreadGate: boolean | null
   createdAt: string

diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts
@@ -5622,6 +5622,16 @@ export const schemaDict = {
               type: 'union',
               refs: ['lex:com.atproto.label.defs#selfLabels'],
             },
+            tags: {
+              type: 'array',
+              maxLength: 8,
+              items: {
+                type: 'string',
+                maxLength: 640,
+                maxGraphemes: 64,
+              },
+              description: 'Additional non-inline tags describing this post.',
+            },
             createdAt: {
               type: 'string',
               format: 'datetime',
@@ -6761,6 +6771,7 @@ export const schemaDict = {
               refs: [
                 'lex:app.bsky.richtext.facet#mention',
                 'lex:app.bsky.richtext.facet#link',
+                'lex:app.bsky.richtext.facet#tag',
               ],
             },
           },
@@ -6788,6 +6799,18 @@ export const schemaDict = {
           },
         },
       },
+      tag: {
+        type: 'object',
+        description: 'A hashtag.',
+        required: ['tag'],
+        properties: {
+          tag: {
+            type: 'string',
+            maxLength: 640,
+            maxGraphemes: 64,
+          },
+        },
+      },
       byteSlice: {
         type: 'object',
         description:

diff --git a/packages/bsky/src/lexicon/types/app/bsky/feed/post.ts b/packages/bsky/src/lexicon/types/app/bsky/feed/post.ts
@@ -29,6 +29,8 @@ export interface Record {
   labels?:
     | ComAtprotoLabelDefs.SelfLabels
     | { $type: string; [k: string]: unknown }
+  /** Additional non-inline tags describing this post. */
+  tags?: string[]
   createdAt: string
   [k: string]: unknown
 }

diff --git a/packages/bsky/src/lexicon/types/app/bsky/richtext/facet.ts b/packages/bsky/src/lexicon/types/app/bsky/richtext/facet.ts
@@ -8,7 +8,7 @@ import { CID } from 'multiformats/cid'
 
 export interface Main {
   index: ByteSlice
-  features: (Mention | Link | { $type: string; [k: string]: unknown })[]
+  features: (Mention | Link | Tag | { $type: string; [k: string]: unknown })[]
   [k: string]: unknown
 }
 
@@ -61,6 +61,22 @@ export function validateLink(v: unknown): ValidationResult {
   return lexicons.validate('app.bsky.richtext.facet#link', v)
 }
 
+/** A hashtag. */
+export interface Tag {
+  tag: string
+  [k: string]: unknown
+}
+
+export function isTag(v: unknown): v is Tag {
+  return (
+    isObj(v) && hasProp(v, '$type') && v.$type === 'app.bsky.richtext.facet#tag'
+  )
+}
+
+export function validateTag(v: unknown): ValidationResult {
+  return lexicons.validate('app.bsky.richtext.facet#tag', v)
+}
+
 /** A text segment. Start is inclusive, end is exclusive. Indices are for utf8-encoded strings. */
 export interface ByteSlice {
   byteStart: number

diff --git a/packages/bsky/src/services/feed/index.ts b/packages/bsky/src/services/feed/index.ts
@@ -147,6 +147,7 @@ export class FeedService {
         'post_agg.likeCount as likeCount',
         'post_agg.repostCount as repostCount',
         'post_agg.replyCount as replyCount',
+        'post.tags as tags',
         db
           .selectFrom('repost')
           .if(!viewer, (q) => q.where(noMatch))

diff --git a/packages/bsky/src/services/indexing/plugins/post.ts b/packages/bsky/src/services/indexing/plugins/post.ts
@@ -76,6 +76,9 @@ const insertFn = async (
     langs: obj.langs?.length
       ? sql<string[]>`${JSON.stringify(obj.langs)}` // sidesteps kysely's array serialization, which is non-jsonb
       : null,
+    tags: obj.tags?.length
+      ? sql<string[]>`${JSON.stringify(obj.tags)}` // sidesteps kysely's array serialization, which is non-jsonb
+      : null,
     indexedAt: timestamp,
   }
   const [insertedPost] = await Promise.all([

diff --git a/packages/bsky/tests/views/posts.test.ts b/packages/bsky/tests/views/posts.test.ts
@@ -1,4 +1,4 @@
-import AtpAgent from '@atproto/api'
+import AtpAgent, { AppBskyFeedPost } from '@atproto/api'
 import { TestNetwork } from '@atproto/dev-env'
 import { forSnapshot, stripViewerFromPost } from '../_util'
 import { SeedClient } from '../seeds/client'
@@ -7,14 +7,15 @@ import basicSeed from '../seeds/basic'
 describe('pds posts views', () => {
   let network: TestNetwork
   let agent: AtpAgent
+  let pdsAgent: AtpAgent
   let sc: SeedClient
 
   beforeAll(async () => {
     network = await TestNetwork.create({
       dbPostgresSchema: 'bsky_views_posts',
     })
     agent = network.bsky.getClient()
-    const pdsAgent = network.pds.getClient()
+    pdsAgent = network.pds.getClient()
     sc = new SeedClient(pdsAgent)
     await basicSeed(sc)
     await network.processAll()
@@ -83,4 +84,27 @@ describe('pds posts views', () => {
     ].sort()
     expect(receivedUris).toEqual(expected)
   })
+
+  it('allows for creating posts with tags', async () => {
+    const post: AppBskyFeedPost.Record = {
+      text: 'hello world',
+      tags: ['javascript', 'hehe'],
+      createdAt: new Date().toISOString(),
+    }
+
+    const { uri } = await pdsAgent.api.app.bsky.feed.post.create(
+      { repo: sc.dids.alice },
+      post,
+      sc.getHeaders(sc.dids.alice),
+    )
+
+    await network.processAll()
+    await network.bsky.processAll()
+
+    const { data } = await agent.api.app.bsky.feed.getPosts({ uris: [uri] })
+
+    expect(data.posts.length).toBe(1)
+    // @ts-ignore we know it's a post record
+    expect(data.posts[0].record.tags).toEqual(['javascript', 'hehe'])
+  })
 })
diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts
@@ -5622,6 +5622,16 @@ export const schemaDict = {
               type: 'union',
               refs: ['lex:com.atproto.label.defs#selfLabels'],
             },
+            tags: {
+              type: 'array',
+              maxLength: 8,
+              items: {
+                type: 'string',
+                maxLength: 640,
+                maxGraphemes: 64,
+              },
+              description: 'Additional non-inline tags describing this post.',
+            },
             createdAt: {
               type: 'string',
               format: 'datetime',
@@ -6761,6 +6771,7 @@ export const schemaDict = {
               refs: [
                 'lex:app.bsky.richtext.facet#mention',
                 'lex:app.bsky.richtext.facet#link',
+                'lex:app.bsky.richtext.facet#tag',
               ],
             },
           },
@@ -6788,6 +6799,18 @@ export const schemaDict = {
           },
         },
       },
+      tag: {
+        type: 'object',
+        description: 'A hashtag.',
+        required: ['tag'],
+        properties: {
+          tag: {
+            type: 'string',
+            maxLength: 640,
+            maxGraphemes: 64,
+          },
+        },
+      },
       byteSlice: {
         type: 'object',
         description:

diff --git a/packages/pds/src/lexicon/types/app/bsky/feed/post.ts b/packages/pds/src/lexicon/types/app/bsky/feed/post.ts
@@ -29,6 +29,8 @@ export interface Record {
   labels?:
     | ComAtprotoLabelDefs.SelfLabels
     | { $type: string; [k: string]: unknown }
+  /** Additional non-inline tags describing this post. */
+  tags?: string[]
   createdAt: string
   [k: string]: unknown
 }

diff --git a/packages/pds/src/lexicon/types/app/bsky/richtext/facet.ts b/packages/pds/src/lexicon/types/app/bsky/richtext/facet.ts
@@ -8,7 +8,7 @@ import { CID } from 'multiformats/cid'
 
 export interface Main {
   index: ByteSlice
-  features: (Mention | Link | { $type: string; [k: string]: unknown })[]
+  features: (Mention | Link | Tag | { $type: string; [k: string]: unknown })[]
   [k: string]: unknown
 }
 
@@ -61,6 +61,22 @@ export function validateLink(v: unknown): ValidationResult {
   return lexicons.validate('app.bsky.richtext.facet#link', v)
 }
 
+/** A hashtag. */
+export interface Tag {
+  tag: string
+  [k: string]: unknown
+}
+
+export function isTag(v: unknown): v is Tag {
+  return (
+    isObj(v) && hasProp(v, '$type') && v.$type === 'app.bsky.richtext.facet#tag'
+  )
+}
+
+export function validateTag(v: unknown): ValidationResult {
+  return lexicons.validate('app.bsky.richtext.facet#tag', v)
+}
+
 /** A text segment. Start is inclusive, end is exclusive. Indices are for utf8-encoded strings. */
 export interface ByteSlice {
   byteStart: number