From 4d60322dacc56f48337610fd9409cc856e85531b Mon Sep 17 00:00:00 2001 From: bnewbold Date: Fri, 22 Sep 2023 19:08:52 -0700 Subject: [PATCH 1/6] auto-moderator tweaks: pass along record URI, create report for takedown action (#1643) * auto-moderator: include record URI in abyss requests * auto-moderator: log attempt at hard takedown; create report as well The motivation is to flag the event to mod team, and to make it easier to confirm that takedown took place. * auto-mod: typo fix * auto-mod: bugfixes * bsky: always create auto-mod report locally, not pushAgent (if possible) * bsky: fix auto-mod build * bsky: URL-encode scanBlob call --- packages/bsky/src/auto-moderator/abyss.ts | 32 ++++++++----- packages/bsky/src/auto-moderator/index.ts | 45 ++++++++++++++++--- .../tests/auto-moderator/takedowns.test.ts | 3 +- 3 files changed, 62 insertions(+), 18 deletions(-) diff --git a/packages/bsky/src/auto-moderator/abyss.ts b/packages/bsky/src/auto-moderator/abyss.ts index fb9ee2c4e98..4799c7067a5 100644 --- a/packages/bsky/src/auto-moderator/abyss.ts +++ b/packages/bsky/src/auto-moderator/abyss.ts @@ -1,5 +1,6 @@ import axios from 'axios' import { CID } from 'multiformats/cid' +import { AtUri } from '@atproto/syntax' import * as ui8 from 'uint8arrays' import { resolveBlob } from '../api/blob-resolver' import { retryHttp } from '../util/retry' @@ -8,7 +9,7 @@ import { IdResolver } from '@atproto/identity' import { labelerLogger as log } from '../logger' export interface ImageFlagger { - scanImage(did: string, cid: CID): Promise + scanImage(did: string, cid: CID, uri: AtUri): Promise } export class Abyss implements ImageFlagger { @@ -22,11 +23,11 @@ export class Abyss implements ImageFlagger { this.auth = basicAuth(this.password) } - async scanImage(did: string, cid: CID): Promise { + async scanImage(did: string, cid: CID, uri: AtUri): Promise { const start = Date.now() const res = await retryHttp(async () => { try { - return await this.makeReq(did, cid) + return await this.makeReq(did, cid, uri) } catch (err) { log.warn({ err, did, cid: cid.toString() }, 'abyss request failed') throw err @@ -39,20 +40,24 @@ export class Abyss implements ImageFlagger { return this.parseRes(res) } - async makeReq(did: string, cid: CID): Promise { + async makeReq(did: string, cid: CID, uri: AtUri): Promise { const { stream, contentType } = await resolveBlob( did, cid, this.ctx.db, this.ctx.idResolver, ) - const { data } = await axios.post(this.getReqUrl({ did }), stream, { - headers: { - 'Content-Type': contentType, - authorization: this.auth, + const { data } = await axios.post( + this.getReqUrl({ did, uri: uri.toString() }), + stream, + { + headers: { + 'Content-Type': contentType, + authorization: this.auth, + }, + timeout: 10000, }, - timeout: 10000, - }) + ) return data } @@ -69,8 +74,11 @@ export class Abyss implements ImageFlagger { return labels } - getReqUrl(params: { did: string }) { - return `${this.endpoint}/xrpc/com.atproto.unspecced.scanBlob?did=${params.did}` + getReqUrl(params: { did: string; uri: string }) { + const search = new URLSearchParams(params) + return `${ + this.endpoint + }/xrpc/com.atproto.unspecced.scanBlob?${search.toString()}` } } diff --git a/packages/bsky/src/auto-moderator/index.ts b/packages/bsky/src/auto-moderator/index.ts index 85cc529bce1..1be099759f1 100644 --- a/packages/bsky/src/auto-moderator/index.ts +++ b/packages/bsky/src/auto-moderator/index.ts @@ -18,7 +18,10 @@ import { ImageUriBuilder } from '../image/uri' import { ImageInvalidator } from '../image/invalidator' import { Abyss } from './abyss' import { FuzzyMatcher, TextFlagger } from './fuzzy-matcher' -import { REASONOTHER } from '../lexicon/types/com/atproto/moderation/defs' +import { + REASONOTHER, + REASONVIOLATION, +} from '../lexicon/types/com/atproto/moderation/defs' export class AutoModerator { public pushAgent?: AtpAgent @@ -172,7 +175,7 @@ export class AutoModerator { async checkImgForTakedown(uri: AtUri, recordCid: CID, imgCids: CID[]) { if (imgCids.length < 0) return const results = await Promise.all( - imgCids.map((cid) => this.imageFlagger?.scanImage(uri.host, cid)), + imgCids.map((cid) => this.imageFlagger?.scanImage(uri.host, cid, uri)), ) const takedownCids: CID[] = [] for (let i = 0; i < results.length; i++) { @@ -207,7 +210,39 @@ export class AutoModerator { takedownCids: CID[], labels: string[], ) { - const reason = `automated takedown for labels: ${labels.join(', ')}` + const reportReason = `automated takedown (${labels.join( + ', ', + )}). account needs review and possibly additional action` + const takedownReason = `automated takedown for labels: ${labels.join(', ')}` + log.warn( + { + uri: uri.toString(), + blobCids: takedownCids, + labels, + }, + 'hard takedown of record (and blobs) based on auto-matching', + ) + + if (this.services.moderation) { + await this.ctx.db.transaction(async (dbTxn) => { + // directly/locally create report, even if we use pushAgent for the takedown. don't have acctual account credentials for pushAgent, only admin auth + if (!this.services.moderation) { + // checked above, outside the transaction + return + } + const modSrvc = this.services.moderation(dbTxn) + await modSrvc.report({ + reportedBy: this.ctx.cfg.labelerDid, + reasonType: REASONVIOLATION, + subject: { + uri: uri, + cid: recordCid, + }, + reason: reportReason, + }) + }) + } + if (this.pushAgent) { await this.pushAgent.com.atproto.admin.takeModerationAction({ action: 'com.atproto.admin.defs#takedown', @@ -217,7 +252,7 @@ export class AutoModerator { cid: recordCid.toString(), }, subjectBlobCids: takedownCids.map((c) => c.toString()), - reason, + reason: takedownReason, createdBy: this.ctx.cfg.labelerDid, }) } else { @@ -230,7 +265,7 @@ export class AutoModerator { action: 'com.atproto.admin.defs#takedown', subject: { uri, cid: recordCid }, subjectBlobCids: takedownCids, - reason, + reason: takedownReason, createdBy: this.ctx.cfg.labelerDid, }) await modSrvc.takedownRecord({ diff --git a/packages/bsky/tests/auto-moderator/takedowns.test.ts b/packages/bsky/tests/auto-moderator/takedowns.test.ts index 27b0c986115..32c5c941642 100644 --- a/packages/bsky/tests/auto-moderator/takedowns.test.ts +++ b/packages/bsky/tests/auto-moderator/takedowns.test.ts @@ -7,6 +7,7 @@ import { TestNetwork } from '@atproto/dev-env' import { ImageRef, SeedClient } from '../seeds/client' import usersSeed from '../seeds/users' import { CID } from 'multiformats/cid' +import { AtUri } from '@atproto/syntax' import { ImageFlagger } from '../../src/auto-moderator/abyss' import { ImageInvalidator } from '../../src/image/invalidator' import { sha256 } from '@atproto/crypto' @@ -157,7 +158,7 @@ class TestInvalidator implements ImageInvalidator { } class TestFlagger implements ImageFlagger { - async scanImage(_did: string, cid: CID): Promise { + async scanImage(_did: string, cid: CID, _uri: AtUri): Promise { if (cid.equals(badCid1)) { return ['kill-it'] } else if (cid.equals(badCid2)) { From d703bc36e6637750ec20df4c8d0c015c7bbd04aa Mon Sep 17 00:00:00 2001 From: devin ivy Date: Mon, 25 Sep 2023 10:27:48 +0200 Subject: [PATCH 2/6] Clear follow viewer state when blocking (#1659) * clear follow viewer state when blocking * tidy --- packages/bsky/src/services/actor/views.ts | 20 +++++++++++++---- .../__snapshots__/block-lists.test.ts.snap | 1 - packages/bsky/tests/views/blocks.test.ts | 22 +++++++++++++++++-- .../tests/views/suggested-follows.test.ts | 2 +- .../proxied/__snapshots__/views.test.ts.snap | 10 +++------ 5 files changed, 40 insertions(+), 15 deletions(-) diff --git a/packages/bsky/src/services/actor/views.ts b/packages/bsky/src/services/actor/views.ts index 80652599f80..ec39805c76d 100644 --- a/packages/bsky/src/services/actor/views.ts +++ b/packages/bsky/src/services/actor/views.ts @@ -194,8 +194,14 @@ export class ActorViews { mutedByList, blockedBy: !!bam.blockedBy([viewer, did]), blocking: bam.blocking([viewer, did]) ?? undefined, - following: prof?.viewerFollowing || undefined, - followedBy: prof?.viewerFollowedBy || undefined, + following: + prof?.viewerFollowing && !bam.block([viewer, did]) + ? prof.viewerFollowing + : undefined, + followedBy: + prof?.viewerFollowedBy && !bam.block([viewer, did]) + ? prof.viewerFollowedBy + : undefined, } : undefined, labels: [...actorLabels, ...selfLabels], @@ -314,8 +320,14 @@ export class ActorViews { mutedByList, blockedBy: !!bam.blockedBy([viewer, did]), blocking: bam.blocking([viewer, did]) ?? undefined, - following: prof?.viewerFollowing || undefined, - followedBy: prof?.viewerFollowedBy || undefined, + following: + prof?.viewerFollowing && !bam.block([viewer, did]) + ? prof.viewerFollowing + : undefined, + followedBy: + prof?.viewerFollowedBy && !bam.block([viewer, did]) + ? prof.viewerFollowedBy + : undefined, } : undefined, labels: [...actorLabels, ...selfLabels], diff --git a/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap b/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap index 009095947c2..364ad4b7d63 100644 --- a/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap +++ b/packages/bsky/tests/views/__snapshots__/block-lists.test.ts.snap @@ -512,7 +512,6 @@ Object { "viewer": Object { "blockedBy": false, "blocking": "record(0)", - "following": "record(4)", "muted": false, }, }, diff --git a/packages/bsky/tests/views/blocks.test.ts b/packages/bsky/tests/views/blocks.test.ts index 30b46e78ab5..0109b93f82a 100644 --- a/packages/bsky/tests/views/blocks.test.ts +++ b/packages/bsky/tests/views/blocks.test.ts @@ -191,17 +191,35 @@ describe('pds views with blocking', () => { { actor: dan }, { headers: await network.serviceHeaders(carol) }, ) - expect(resCarol.data.viewer?.blocking).toBeUndefined + expect(resCarol.data.viewer?.blocking).toBeUndefined() expect(resCarol.data.viewer?.blockedBy).toBe(true) const resDan = await agent.api.app.bsky.actor.getProfile( { actor: carol }, { headers: await network.serviceHeaders(dan) }, ) - expect(resDan.data.viewer?.blocking).toBeDefined + expect(resDan.data.viewer?.blocking).toBeDefined() expect(resDan.data.viewer?.blockedBy).toBe(false) }) + it('unsets viewer follow state when blocked', async () => { + // there are follows between carol and dan + const { data: profile } = await agent.api.app.bsky.actor.getProfile( + { actor: carol }, + { headers: await network.serviceHeaders(dan) }, + ) + expect(profile.viewer?.following).toBeUndefined() + expect(profile.viewer?.followedBy).toBeUndefined() + const { data: result } = await agent.api.app.bsky.graph.getBlocks( + {}, + { headers: await network.serviceHeaders(dan) }, + ) + const blocked = result.blocks.find((block) => block.did === carol) + expect(blocked).toBeDefined() + expect(blocked?.viewer?.following).toBeUndefined() + expect(blocked?.viewer?.followedBy).toBeUndefined() + }) + it('returns block status on getProfiles', async () => { const resCarol = await agent.api.app.bsky.actor.getProfiles( { actors: [alice, dan] }, diff --git a/packages/bsky/tests/views/suggested-follows.test.ts b/packages/bsky/tests/views/suggested-follows.test.ts index 6a2f3ebe1d7..1d8cb5a91ba 100644 --- a/packages/bsky/tests/views/suggested-follows.test.ts +++ b/packages/bsky/tests/views/suggested-follows.test.ts @@ -11,7 +11,7 @@ describe('suggested follows', () => { beforeAll(async () => { network = await TestNetwork.create({ - dbPostgresSchema: 'bsky_views_suggestions', + dbPostgresSchema: 'bsky_views_suggested_follows', }) agent = network.bsky.getClient() pdsAgent = network.pds.getClient() diff --git a/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap b/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap index d6707292de4..994589ff0b2 100644 --- a/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap +++ b/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap @@ -2959,8 +2959,6 @@ Object { "viewer": Object { "blockedBy": false, "blocking": "record(0)", - "followedBy": "record(2)", - "following": "record(1)", "muted": false, }, }, @@ -2977,7 +2975,7 @@ Object { "cts": "1970-01-01T00:00:00.000Z", "neg": false, "src": "user(1)", - "uri": "record(6)", + "uri": "record(2)", "val": "self-label-a", }, Object { @@ -2985,15 +2983,13 @@ Object { "cts": "1970-01-01T00:00:00.000Z", "neg": false, "src": "user(1)", - "uri": "record(6)", + "uri": "record(2)", "val": "self-label-b", }, ], "viewer": Object { "blockedBy": false, - "blocking": "record(3)", - "followedBy": "record(5)", - "following": "record(4)", + "blocking": "record(1)", "muted": false, }, }, From d96f7d9b84c6fbab9711059c8584a77d892dcedd Mon Sep 17 00:00:00 2001 From: Eric Bailey Date: Mon, 25 Sep 2023 11:14:57 -0500 Subject: [PATCH 3/6] add `tags` to posts (#1637) * add tags to post lex * kiss * add richtext facet and validation attrs * add tag validation attrs to post * codegen * add maxLength for tags, add description * validate post tags on write * add test * handle tags in indexer * add tags to postView, codegen * return tags on post thread view * format * revert formatting change to docs * use establish validation pattern * add changeset (cherry picked from commit fcb6fe7c26144662f791c7900afcd84c7bf1be6b) * remove tags from postView, codegen * remove tags from thread view * revert unused changes --- .changeset/three-snakes-turn.md | 7 +++ lexicons/app/bsky/feed/post.json | 6 +++ lexicons/app/bsky/richtext/facet.json | 10 ++++- packages/api/src/client/lexicons.ts | 23 ++++++++++ .../src/client/types/app/bsky/feed/post.ts | 2 + .../client/types/app/bsky/richtext/facet.ts | 18 +++++++- .../20230920T213858047Z-add-tags-to-post.ts | 9 ++++ packages/bsky/src/db/migrations/index.ts | 1 + packages/bsky/src/db/tables/post.ts | 1 + packages/bsky/src/lexicon/lexicons.ts | 23 ++++++++++ .../src/lexicon/types/app/bsky/feed/post.ts | 2 + .../lexicon/types/app/bsky/richtext/facet.ts | 18 +++++++- packages/bsky/src/services/feed/index.ts | 1 + .../src/services/indexing/plugins/post.ts | 3 ++ packages/bsky/tests/views/posts.test.ts | 28 +++++++++++- packages/pds/src/lexicon/lexicons.ts | 23 ++++++++++ .../src/lexicon/types/app/bsky/feed/post.ts | 2 + .../lexicon/types/app/bsky/richtext/facet.ts | 18 +++++++- packages/pds/src/repo/prepare.ts | 2 + packages/pds/tests/create-post.test.ts | 45 +++++++++++++++++++ 20 files changed, 236 insertions(+), 6 deletions(-) create mode 100644 .changeset/three-snakes-turn.md create mode 100644 packages/bsky/src/db/migrations/20230920T213858047Z-add-tags-to-post.ts create mode 100644 packages/pds/tests/create-post.test.ts diff --git a/.changeset/three-snakes-turn.md b/.changeset/three-snakes-turn.md new file mode 100644 index 00000000000..cd086115761 --- /dev/null +++ b/.changeset/three-snakes-turn.md @@ -0,0 +1,7 @@ +--- +'@atproto/bsky': patch +'@atproto/api': patch +'@atproto/pds': patch +--- + +Introduce general support for tags on posts diff --git a/lexicons/app/bsky/feed/post.json b/lexicons/app/bsky/feed/post.json index 5622b5cfd50..b21f01b6050 100644 --- a/lexicons/app/bsky/feed/post.json +++ b/lexicons/app/bsky/feed/post.json @@ -38,6 +38,12 @@ "type": "union", "refs": ["com.atproto.label.defs#selfLabels"] }, + "tags": { + "type": "array", + "maxLength": 8, + "items": { "type": "string", "maxLength": 640, "maxGraphemes": 64 }, + "description": "Additional non-inline tags describing this post." + }, "createdAt": { "type": "string", "format": "datetime" } } } diff --git a/lexicons/app/bsky/richtext/facet.json b/lexicons/app/bsky/richtext/facet.json index 9addf2f34b7..ea8f2cba288 100644 --- a/lexicons/app/bsky/richtext/facet.json +++ b/lexicons/app/bsky/richtext/facet.json @@ -9,7 +9,7 @@ "index": { "type": "ref", "ref": "#byteSlice" }, "features": { "type": "array", - "items": { "type": "union", "refs": ["#mention", "#link"] } + "items": { "type": "union", "refs": ["#mention", "#link", "#tag"] } } } }, @@ -29,6 +29,14 @@ "uri": { "type": "string", "format": "uri" } } }, + "tag": { + "type": "object", + "description": "A hashtag.", + "required": ["tag"], + "properties": { + "tag": { "type": "string", "maxLength": 640, "maxGraphemes": 64 } + } + }, "byteSlice": { "type": "object", "description": "A text segment. Start is inclusive, end is exclusive. Indices are for utf8-encoded strings.", diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index 6b9b7b7f14f..a5cbf08d608 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -5622,6 +5622,16 @@ export const schemaDict = { type: 'union', refs: ['lex:com.atproto.label.defs#selfLabels'], }, + tags: { + type: 'array', + maxLength: 8, + items: { + type: 'string', + maxLength: 640, + maxGraphemes: 64, + }, + description: 'Additional non-inline tags describing this post.', + }, createdAt: { type: 'string', format: 'datetime', @@ -6761,6 +6771,7 @@ export const schemaDict = { refs: [ 'lex:app.bsky.richtext.facet#mention', 'lex:app.bsky.richtext.facet#link', + 'lex:app.bsky.richtext.facet#tag', ], }, }, @@ -6788,6 +6799,18 @@ export const schemaDict = { }, }, }, + tag: { + type: 'object', + description: 'A hashtag.', + required: ['tag'], + properties: { + tag: { + type: 'string', + maxLength: 640, + maxGraphemes: 64, + }, + }, + }, byteSlice: { type: 'object', description: diff --git a/packages/api/src/client/types/app/bsky/feed/post.ts b/packages/api/src/client/types/app/bsky/feed/post.ts index 1e326692640..a3299e19035 100644 --- a/packages/api/src/client/types/app/bsky/feed/post.ts +++ b/packages/api/src/client/types/app/bsky/feed/post.ts @@ -29,6 +29,8 @@ export interface Record { labels?: | ComAtprotoLabelDefs.SelfLabels | { $type: string; [k: string]: unknown } + /** Additional non-inline tags describing this post. */ + tags?: string[] createdAt: string [k: string]: unknown } diff --git a/packages/api/src/client/types/app/bsky/richtext/facet.ts b/packages/api/src/client/types/app/bsky/richtext/facet.ts index cea86685a0f..96573bb06fe 100644 --- a/packages/api/src/client/types/app/bsky/richtext/facet.ts +++ b/packages/api/src/client/types/app/bsky/richtext/facet.ts @@ -8,7 +8,7 @@ import { CID } from 'multiformats/cid' export interface Main { index: ByteSlice - features: (Mention | Link | { $type: string; [k: string]: unknown })[] + features: (Mention | Link | Tag | { $type: string; [k: string]: unknown })[] [k: string]: unknown } @@ -61,6 +61,22 @@ export function validateLink(v: unknown): ValidationResult { return lexicons.validate('app.bsky.richtext.facet#link', v) } +/** A hashtag. */ +export interface Tag { + tag: string + [k: string]: unknown +} + +export function isTag(v: unknown): v is Tag { + return ( + isObj(v) && hasProp(v, '$type') && v.$type === 'app.bsky.richtext.facet#tag' + ) +} + +export function validateTag(v: unknown): ValidationResult { + return lexicons.validate('app.bsky.richtext.facet#tag', v) +} + /** A text segment. Start is inclusive, end is exclusive. Indices are for utf8-encoded strings. */ export interface ByteSlice { byteStart: number diff --git a/packages/bsky/src/db/migrations/20230920T213858047Z-add-tags-to-post.ts b/packages/bsky/src/db/migrations/20230920T213858047Z-add-tags-to-post.ts new file mode 100644 index 00000000000..9d4e5bd4cfb --- /dev/null +++ b/packages/bsky/src/db/migrations/20230920T213858047Z-add-tags-to-post.ts @@ -0,0 +1,9 @@ +import { Kysely } from 'kysely' + +export async function up(db: Kysely): Promise { + await db.schema.alterTable('post').addColumn('tags', 'jsonb').execute() +} + +export async function down(db: Kysely): Promise { + await db.schema.alterTable('post').dropColumn('tags').execute() +} diff --git a/packages/bsky/src/db/migrations/index.ts b/packages/bsky/src/db/migrations/index.ts index bf18d8dd15b..9e8bfe9cf7f 100644 --- a/packages/bsky/src/db/migrations/index.ts +++ b/packages/bsky/src/db/migrations/index.ts @@ -28,3 +28,4 @@ export * as _20230817T195936007Z from './20230817T195936007Z-native-notification export * as _20230830T205507322Z from './20230830T205507322Z-suggested-feeds' export * as _20230904T211011773Z from './20230904T211011773Z-block-lists' export * as _20230906T222220386Z from './20230906T222220386Z-thread-gating' +export * as _20230920T213858047Z from './20230920T213858047Z-add-tags-to-post' diff --git a/packages/bsky/src/db/tables/post.ts b/packages/bsky/src/db/tables/post.ts index c627efa39e7..6c01b76c8e0 100644 --- a/packages/bsky/src/db/tables/post.ts +++ b/packages/bsky/src/db/tables/post.ts @@ -12,6 +12,7 @@ export interface Post { replyParent: string | null replyParentCid: string | null langs: string[] | null + tags: string[] | null invalidReplyRoot: boolean | null violatesThreadGate: boolean | null createdAt: string diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index 6b9b7b7f14f..a5cbf08d608 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -5622,6 +5622,16 @@ export const schemaDict = { type: 'union', refs: ['lex:com.atproto.label.defs#selfLabels'], }, + tags: { + type: 'array', + maxLength: 8, + items: { + type: 'string', + maxLength: 640, + maxGraphemes: 64, + }, + description: 'Additional non-inline tags describing this post.', + }, createdAt: { type: 'string', format: 'datetime', @@ -6761,6 +6771,7 @@ export const schemaDict = { refs: [ 'lex:app.bsky.richtext.facet#mention', 'lex:app.bsky.richtext.facet#link', + 'lex:app.bsky.richtext.facet#tag', ], }, }, @@ -6788,6 +6799,18 @@ export const schemaDict = { }, }, }, + tag: { + type: 'object', + description: 'A hashtag.', + required: ['tag'], + properties: { + tag: { + type: 'string', + maxLength: 640, + maxGraphemes: 64, + }, + }, + }, byteSlice: { type: 'object', description: diff --git a/packages/bsky/src/lexicon/types/app/bsky/feed/post.ts b/packages/bsky/src/lexicon/types/app/bsky/feed/post.ts index 8942bc724cd..93870b4452d 100644 --- a/packages/bsky/src/lexicon/types/app/bsky/feed/post.ts +++ b/packages/bsky/src/lexicon/types/app/bsky/feed/post.ts @@ -29,6 +29,8 @@ export interface Record { labels?: | ComAtprotoLabelDefs.SelfLabels | { $type: string; [k: string]: unknown } + /** Additional non-inline tags describing this post. */ + tags?: string[] createdAt: string [k: string]: unknown } diff --git a/packages/bsky/src/lexicon/types/app/bsky/richtext/facet.ts b/packages/bsky/src/lexicon/types/app/bsky/richtext/facet.ts index a7369ee8d57..2c5b2d723a9 100644 --- a/packages/bsky/src/lexicon/types/app/bsky/richtext/facet.ts +++ b/packages/bsky/src/lexicon/types/app/bsky/richtext/facet.ts @@ -8,7 +8,7 @@ import { CID } from 'multiformats/cid' export interface Main { index: ByteSlice - features: (Mention | Link | { $type: string; [k: string]: unknown })[] + features: (Mention | Link | Tag | { $type: string; [k: string]: unknown })[] [k: string]: unknown } @@ -61,6 +61,22 @@ export function validateLink(v: unknown): ValidationResult { return lexicons.validate('app.bsky.richtext.facet#link', v) } +/** A hashtag. */ +export interface Tag { + tag: string + [k: string]: unknown +} + +export function isTag(v: unknown): v is Tag { + return ( + isObj(v) && hasProp(v, '$type') && v.$type === 'app.bsky.richtext.facet#tag' + ) +} + +export function validateTag(v: unknown): ValidationResult { + return lexicons.validate('app.bsky.richtext.facet#tag', v) +} + /** A text segment. Start is inclusive, end is exclusive. Indices are for utf8-encoded strings. */ export interface ByteSlice { byteStart: number diff --git a/packages/bsky/src/services/feed/index.ts b/packages/bsky/src/services/feed/index.ts index f955979e81e..dab9673d9db 100644 --- a/packages/bsky/src/services/feed/index.ts +++ b/packages/bsky/src/services/feed/index.ts @@ -147,6 +147,7 @@ export class FeedService { 'post_agg.likeCount as likeCount', 'post_agg.repostCount as repostCount', 'post_agg.replyCount as replyCount', + 'post.tags as tags', db .selectFrom('repost') .if(!viewer, (q) => q.where(noMatch)) diff --git a/packages/bsky/src/services/indexing/plugins/post.ts b/packages/bsky/src/services/indexing/plugins/post.ts index f57bc10179b..40835348f01 100644 --- a/packages/bsky/src/services/indexing/plugins/post.ts +++ b/packages/bsky/src/services/indexing/plugins/post.ts @@ -76,6 +76,9 @@ const insertFn = async ( langs: obj.langs?.length ? sql`${JSON.stringify(obj.langs)}` // sidesteps kysely's array serialization, which is non-jsonb : null, + tags: obj.tags?.length + ? sql`${JSON.stringify(obj.tags)}` // sidesteps kysely's array serialization, which is non-jsonb + : null, indexedAt: timestamp, } const [insertedPost] = await Promise.all([ diff --git a/packages/bsky/tests/views/posts.test.ts b/packages/bsky/tests/views/posts.test.ts index 99ec565dc59..6fa12a085df 100644 --- a/packages/bsky/tests/views/posts.test.ts +++ b/packages/bsky/tests/views/posts.test.ts @@ -1,4 +1,4 @@ -import AtpAgent from '@atproto/api' +import AtpAgent, { AppBskyFeedPost } from '@atproto/api' import { TestNetwork } from '@atproto/dev-env' import { forSnapshot, stripViewerFromPost } from '../_util' import { SeedClient } from '../seeds/client' @@ -7,6 +7,7 @@ import basicSeed from '../seeds/basic' describe('pds posts views', () => { let network: TestNetwork let agent: AtpAgent + let pdsAgent: AtpAgent let sc: SeedClient beforeAll(async () => { @@ -14,7 +15,7 @@ describe('pds posts views', () => { dbPostgresSchema: 'bsky_views_posts', }) agent = network.bsky.getClient() - const pdsAgent = network.pds.getClient() + pdsAgent = network.pds.getClient() sc = new SeedClient(pdsAgent) await basicSeed(sc) await network.processAll() @@ -83,4 +84,27 @@ describe('pds posts views', () => { ].sort() expect(receivedUris).toEqual(expected) }) + + it('allows for creating posts with tags', async () => { + const post: AppBskyFeedPost.Record = { + text: 'hello world', + tags: ['javascript', 'hehe'], + createdAt: new Date().toISOString(), + } + + const { uri } = await pdsAgent.api.app.bsky.feed.post.create( + { repo: sc.dids.alice }, + post, + sc.getHeaders(sc.dids.alice), + ) + + await network.processAll() + await network.bsky.processAll() + + const { data } = await agent.api.app.bsky.feed.getPosts({ uris: [uri] }) + + expect(data.posts.length).toBe(1) + // @ts-ignore we know it's a post record + expect(data.posts[0].record.tags).toEqual(['javascript', 'hehe']) + }) }) diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index 6b9b7b7f14f..a5cbf08d608 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -5622,6 +5622,16 @@ export const schemaDict = { type: 'union', refs: ['lex:com.atproto.label.defs#selfLabels'], }, + tags: { + type: 'array', + maxLength: 8, + items: { + type: 'string', + maxLength: 640, + maxGraphemes: 64, + }, + description: 'Additional non-inline tags describing this post.', + }, createdAt: { type: 'string', format: 'datetime', @@ -6761,6 +6771,7 @@ export const schemaDict = { refs: [ 'lex:app.bsky.richtext.facet#mention', 'lex:app.bsky.richtext.facet#link', + 'lex:app.bsky.richtext.facet#tag', ], }, }, @@ -6788,6 +6799,18 @@ export const schemaDict = { }, }, }, + tag: { + type: 'object', + description: 'A hashtag.', + required: ['tag'], + properties: { + tag: { + type: 'string', + maxLength: 640, + maxGraphemes: 64, + }, + }, + }, byteSlice: { type: 'object', description: diff --git a/packages/pds/src/lexicon/types/app/bsky/feed/post.ts b/packages/pds/src/lexicon/types/app/bsky/feed/post.ts index 8942bc724cd..93870b4452d 100644 --- a/packages/pds/src/lexicon/types/app/bsky/feed/post.ts +++ b/packages/pds/src/lexicon/types/app/bsky/feed/post.ts @@ -29,6 +29,8 @@ export interface Record { labels?: | ComAtprotoLabelDefs.SelfLabels | { $type: string; [k: string]: unknown } + /** Additional non-inline tags describing this post. */ + tags?: string[] createdAt: string [k: string]: unknown } diff --git a/packages/pds/src/lexicon/types/app/bsky/richtext/facet.ts b/packages/pds/src/lexicon/types/app/bsky/richtext/facet.ts index a7369ee8d57..2c5b2d723a9 100644 --- a/packages/pds/src/lexicon/types/app/bsky/richtext/facet.ts +++ b/packages/pds/src/lexicon/types/app/bsky/richtext/facet.ts @@ -8,7 +8,7 @@ import { CID } from 'multiformats/cid' export interface Main { index: ByteSlice - features: (Mention | Link | { $type: string; [k: string]: unknown })[] + features: (Mention | Link | Tag | { $type: string; [k: string]: unknown })[] [k: string]: unknown } @@ -61,6 +61,22 @@ export function validateLink(v: unknown): ValidationResult { return lexicons.validate('app.bsky.richtext.facet#link', v) } +/** A hashtag. */ +export interface Tag { + tag: string + [k: string]: unknown +} + +export function isTag(v: unknown): v is Tag { + return ( + isObj(v) && hasProp(v, '$type') && v.$type === 'app.bsky.richtext.facet#tag' + ) +} + +export function validateTag(v: unknown): ValidationResult { + return lexicons.validate('app.bsky.richtext.facet#tag', v) +} + /** A text segment. Start is inclusive, end is exclusive. Indices are for utf8-encoded strings. */ export interface ByteSlice { byteStart: number diff --git a/packages/pds/src/repo/prepare.ts b/packages/pds/src/repo/prepare.ts index 581701f1f01..2147ef552b6 100644 --- a/packages/pds/src/repo/prepare.ts +++ b/packages/pds/src/repo/prepare.ts @@ -299,6 +299,8 @@ function assertNoExplicitSlurs(rkey: string, record: RepoRecord) { } else if (isFeedGenerator(record)) { toCheck += ' ' + rkey toCheck += ' ' + record.displayName + } else if (isPost(record)) { + toCheck += record.tags?.join(' ') } if (hasExplicitSlur(toCheck)) { throw new InvalidRecordError('Unacceptable slur in record') diff --git a/packages/pds/tests/create-post.test.ts b/packages/pds/tests/create-post.test.ts new file mode 100644 index 00000000000..e2763981fb0 --- /dev/null +++ b/packages/pds/tests/create-post.test.ts @@ -0,0 +1,45 @@ +import AtpAgent, { AppBskyFeedPost, AtUri } from '@atproto/api' +import { runTestServer, TestServerInfo } from './_util' +import { SeedClient } from './seeds/client' +import basicSeed from './seeds/basic' + +describe('pds posts record creation', () => { + let server: TestServerInfo + let agent: AtpAgent + let sc: SeedClient + + beforeAll(async () => { + server = await runTestServer({ + dbPostgresSchema: 'views_posts', + }) + agent = new AtpAgent({ service: server.url }) + sc = new SeedClient(agent) + await basicSeed(sc) + await server.processAll() + }) + + afterAll(async () => { + await server.close() + }) + + it('allows for creating posts with tags', async () => { + const post: AppBskyFeedPost.Record = { + text: 'hello world', + tags: ['javascript', 'hehe'], + createdAt: new Date().toISOString(), + } + + const res = await agent.api.app.bsky.feed.post.create( + { repo: sc.dids.alice }, + post, + sc.getHeaders(sc.dids.alice), + ) + const { value: record } = await agent.api.app.bsky.feed.post.get({ + repo: sc.dids.alice, + rkey: new AtUri(res.uri).rkey, + }) + + expect(record).toBeTruthy() + expect(record.tags).toEqual(['javascript', 'hehe']) + }) +}) From 3ea11a1a79a7f6a54dba2a529f77903d8008a49d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 25 Sep 2023 11:17:11 -0500 Subject: [PATCH 4/6] Version packages (#1664) Co-authored-by: github-actions[bot] --- .changeset/three-snakes-turn.md | 7 ------- packages/api/CHANGELOG.md | 6 ++++++ packages/api/package.json | 2 +- packages/bsky/CHANGELOG.md | 9 +++++++++ packages/bsky/package.json | 2 +- packages/dev-env/CHANGELOG.md | 9 +++++++++ packages/dev-env/package.json | 2 +- packages/pds/CHANGELOG.md | 9 +++++++++ packages/pds/package.json | 2 +- 9 files changed, 37 insertions(+), 11 deletions(-) delete mode 100644 .changeset/three-snakes-turn.md diff --git a/.changeset/three-snakes-turn.md b/.changeset/three-snakes-turn.md deleted file mode 100644 index cd086115761..00000000000 --- a/.changeset/three-snakes-turn.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@atproto/bsky': patch -'@atproto/api': patch -'@atproto/pds': patch ---- - -Introduce general support for tags on posts diff --git a/packages/api/CHANGELOG.md b/packages/api/CHANGELOG.md index 79e6807686c..f438c52525d 100644 --- a/packages/api/CHANGELOG.md +++ b/packages/api/CHANGELOG.md @@ -1,5 +1,11 @@ # @atproto/api +## 0.6.17 + +### Patch Changes + +- [#1637](https://github.com/bluesky-social/atproto/pull/1637) [`d96f7d9b`](https://github.com/bluesky-social/atproto/commit/d96f7d9b84c6fbab9711059c8584a77d892dcedd) Thanks [@estrattonbailey](https://github.com/estrattonbailey)! - Introduce general support for tags on posts + ## 0.6.16 ### Patch Changes diff --git a/packages/api/package.json b/packages/api/package.json index 76cf7f1b8e7..8165fa63dd2 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/api", - "version": "0.6.16", + "version": "0.6.17", "license": "MIT", "description": "Client library for atproto and Bluesky", "keywords": [ diff --git a/packages/bsky/CHANGELOG.md b/packages/bsky/CHANGELOG.md index 8e449ba7cf0..0d973cc3d50 100644 --- a/packages/bsky/CHANGELOG.md +++ b/packages/bsky/CHANGELOG.md @@ -1,5 +1,14 @@ # @atproto/bsky +## 0.0.8 + +### Patch Changes + +- [#1637](https://github.com/bluesky-social/atproto/pull/1637) [`d96f7d9b`](https://github.com/bluesky-social/atproto/commit/d96f7d9b84c6fbab9711059c8584a77d892dcedd) Thanks [@estrattonbailey](https://github.com/estrattonbailey)! - Introduce general support for tags on posts + +- Updated dependencies [[`d96f7d9b`](https://github.com/bluesky-social/atproto/commit/d96f7d9b84c6fbab9711059c8584a77d892dcedd)]: + - @atproto/api@0.6.17 + ## 0.0.7 ### Patch Changes diff --git a/packages/bsky/package.json b/packages/bsky/package.json index 8195d27b539..ec203970552 100644 --- a/packages/bsky/package.json +++ b/packages/bsky/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/bsky", - "version": "0.0.7", + "version": "0.0.8", "license": "MIT", "description": "Reference implementation of app.bsky App View (Bluesky API)", "keywords": [ diff --git a/packages/dev-env/CHANGELOG.md b/packages/dev-env/CHANGELOG.md index 38a56abaa99..130a330007b 100644 --- a/packages/dev-env/CHANGELOG.md +++ b/packages/dev-env/CHANGELOG.md @@ -1,5 +1,14 @@ # @atproto/dev-env +## 0.2.8 + +### Patch Changes + +- Updated dependencies [[`d96f7d9b`](https://github.com/bluesky-social/atproto/commit/d96f7d9b84c6fbab9711059c8584a77d892dcedd)]: + - @atproto/bsky@0.0.8 + - @atproto/api@0.6.17 + - @atproto/pds@0.1.17 + ## 0.2.7 ### Patch Changes diff --git a/packages/dev-env/package.json b/packages/dev-env/package.json index e688319dcb1..d945103437e 100644 --- a/packages/dev-env/package.json +++ b/packages/dev-env/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/dev-env", - "version": "0.2.7", + "version": "0.2.8", "license": "MIT", "description": "Local development environment helper for atproto development", "keywords": [ diff --git a/packages/pds/CHANGELOG.md b/packages/pds/CHANGELOG.md index 17f13cf1c85..3a450244149 100644 --- a/packages/pds/CHANGELOG.md +++ b/packages/pds/CHANGELOG.md @@ -1,5 +1,14 @@ # @atproto/pds +## 0.1.17 + +### Patch Changes + +- [#1637](https://github.com/bluesky-social/atproto/pull/1637) [`d96f7d9b`](https://github.com/bluesky-social/atproto/commit/d96f7d9b84c6fbab9711059c8584a77d892dcedd) Thanks [@estrattonbailey](https://github.com/estrattonbailey)! - Introduce general support for tags on posts + +- Updated dependencies [[`d96f7d9b`](https://github.com/bluesky-social/atproto/commit/d96f7d9b84c6fbab9711059c8584a77d892dcedd)]: + - @atproto/api@0.6.17 + ## 0.1.16 ### Patch Changes diff --git a/packages/pds/package.json b/packages/pds/package.json index e227bd4c4c9..de690dad916 100644 --- a/packages/pds/package.json +++ b/packages/pds/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/pds", - "version": "0.1.16", + "version": "0.1.17", "license": "MIT", "description": "Reference implementation of atproto Personal Data Server (PDS)", "keywords": [ From caef7c183206b2b3384e3fe2dbee6bc953c27d04 Mon Sep 17 00:00:00 2001 From: bnewbold Date: Mon, 25 Sep 2023 11:18:34 -0700 Subject: [PATCH 5/6] Makefile: run code formatting after codegen (#1660) --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index f8c36ce2bb0..f9e90a86a44 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,8 @@ codegen: ## Re-generate packages from lexicon/ files cd packages/api; pnpm run codegen cd packages/pds; pnpm run codegen cd packages/bsky; pnpm run codegen + # clean up codegen output + pnpm format .PHONY: lint lint: ## Run style checks and verify syntax From 11bf4d302e8ac9a5c5d627268433b44ecf2870b6 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Mon, 25 Sep 2023 14:17:44 -0500 Subject: [PATCH 6/6] Reverse order of blocks from sync.getRepo (#1665) * reverse order of blocks from sync.getRepo * write to car while fetching next page --- packages/pds/src/sql-repo-storage.ts | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/packages/pds/src/sql-repo-storage.ts b/packages/pds/src/sql-repo-storage.ts index a7b6a5ae1ea..7522e325bfa 100644 --- a/packages/pds/src/sql-repo-storage.ts +++ b/packages/pds/src/sql-repo-storage.ts @@ -213,8 +213,22 @@ export class SqlRepoStorage extends ReadableBlockstore implements RepoStorage { } return writeCarStream(root, async (car) => { let cursor: RevCursor | undefined = undefined + const writeRows = async ( + rows: { cid: string; content: Uint8Array }[], + ) => { + for (const row of rows) { + await car.put({ + cid: CID.parse(row.cid), + bytes: row.content, + }) + } + } + // allow us to write to car while fetching the next page + let writePromise: Promise = Promise.resolve() do { const res = await this.getBlockRange(since, cursor) + await writePromise + writePromise = writeRows(res) for (const row of res) { await car.put({ cid: CID.parse(row.cid), @@ -231,6 +245,8 @@ export class SqlRepoStorage extends ReadableBlockstore implements RepoStorage { cursor = undefined } } while (cursor) + // ensure we flush the last page of blocks + await writePromise }) } @@ -240,17 +256,18 @@ export class SqlRepoStorage extends ReadableBlockstore implements RepoStorage { .selectFrom('ipld_block') .where('creator', '=', this.did) .select(['cid', 'repoRev', 'content']) - .orderBy('repoRev', 'asc') - .orderBy('cid', 'asc') + .orderBy('repoRev', 'desc') + .orderBy('cid', 'desc') .limit(500) if (cursor) { // use this syntax to ensure we hit the index builder = builder.where( - sql`((${ref('repoRev')}, ${ref('cid')}) > (${ + sql`((${ref('repoRev')}, ${ref('cid')}) < (${ cursor.rev }, ${cursor.cid.toString()}))`, ) - } else if (since) { + } + if (since) { builder = builder.where('repoRev', '>', since) } return builder.execute()