From cc11adda875f9d97c638fac2872696ee3a3d8ce2 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Fri, 8 Mar 2024 19:41:31 -0600 Subject: [PATCH] Ozone: Add sigs to all labels (#2236) * sketching out label sequencer * refactor sequencer * sequencer tests * tests * add query labels endpoint & tests * add pagination * fix label formatting on temp * tidy * format labels * make use listen/notify for sequencer * ensure sig on all outgoing labels from ozone * fixing up tests * fix sequencer tests * fix hanging server test * add log on failure to update label * update description for sig * use bytes for label sigs * fix tests * add ver to labels * tidy up background queue * store signing keys as ids * fix sequencer teest --- lexicons/com/atproto/label/defs.json | 8 ++ packages/api/src/client/lexicons.ts | 8 ++ .../client/types/com/atproto/label/defs.ts | 4 + packages/bsky/src/lexicon/lexicons.ts | 8 ++ .../lexicon/types/com/atproto/label/defs.ts | 4 + packages/ozone/src/api/label/queryLabels.ts | 6 +- packages/ozone/src/api/temp/fetchLabels.ts | 6 +- packages/ozone/src/context.ts | 22 ++++- packages/ozone/src/daemon/context.ts | 5 +- .../20240228T003647759Z-add-label-sigs.ts | 23 ++++++ packages/ozone/src/db/migrations/index.ts | 1 + packages/ozone/src/db/schema/index.ts | 2 + packages/ozone/src/db/schema/label.ts | 2 + packages/ozone/src/db/schema/signing_key.ts | 10 +++ packages/ozone/src/lexicon/lexicons.ts | 8 ++ .../lexicon/types/com/atproto/label/defs.ts | 4 + packages/ozone/src/mod-service/index.ts | 48 ++++++----- packages/ozone/src/mod-service/util.ts | 51 ++++++++++-- packages/ozone/src/mod-service/views.ts | 34 +++++++- packages/ozone/src/sequencer/sequencer.ts | 23 +++--- packages/ozone/src/util.ts | 21 +++++ .../__snapshots__/get-record.test.ts.snap | 8 ++ .../tests/__snapshots__/get-repo.test.ts.snap | 4 + packages/ozone/tests/_util.ts | 5 ++ packages/ozone/tests/query-labels.test.ts | 82 ++++++++++++++++++- packages/ozone/tests/sequencer.test.ts | 4 +- packages/pds/src/lexicon/lexicons.ts | 8 ++ .../lexicon/types/com/atproto/label/defs.ts | 4 + 28 files changed, 359 insertions(+), 54 deletions(-) create mode 100644 packages/ozone/src/db/migrations/20240228T003647759Z-add-label-sigs.ts create mode 100644 packages/ozone/src/db/schema/signing_key.ts diff --git a/lexicons/com/atproto/label/defs.json b/lexicons/com/atproto/label/defs.json index cd8e03e116c..229ad192f2a 100644 --- a/lexicons/com/atproto/label/defs.json +++ b/lexicons/com/atproto/label/defs.json @@ -7,6 +7,10 @@ "description": "Metadata tag on an atproto resource (eg, repo or record).", "required": ["src", "uri", "val", "cts"], "properties": { + "ver": { + "type": "integer", + "description": "The AT Protocol version of the label object." + }, "src": { "type": "string", "format": "did", @@ -35,6 +39,10 @@ "type": "string", "format": "datetime", "description": "Timestamp when this label was created." + }, + "sig": { + "type": "bytes", + "description": "Signature of dag-cbor encoded label." } } }, diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index 635e9e19e60..afc7ed866b1 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -2202,6 +2202,10 @@ export const schemaDict = { 'Metadata tag on an atproto resource (eg, repo or record).', required: ['src', 'uri', 'val', 'cts'], properties: { + ver: { + type: 'integer', + description: 'The AT Protocol version of the label object.', + }, src: { type: 'string', format: 'did', @@ -2235,6 +2239,10 @@ export const schemaDict = { format: 'datetime', description: 'Timestamp when this label was created.', }, + sig: { + type: 'bytes', + description: 'Signature of dag-cbor encoded label.', + }, }, }, selfLabels: { diff --git a/packages/api/src/client/types/com/atproto/label/defs.ts b/packages/api/src/client/types/com/atproto/label/defs.ts index c1641432c3a..425978d6d0e 100644 --- a/packages/api/src/client/types/com/atproto/label/defs.ts +++ b/packages/api/src/client/types/com/atproto/label/defs.ts @@ -8,6 +8,8 @@ import { CID } from 'multiformats/cid' /** Metadata tag on an atproto resource (eg, repo or record). */ export interface Label { + /** The AT Protocol version of the label object. */ + ver?: number /** DID of the actor who created this label. */ src: string /** AT URI of the record, repository (account), or other resource that this label applies to. */ @@ -20,6 +22,8 @@ export interface Label { neg?: boolean /** Timestamp when this label was created. */ cts: string + /** Signature of dag-cbor encoded label. */ + sig?: Uint8Array [k: string]: unknown } diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index 635e9e19e60..afc7ed866b1 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -2202,6 +2202,10 @@ export const schemaDict = { 'Metadata tag on an atproto resource (eg, repo or record).', required: ['src', 'uri', 'val', 'cts'], properties: { + ver: { + type: 'integer', + description: 'The AT Protocol version of the label object.', + }, src: { type: 'string', format: 'did', @@ -2235,6 +2239,10 @@ export const schemaDict = { format: 'datetime', description: 'Timestamp when this label was created.', }, + sig: { + type: 'bytes', + description: 'Signature of dag-cbor encoded label.', + }, }, }, selfLabels: { diff --git a/packages/bsky/src/lexicon/types/com/atproto/label/defs.ts b/packages/bsky/src/lexicon/types/com/atproto/label/defs.ts index 66226677a5b..935b2aa4bb3 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/label/defs.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/label/defs.ts @@ -8,6 +8,8 @@ import { CID } from 'multiformats/cid' /** Metadata tag on an atproto resource (eg, repo or record). */ export interface Label { + /** The AT Protocol version of the label object. */ + ver?: number /** DID of the actor who created this label. */ src: string /** AT URI of the record, repository (account), or other resource that this label applies to. */ @@ -20,6 +22,8 @@ export interface Label { neg?: boolean /** Timestamp when this label was created. */ cts: string + /** Signature of dag-cbor encoded label. */ + sig?: Uint8Array [k: string]: unknown } diff --git a/packages/ozone/src/api/label/queryLabels.ts b/packages/ozone/src/api/label/queryLabels.ts index 6de6380d194..38d83020fe5 100644 --- a/packages/ozone/src/api/label/queryLabels.ts +++ b/packages/ozone/src/api/label/queryLabels.ts @@ -2,7 +2,6 @@ import { Server } from '../../lexicon' import AppContext from '../../context' import { InvalidRequestError } from '@atproto/xrpc-server' import { sql } from 'kysely' -import { formatLabel } from '../../mod-service/util' export default function (server: Server, ctx: AppContext) { server.com.atproto.label.queryLabels(async ({ params }) => { @@ -44,7 +43,10 @@ export default function (server: Server, ctx: AppContext) { const res = await builder.execute() - const labels = res.map((l) => formatLabel(l)) + const modSrvc = ctx.modService(ctx.db) + const labels = await Promise.all( + res.map((l) => modSrvc.views.formatLabelAndEnsureSig(l)), + ) const resCursor = res.at(-1)?.id.toString(10) return { diff --git a/packages/ozone/src/api/temp/fetchLabels.ts b/packages/ozone/src/api/temp/fetchLabels.ts index d1a3f5e8e26..716c87a86bd 100644 --- a/packages/ozone/src/api/temp/fetchLabels.ts +++ b/packages/ozone/src/api/temp/fetchLabels.ts @@ -1,6 +1,5 @@ import { Server } from '../../lexicon' import AppContext from '../../context' -import { formatLabel } from '../../mod-service/util' import { UNSPECCED_TAKEDOWN_BLOBS_LABEL, UNSPECCED_TAKEDOWN_LABEL, @@ -29,7 +28,10 @@ export default function (server: Server, ctx: AppContext) { .limit(limit) .execute() - const labels = labelRes.map((l) => formatLabel(l)) + const modSrvc = ctx.modService(ctx.db) + const labels = await Promise.all( + labelRes.map((l) => modSrvc.views.formatLabelAndEnsureSig(l)), + ) return { encoding: 'application/json', diff --git a/packages/ozone/src/context.ts b/packages/ozone/src/context.ts index 59a1baff704..b7d9a3e9e89 100644 --- a/packages/ozone/src/context.ts +++ b/packages/ozone/src/context.ts @@ -16,6 +16,7 @@ import { } from './communication-service/template' import { AuthVerifier } from './auth-verifier' import { ImageInvalidator } from './image-invalidator' +import { getSigningKeyId } from './util' export type AppContextOptions = { db: Database @@ -25,6 +26,7 @@ export type AppContextOptions = { appviewAgent: AtpAgent pdsAgent: AtpAgent | undefined signingKey: Keypair + signingKeyId: number idResolver: IdResolver imgInvalidator?: ImageInvalidator backgroundQueue: BackgroundQueue @@ -48,6 +50,7 @@ export class AppContext { poolIdleTimeoutMs: cfg.db.poolIdleTimeoutMs, }) const signingKey = await Secp256k1Keypair.import(secrets.signingKeyHex) + const signingKeyId = await getSigningKeyId(db, signingKey.did()) const appviewAgent = new AtpAgent({ service: cfg.appview.url }) const pdsAgent = cfg.pds ? new AtpAgent({ service: cfg.pds.url }) @@ -71,20 +74,20 @@ export class AppContext { }) const modService = ModerationService.creator( + signingKey, + signingKeyId, cfg, backgroundQueue, idResolver, eventPusher, appviewAgent, createAuthHeaders, - cfg.service.did, overrides?.imgInvalidator, - cfg.cdn.paths, ) const communicationTemplateService = CommunicationTemplateService.creator() - const sequencer = new Sequencer(db) + const sequencer = new Sequencer(modService(db)) const authVerifier = new AuthVerifier(idResolver, { serviceDid: cfg.service.did, @@ -103,6 +106,7 @@ export class AppContext { appviewAgent, pdsAgent, signingKey, + signingKeyId, idResolver, backgroundQueue, sequencer, @@ -149,6 +153,10 @@ export class AppContext { return this.opts.signingKey } + get signingKeyId(): number { + return this.opts.signingKeyId + } + get plcClient(): plc.Client { return new plc.Client(this.cfg.identity.plcUrl) } @@ -188,6 +196,12 @@ export class AppContext { async appviewAuth() { return this.serviceAuthHeaders(this.cfg.appview.did) } -} + devOverride(overrides: Partial) { + this.opts = { + ...this.opts, + ...overrides, + } + } +} export default AppContext diff --git a/packages/ozone/src/daemon/context.ts b/packages/ozone/src/daemon/context.ts index 12c0ece9b17..64cf3c9423e 100644 --- a/packages/ozone/src/daemon/context.ts +++ b/packages/ozone/src/daemon/context.ts @@ -8,6 +8,7 @@ import { EventReverser } from './event-reverser' import { ModerationService, ModerationServiceCreator } from '../mod-service' import { BackgroundQueue } from '../background' import { IdResolver } from '@atproto/identity' +import { getSigningKeyId } from '../util' export type DaemonContextOptions = { db: Database @@ -31,6 +32,7 @@ export class DaemonContext { schema: cfg.db.postgresSchema, }) const signingKey = await Secp256k1Keypair.import(secrets.signingKeyHex) + const signingKeyId = await getSigningKeyId(db, signingKey.did()) const appviewAgent = new AtpAgent({ service: cfg.appview.url }) const createAuthHeaders = (aud: string) => @@ -51,13 +53,14 @@ export class DaemonContext { }) const modService = ModerationService.creator( + signingKey, + signingKeyId, cfg, backgroundQueue, idResolver, eventPusher, appviewAgent, createAuthHeaders, - cfg.service.did, ) const eventReverser = new EventReverser(db, modService) diff --git a/packages/ozone/src/db/migrations/20240228T003647759Z-add-label-sigs.ts b/packages/ozone/src/db/migrations/20240228T003647759Z-add-label-sigs.ts new file mode 100644 index 00000000000..098d4c4b672 --- /dev/null +++ b/packages/ozone/src/db/migrations/20240228T003647759Z-add-label-sigs.ts @@ -0,0 +1,23 @@ +import { Kysely, sql } from 'kysely' + +export async function up(db: Kysely): Promise { + await db.schema + .alterTable('label') + .addColumn('sig', sql`bytea`) + .execute() + await db.schema + .alterTable('label') + .addColumn('signingKeyId', 'integer') + .execute() + await db.schema + .createTable('signing_key') + .addColumn('id', 'serial', (col) => col.primaryKey()) + .addColumn('key', 'varchar', (col) => col.notNull().unique()) + .execute() +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable('signing_key') + await db.schema.alterTable('label').dropColumn('sig').execute() + await db.schema.alterTable('label').dropColumn('signingKey').execute() +} diff --git a/packages/ozone/src/db/migrations/index.ts b/packages/ozone/src/db/migrations/index.ts index 1a823f860c5..1281f12c7f1 100644 --- a/packages/ozone/src/db/migrations/index.ts +++ b/packages/ozone/src/db/migrations/index.ts @@ -6,3 +6,4 @@ export * as _20231219T205730722Z from './20231219T205730722Z-init' export * as _20240116T085607200Z from './20240116T085607200Z-communication-template' export * as _20240201T051104136Z from './20240201T051104136Z-mod-event-blobs' export * as _20240208T213404429Z from './20240208T213404429Z-add-tags-column-to-moderation-subject' +export * as _20240228T003647759Z from './20240228T003647759Z-add-label-sigs' diff --git a/packages/ozone/src/db/schema/index.ts b/packages/ozone/src/db/schema/index.ts index b522a75ef9f..48e3f15cdc5 100644 --- a/packages/ozone/src/db/schema/index.ts +++ b/packages/ozone/src/db/schema/index.ts @@ -5,11 +5,13 @@ import * as repoPushEvent from './repo_push_event' import * as recordPushEvent from './record_push_event' import * as blobPushEvent from './blob_push_event' import * as label from './label' +import * as signingKey from './signing_key' import * as communicationTemplate from './communication_template' export type DatabaseSchemaType = modEvent.PartialDB & modSubjectStatus.PartialDB & label.PartialDB & + signingKey.PartialDB & repoPushEvent.PartialDB & recordPushEvent.PartialDB & blobPushEvent.PartialDB & diff --git a/packages/ozone/src/db/schema/label.ts b/packages/ozone/src/db/schema/label.ts index f50a6119ab3..d19937ce59d 100644 --- a/packages/ozone/src/db/schema/label.ts +++ b/packages/ozone/src/db/schema/label.ts @@ -10,6 +10,8 @@ export interface Label { val: string neg: boolean cts: string + sig: Buffer | null + signingKeyId: number | null } export type LabelRow = Selectable