diff --git a/lexicons/app/bsky/unspecced/applyLabels.json b/lexicons/app/bsky/unspecced/applyLabels.json deleted file mode 100644 index 24c9e716ad5..00000000000 --- a/lexicons/app/bsky/unspecced/applyLabels.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "lexicon": 1, - "id": "app.bsky.unspecced.applyLabels", - "defs": { - "main": { - "type": "procedure", - "description": "Allow a labeler to apply labels directly.", - "input": { - "encoding": "application/json", - "schema": { - "type": "object", - "required": ["labels"], - "properties": { - "labels": { - "type": "array", - "items": { "type": "ref", "ref": "com.atproto.label.defs#label" } - } - } - } - } - } - } -} diff --git a/packages/api/src/client/index.ts b/packages/api/src/client/index.ts index 4d9ae10936b..982117cef02 100644 --- a/packages/api/src/client/index.ts +++ b/packages/api/src/client/index.ts @@ -129,7 +129,6 @@ import * as AppBskyNotificationListNotifications from './types/app/bsky/notifica import * as AppBskyNotificationRegisterPush from './types/app/bsky/notification/registerPush' import * as AppBskyNotificationUpdateSeen from './types/app/bsky/notification/updateSeen' import * as AppBskyRichtextFacet from './types/app/bsky/richtext/facet' -import * as AppBskyUnspeccedApplyLabels from './types/app/bsky/unspecced/applyLabels' import * as AppBskyUnspeccedDefs from './types/app/bsky/unspecced/defs' import * as AppBskyUnspeccedGetPopular from './types/app/bsky/unspecced/getPopular' import * as AppBskyUnspeccedGetPopularFeedGenerators from './types/app/bsky/unspecced/getPopularFeedGenerators' @@ -259,7 +258,6 @@ export * as AppBskyNotificationListNotifications from './types/app/bsky/notifica export * as AppBskyNotificationRegisterPush from './types/app/bsky/notification/registerPush' export * as AppBskyNotificationUpdateSeen from './types/app/bsky/notification/updateSeen' export * as AppBskyRichtextFacet from './types/app/bsky/richtext/facet' -export * as AppBskyUnspeccedApplyLabels from './types/app/bsky/unspecced/applyLabels' export * as AppBskyUnspeccedDefs from './types/app/bsky/unspecced/defs' export * as AppBskyUnspeccedGetPopular from './types/app/bsky/unspecced/getPopular' export * as AppBskyUnspeccedGetPopularFeedGenerators from './types/app/bsky/unspecced/getPopularFeedGenerators' @@ -2253,17 +2251,6 @@ export class UnspeccedNS { this._service = service } - applyLabels( - data?: AppBskyUnspeccedApplyLabels.InputSchema, - opts?: AppBskyUnspeccedApplyLabels.CallOptions, - ): Promise { - return this._service.xrpc - .call('app.bsky.unspecced.applyLabels', opts?.qp, data, opts) - .catch((e) => { - throw AppBskyUnspeccedApplyLabels.toKnownErr(e) - }) - } - getPopular( params?: AppBskyUnspeccedGetPopular.QueryParams, opts?: AppBskyUnspeccedGetPopular.CallOptions, diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index b624ce335ed..177b63808f4 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -6905,32 +6905,6 @@ export const schemaDict = { }, }, }, - AppBskyUnspeccedApplyLabels: { - lexicon: 1, - id: 'app.bsky.unspecced.applyLabels', - defs: { - main: { - type: 'procedure', - description: 'Allow a labeler to apply labels directly.', - input: { - encoding: 'application/json', - schema: { - type: 'object', - required: ['labels'], - properties: { - labels: { - type: 'array', - items: { - type: 'ref', - ref: 'lex:com.atproto.label.defs#label', - }, - }, - }, - }, - }, - }, - }, - }, AppBskyUnspeccedDefs: { lexicon: 1, id: 'app.bsky.unspecced.defs', @@ -7359,7 +7333,6 @@ export const ids = { AppBskyNotificationRegisterPush: 'app.bsky.notification.registerPush', AppBskyNotificationUpdateSeen: 'app.bsky.notification.updateSeen', AppBskyRichtextFacet: 'app.bsky.richtext.facet', - AppBskyUnspeccedApplyLabels: 'app.bsky.unspecced.applyLabels', AppBskyUnspeccedDefs: 'app.bsky.unspecced.defs', AppBskyUnspeccedGetPopular: 'app.bsky.unspecced.getPopular', AppBskyUnspeccedGetPopularFeedGenerators: diff --git a/packages/api/src/client/types/app/bsky/unspecced/applyLabels.ts b/packages/api/src/client/types/app/bsky/unspecced/applyLabels.ts deleted file mode 100644 index c8e72746a42..00000000000 --- a/packages/api/src/client/types/app/bsky/unspecced/applyLabels.ts +++ /dev/null @@ -1,33 +0,0 @@ -/** - * GENERATED CODE - DO NOT MODIFY - */ -import { Headers, XRPCError } from '@atproto/xrpc' -import { ValidationResult, BlobRef } from '@atproto/lexicon' -import { isObj, hasProp } from '../../../../util' -import { lexicons } from '../../../../lexicons' -import { CID } from 'multiformats/cid' -import * as ComAtprotoLabelDefs from '../../../com/atproto/label/defs' - -export interface QueryParams {} - -export interface InputSchema { - labels: ComAtprotoLabelDefs.Label[] - [k: string]: unknown -} - -export interface CallOptions { - headers?: Headers - qp?: QueryParams - encoding: 'application/json' -} - -export interface Response { - success: boolean - headers: Headers -} - -export function toKnownErr(e: any) { - if (e instanceof XRPCError) { - } - return e -} diff --git a/packages/bsky/package.json b/packages/bsky/package.json index 33cafe53947..5281805279f 100644 --- a/packages/bsky/package.json +++ b/packages/bsky/package.json @@ -50,7 +50,6 @@ "http-errors": "^2.0.0", "http-terminator": "^3.2.0", "ioredis": "^5.3.2", - "iso-datestring-validator": "^2.2.2", "kysely": "^0.22.0", "multiformats": "^9.9.0", "p-queue": "^6.6.2", diff --git a/packages/bsky/src/auto-moderator/index.ts b/packages/bsky/src/auto-moderator/index.ts index 1be099759f1..30befc19110 100644 --- a/packages/bsky/src/auto-moderator/index.ts +++ b/packages/bsky/src/auto-moderator/index.ts @@ -280,28 +280,12 @@ export class AutoModerator { async storeLabels(uri: AtUri, cid: CID, labels: string[]): Promise { if (labels.length < 1) return const labelSrvc = this.services.label(this.ctx.db) - const formatted = await labelSrvc.formatAndCreate( + await labelSrvc.formatAndCreate( this.ctx.cfg.labelerDid, uri.toString(), cid.toString(), { create: labels }, ) - if (this.pushAgent) { - const agent = this.pushAgent - try { - await agent.api.app.bsky.unspecced.applyLabels({ labels: formatted }) - } catch (err) { - log.error( - { - err, - uri: uri.toString(), - labels, - receiver: agent.service.toString(), - }, - 'failed to push labels', - ) - } - } } async processAll() { diff --git a/packages/bsky/src/lexicon/index.ts b/packages/bsky/src/lexicon/index.ts index 3ec69a3503f..3dd2b5104cc 100644 --- a/packages/bsky/src/lexicon/index.ts +++ b/packages/bsky/src/lexicon/index.ts @@ -107,7 +107,6 @@ import * as AppBskyNotificationGetUnreadCount from './types/app/bsky/notificatio import * as AppBskyNotificationListNotifications from './types/app/bsky/notification/listNotifications' import * as AppBskyNotificationRegisterPush from './types/app/bsky/notification/registerPush' import * as AppBskyNotificationUpdateSeen from './types/app/bsky/notification/updateSeen' -import * as AppBskyUnspeccedApplyLabels from './types/app/bsky/unspecced/applyLabels' import * as AppBskyUnspeccedGetPopular from './types/app/bsky/unspecced/getPopular' import * as AppBskyUnspeccedGetPopularFeedGenerators from './types/app/bsky/unspecced/getPopularFeedGenerators' import * as AppBskyUnspeccedGetTimelineSkeleton from './types/app/bsky/unspecced/getTimelineSkeleton' @@ -1402,17 +1401,6 @@ export class UnspeccedNS { this._server = server } - applyLabels( - cfg: ConfigOf< - AV, - AppBskyUnspeccedApplyLabels.Handler>, - AppBskyUnspeccedApplyLabels.HandlerReqCtx> - >, - ) { - const nsid = 'app.bsky.unspecced.applyLabels' // @ts-ignore - return this._server.xrpc.method(nsid, cfg) - } - getPopular( cfg: ConfigOf< AV, diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index b624ce335ed..177b63808f4 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -6905,32 +6905,6 @@ export const schemaDict = { }, }, }, - AppBskyUnspeccedApplyLabels: { - lexicon: 1, - id: 'app.bsky.unspecced.applyLabels', - defs: { - main: { - type: 'procedure', - description: 'Allow a labeler to apply labels directly.', - input: { - encoding: 'application/json', - schema: { - type: 'object', - required: ['labels'], - properties: { - labels: { - type: 'array', - items: { - type: 'ref', - ref: 'lex:com.atproto.label.defs#label', - }, - }, - }, - }, - }, - }, - }, - }, AppBskyUnspeccedDefs: { lexicon: 1, id: 'app.bsky.unspecced.defs', @@ -7359,7 +7333,6 @@ export const ids = { AppBskyNotificationRegisterPush: 'app.bsky.notification.registerPush', AppBskyNotificationUpdateSeen: 'app.bsky.notification.updateSeen', AppBskyRichtextFacet: 'app.bsky.richtext.facet', - AppBskyUnspeccedApplyLabels: 'app.bsky.unspecced.applyLabels', AppBskyUnspeccedDefs: 'app.bsky.unspecced.defs', AppBskyUnspeccedGetPopular: 'app.bsky.unspecced.getPopular', AppBskyUnspeccedGetPopularFeedGenerators: diff --git a/packages/bsky/src/lexicon/types/app/bsky/unspecced/applyLabels.ts b/packages/bsky/src/lexicon/types/app/bsky/unspecced/applyLabels.ts deleted file mode 100644 index 1d359a9547d..00000000000 --- a/packages/bsky/src/lexicon/types/app/bsky/unspecced/applyLabels.ts +++ /dev/null @@ -1,39 +0,0 @@ -/** - * GENERATED CODE - DO NOT MODIFY - */ -import express from 'express' -import { ValidationResult, BlobRef } from '@atproto/lexicon' -import { lexicons } from '../../../../lexicons' -import { isObj, hasProp } from '../../../../util' -import { CID } from 'multiformats/cid' -import { HandlerAuth } from '@atproto/xrpc-server' -import * as ComAtprotoLabelDefs from '../../../com/atproto/label/defs' - -export interface QueryParams {} - -export interface InputSchema { - labels: ComAtprotoLabelDefs.Label[] - [k: string]: unknown -} - -export interface HandlerInput { - encoding: 'application/json' - body: InputSchema -} - -export interface HandlerError { - status: number - message?: string -} - -export type HandlerOutput = HandlerError | void -export type HandlerReqCtx = { - auth: HA - params: QueryParams - input: HandlerInput - req: express.Request - res: express.Response -} -export type Handler = ( - ctx: HandlerReqCtx, -) => Promise | HandlerOutput diff --git a/packages/bsky/src/services/indexing/plugins/block.ts b/packages/bsky/src/services/indexing/plugins/block.ts index c0c8fe9d770..bf8ae9e5029 100644 --- a/packages/bsky/src/services/indexing/plugins/block.ts +++ b/packages/bsky/src/services/indexing/plugins/block.ts @@ -1,11 +1,11 @@ import { Selectable } from 'kysely' import { AtUri } from '@atproto/syntax' +import { toSimplifiedISOSafe } from '@atproto/common' import { CID } from 'multiformats/cid' import * as Block from '../../../lexicon/types/app/bsky/graph/block' import * as lex from '../../../lexicon/lexicons' import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' import RecordProcessor from '../processor' -import { toSimplifiedISOSafe } from '../util' import { PrimaryDatabase } from '../../../db' import { BackgroundQueue } from '../../../background' import { NotificationServer } from '../../../notifications' diff --git a/packages/bsky/src/services/indexing/plugins/feed-generator.ts b/packages/bsky/src/services/indexing/plugins/feed-generator.ts index a0c32ff9129..e4ae5eb4f5a 100644 --- a/packages/bsky/src/services/indexing/plugins/feed-generator.ts +++ b/packages/bsky/src/services/indexing/plugins/feed-generator.ts @@ -1,5 +1,6 @@ import { Selectable } from 'kysely' import { AtUri } from '@atproto/syntax' +import { toSimplifiedISOSafe } from '@atproto/common' import { CID } from 'multiformats/cid' import * as FeedGenerator from '../../../lexicon/types/app/bsky/feed/generator' import * as lex from '../../../lexicon/lexicons' @@ -7,7 +8,6 @@ import { PrimaryDatabase } from '../../../db' import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' import { BackgroundQueue } from '../../../background' import RecordProcessor from '../processor' -import { toSimplifiedISOSafe } from '../util' import { NotificationServer } from '../../../notifications' const lexId = lex.ids.AppBskyFeedGenerator diff --git a/packages/bsky/src/services/indexing/plugins/follow.ts b/packages/bsky/src/services/indexing/plugins/follow.ts index d6cf1996f98..e9a344db2fd 100644 --- a/packages/bsky/src/services/indexing/plugins/follow.ts +++ b/packages/bsky/src/services/indexing/plugins/follow.ts @@ -1,11 +1,11 @@ import { Selectable } from 'kysely' import { AtUri } from '@atproto/syntax' +import { toSimplifiedISOSafe } from '@atproto/common' import { CID } from 'multiformats/cid' import * as Follow from '../../../lexicon/types/app/bsky/graph/follow' import * as lex from '../../../lexicon/lexicons' import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' import RecordProcessor from '../processor' -import { toSimplifiedISOSafe } from '../util' import { PrimaryDatabase } from '../../../db' import { countAll, excluded } from '../../../db/util' import { BackgroundQueue } from '../../../background' diff --git a/packages/bsky/src/services/indexing/plugins/like.ts b/packages/bsky/src/services/indexing/plugins/like.ts index 7899b48b1fd..01e0fa5c4fd 100644 --- a/packages/bsky/src/services/indexing/plugins/like.ts +++ b/packages/bsky/src/services/indexing/plugins/like.ts @@ -1,11 +1,11 @@ import { Selectable } from 'kysely' import { AtUri } from '@atproto/syntax' +import { toSimplifiedISOSafe } from '@atproto/common' import { CID } from 'multiformats/cid' import * as Like from '../../../lexicon/types/app/bsky/feed/like' import * as lex from '../../../lexicon/lexicons' import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' import RecordProcessor from '../processor' -import { toSimplifiedISOSafe } from '../util' import { countAll, excluded } from '../../../db/util' import { PrimaryDatabase } from '../../../db' import { BackgroundQueue } from '../../../background' diff --git a/packages/bsky/src/services/indexing/plugins/list-block.ts b/packages/bsky/src/services/indexing/plugins/list-block.ts index 4285ca8d4bc..33dc7cfc51a 100644 --- a/packages/bsky/src/services/indexing/plugins/list-block.ts +++ b/packages/bsky/src/services/indexing/plugins/list-block.ts @@ -1,5 +1,6 @@ import { Selectable } from 'kysely' import { AtUri } from '@atproto/syntax' +import { toSimplifiedISOSafe } from '@atproto/common' import { CID } from 'multiformats/cid' import * as ListBlock from '../../../lexicon/types/app/bsky/graph/listblock' import * as lex from '../../../lexicon/lexicons' @@ -8,7 +9,6 @@ import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' import RecordProcessor from '../processor' import { BackgroundQueue } from '../../../background' import { NotificationServer } from '../../../notifications' -import { toSimplifiedISOSafe } from '../util' const lexId = lex.ids.AppBskyGraphListblock type IndexedListBlock = Selectable diff --git a/packages/bsky/src/services/indexing/plugins/list-item.ts b/packages/bsky/src/services/indexing/plugins/list-item.ts index 231fb761e16..2ab125062a7 100644 --- a/packages/bsky/src/services/indexing/plugins/list-item.ts +++ b/packages/bsky/src/services/indexing/plugins/list-item.ts @@ -1,11 +1,11 @@ import { Selectable } from 'kysely' import { AtUri } from '@atproto/syntax' +import { toSimplifiedISOSafe } from '@atproto/common' import { CID } from 'multiformats/cid' import * as ListItem from '../../../lexicon/types/app/bsky/graph/listitem' import * as lex from '../../../lexicon/lexicons' import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' import RecordProcessor from '../processor' -import { toSimplifiedISOSafe } from '../util' import { InvalidRequestError } from '@atproto/xrpc-server' import { PrimaryDatabase } from '../../../db' import { BackgroundQueue } from '../../../background' diff --git a/packages/bsky/src/services/indexing/plugins/list.ts b/packages/bsky/src/services/indexing/plugins/list.ts index c74c09c274f..293c457c4fb 100644 --- a/packages/bsky/src/services/indexing/plugins/list.ts +++ b/packages/bsky/src/services/indexing/plugins/list.ts @@ -1,11 +1,11 @@ import { Selectable } from 'kysely' import { AtUri } from '@atproto/syntax' +import { toSimplifiedISOSafe } from '@atproto/common' import { CID } from 'multiformats/cid' import * as List from '../../../lexicon/types/app/bsky/graph/list' import * as lex from '../../../lexicon/lexicons' import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' import RecordProcessor from '../processor' -import { toSimplifiedISOSafe } from '../util' import { PrimaryDatabase } from '../../../db' import { BackgroundQueue } from '../../../background' import { NotificationServer } from '../../../notifications' diff --git a/packages/bsky/src/services/indexing/plugins/post.ts b/packages/bsky/src/services/indexing/plugins/post.ts index 40835348f01..7173c04a991 100644 --- a/packages/bsky/src/services/indexing/plugins/post.ts +++ b/packages/bsky/src/services/indexing/plugins/post.ts @@ -1,6 +1,7 @@ import { Insertable, Selectable, sql } from 'kysely' import { CID } from 'multiformats/cid' import { AtUri } from '@atproto/syntax' +import { toSimplifiedISOSafe } from '@atproto/common' import { jsonStringToLex } from '@atproto/lexicon' import { Record as PostRecord, @@ -19,7 +20,6 @@ import * as lex from '../../../lexicon/lexicons' import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' import RecordProcessor from '../processor' import { Notification } from '../../../db/tables/notification' -import { toSimplifiedISOSafe } from '../util' import { PrimaryDatabase } from '../../../db' import { countAll, excluded } from '../../../db/util' import { BackgroundQueue } from '../../../background' diff --git a/packages/bsky/src/services/indexing/plugins/repost.ts b/packages/bsky/src/services/indexing/plugins/repost.ts index aa93d7b0f61..9c46b9b3376 100644 --- a/packages/bsky/src/services/indexing/plugins/repost.ts +++ b/packages/bsky/src/services/indexing/plugins/repost.ts @@ -1,11 +1,11 @@ import { Selectable } from 'kysely' import { CID } from 'multiformats/cid' import { AtUri } from '@atproto/syntax' +import { toSimplifiedISOSafe } from '@atproto/common' import * as Repost from '../../../lexicon/types/app/bsky/feed/repost' import * as lex from '../../../lexicon/lexicons' import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' import RecordProcessor from '../processor' -import { toSimplifiedISOSafe } from '../util' import { PrimaryDatabase } from '../../../db' import { countAll, excluded } from '../../../db/util' import { BackgroundQueue } from '../../../background' diff --git a/packages/bsky/src/services/indexing/plugins/thread-gate.ts b/packages/bsky/src/services/indexing/plugins/thread-gate.ts index fb0928f2459..37f3ddb062e 100644 --- a/packages/bsky/src/services/indexing/plugins/thread-gate.ts +++ b/packages/bsky/src/services/indexing/plugins/thread-gate.ts @@ -1,11 +1,11 @@ import { AtUri } from '@atproto/syntax' import { InvalidRequestError } from '@atproto/xrpc-server' +import { toSimplifiedISOSafe } from '@atproto/common' import { CID } from 'multiformats/cid' import * as Threadgate from '../../../lexicon/types/app/bsky/feed/threadgate' import * as lex from '../../../lexicon/lexicons' import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' import RecordProcessor from '../processor' -import { toSimplifiedISOSafe } from '../util' import { PrimaryDatabase } from '../../../db' import { BackgroundQueue } from '../../../background' import { NotificationServer } from '../../../notifications' diff --git a/packages/bsky/src/services/label/index.ts b/packages/bsky/src/services/label/index.ts index 855038ab14c..7d351b95011 100644 --- a/packages/bsky/src/services/label/index.ts +++ b/packages/bsky/src/services/label/index.ts @@ -1,9 +1,9 @@ import { sql } from 'kysely' import { AtUri } from '@atproto/syntax' +import { toSimplifiedISOSafe } from '@atproto/common' import { Database } from '../../db' import { Label, isSelfLabels } from '../../lexicon/types/com/atproto/label/defs' import { ids } from '../../lexicon/lexicons' -import { toSimplifiedISOSafe } from '../indexing/util' import { LabelCache } from '../../label-cache' export type Labels = Record diff --git a/packages/common/package.json b/packages/common/package.json index 4794bdae698..5b88803035f 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -27,6 +27,7 @@ "@atproto/common-web": "workspace:^", "@ipld/dag-cbor": "^7.0.3", "cbor-x": "^1.5.1", + "iso-datestring-validator": "^2.2.2", "multiformats": "^9.9.0", "pino": "^8.15.0", "zod": "3.21.4" diff --git a/packages/bsky/src/services/indexing/util.ts b/packages/common/src/dates.ts similarity index 100% rename from packages/bsky/src/services/indexing/util.ts rename to packages/common/src/dates.ts diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts index fd049fc2e5d..524a090c5ab 100644 --- a/packages/common/src/index.ts +++ b/packages/common/src/index.ts @@ -1,4 +1,5 @@ export * from '@atproto/common-web' +export * from './dates' export * from './fs' export * from './ipld' export * from './ipld-multi' diff --git a/packages/dev-env/build.js b/packages/dev-env/build.js index 98bb2df6133..60634cf4503 100644 --- a/packages/dev-env/build.js +++ b/packages/dev-env/build.js @@ -6,7 +6,7 @@ const buildShallow = require('esbuild').build({ logLevel: 'info', - entryPoints: ['src/index.ts', 'src/bin.ts', 'src/bin-network.ts'], + entryPoints: ['src/index.ts', 'src/bin.ts'], bundle: true, sourcemap: true, outdir: 'dist', diff --git a/packages/dev-env/package.json b/packages/dev-env/package.json index 926ff0b412d..6ae8cc5d151 100644 --- a/packages/dev-env/package.json +++ b/packages/dev-env/package.json @@ -22,8 +22,7 @@ "build": "node ./build.js", "postbuild": "tsc --build tsconfig.build.json", "update-main-to-dist": "node ../../update-main-to-dist.js packages/dev-env", - "start": "node dist/bin.js", - "start:network": "../dev-infra/with-test-redis-and-db.sh node dist/bin-network.js" + "start": "../dev-infra/with-test-redis-and-db.sh node dist/bin.js" }, "dependencies": { "@atproto/api": "workspace:^", diff --git a/packages/dev-env/src/bin-network.ts b/packages/dev-env/src/bin-network.ts deleted file mode 100644 index 193c7ea968a..00000000000 --- a/packages/dev-env/src/bin-network.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { generateMockSetup } from './mock' -import { TestNetwork } from './network' - -const run = async () => { - console.log(` -██████╗ -██╔═══██╗ -██║██╗██║ -██║██║██║ -╚█║████╔╝ - ╚╝╚═══╝ protocol - -[ created by Bluesky ]`) - - const network = await TestNetwork.create({ - pds: { - port: 2583, - publicUrl: 'http://localhost:2583', - enableLabelsCache: true, - dbPostgresSchema: 'pds', - }, - bsky: { - dbPostgresSchema: 'bsky', - }, - plc: { port: 2582 }, - }) - await generateMockSetup(network) - - console.log( - `👤 DID Placeholder server started http://localhost:${network.plc.port}`, - ) - console.log( - `🌞 Personal Data server started http://localhost:${network.pds.port}`, - ) - console.log(`🌅 Bsky Appview started http://localhost:${network.bsky.port}`) - for (const fg of network.feedGens) { - console.log(`🤖 Feed Generator started http://localhost:${fg.port}`) - } -} - -run() diff --git a/packages/dev-env/src/bin.ts b/packages/dev-env/src/bin.ts index 532265e18ec..56f0138b3b2 100644 --- a/packages/dev-env/src/bin.ts +++ b/packages/dev-env/src/bin.ts @@ -1,5 +1,5 @@ import { generateMockSetup } from './mock' -import { TestNetworkNoAppView } from './network-no-appview' +import { TestNetwork } from './network' const run = async () => { console.log(` @@ -12,11 +12,14 @@ const run = async () => { [ created by Bluesky ]`) - const network = await TestNetworkNoAppView.create({ + const network = await TestNetwork.create({ pds: { port: 2583, - enableLabelsCache: true, publicUrl: 'http://localhost:2583', + dbPostgresSchema: 'pds', + }, + bsky: { + dbPostgresSchema: 'bsky', }, plc: { port: 2582 }, }) @@ -28,6 +31,7 @@ const run = async () => { console.log( `🌞 Personal Data server started http://localhost:${network.pds.port}`, ) + console.log(`🌅 Bsky Appview started http://localhost:${network.bsky.port}`) for (const fg of network.feedGens) { console.log(`🤖 Feed Generator started http://localhost:${fg.port}`) } diff --git a/packages/dev-env/src/mock/index.ts b/packages/dev-env/src/mock/index.ts index afa661f7ecb..10f76b1c259 100644 --- a/packages/dev-env/src/mock/index.ts +++ b/packages/dev-env/src/mock/index.ts @@ -4,7 +4,7 @@ import { REASONSPAM, REASONOTHER, } from '@atproto/api/src/client/types/com/atproto/moderation/defs' -import { TestNetworkNoAppView } from '../index' +import { TestNetwork } from '../index' import { postTexts, replyTexts } from './data' import labeledImgB64 from './img/labeled-img-b64' import blurHashB64 from './img/blur-hash-avatar-b64' @@ -23,7 +23,7 @@ function* dateGen() { return '' } -export async function generateMockSetup(env: TestNetworkNoAppView) { +export async function generateMockSetup(env: TestNetwork) { const date = dateGen() const rand = (n: number) => Math.floor(Math.random() * n) @@ -186,29 +186,27 @@ export async function generateMockSetup(env: TestNetworkNoAppView) { }, ) - const ctx = env.pds.ctx + const ctx = env.bsky.ctx if (ctx) { - await ctx.db.db - .insertInto('label') - .values([ - { - src: ctx.cfg.labelerDid, - uri: labeledPost.uri, - cid: labeledPost.cid, - val: 'nudity', - neg: 0, - cts: new Date().toISOString(), - }, - { - src: ctx.cfg.labelerDid, - uri: filteredPost.uri, - cid: filteredPost.cid, - val: 'dmca-violation', - neg: 0, - cts: new Date().toISOString(), - }, - ]) - .execute() + const labelSrvc = ctx.services.label(ctx.db.getPrimary()) + await labelSrvc.createLabels([ + { + src: ctx.cfg.labelerDid, + uri: labeledPost.uri, + cid: labeledPost.cid, + val: 'nudity', + neg: false, + cts: new Date().toISOString(), + }, + { + src: ctx.cfg.labelerDid, + uri: filteredPost.uri, + cid: filteredPost.cid, + val: 'dmca-violation', + neg: false, + cts: new Date().toISOString(), + }, + ]) } // a set of replies diff --git a/packages/dev-env/src/pds.ts b/packages/dev-env/src/pds.ts index e94130f41ff..2cdcadfe70c 100644 --- a/packages/dev-env/src/pds.ts +++ b/packages/dev-env/src/pds.ts @@ -2,7 +2,6 @@ import getPort from 'get-port' import * as ui8 from 'uint8arrays' import * as pds from '@atproto/pds' import { Secp256k1Keypair, randomStr } from '@atproto/crypto' -import { MessageDispatcher } from '@atproto/pds/src/event-stream/message-queue' import { AtpAgent } from '@atproto/api' import { Client as PlcClient } from '@did-plc/lib' import { DAY, HOUR } from '@atproto/common-web' @@ -60,9 +59,6 @@ export class TestPds { maxSubscriptionBuffer: 200, repoBackfillLimitMs: 1000 * 60 * 60, // 1hr sequencerLeaderLockId: uniqueLockId(), - labelerDid: 'did:example:labeler', - labelerKeywords: { label_me: 'test-label', label_me_2: 'test-label-2' }, - feedGenDid: 'did:example:feedGen', dbTxLockNonce: await randomStr(32, 'base32'), bskyAppViewEndpoint: cfg.bskyAppViewEndpoint ?? 'http://fake_address', bskyAppViewDid: cfg.bskyAppViewDid ?? 'did:example:fake', @@ -80,11 +76,6 @@ export class TestPds { : pds.Database.memory() await db.migrateToLatestOrThrow() - if (cfg.bskyAppViewEndpoint && !cfg.enableInProcessAppView) { - // Disable communication to app view within pds - MessageDispatcher.prototype.send = async () => {} - } - const server = pds.PDS.create({ db, blobstore, @@ -95,8 +86,6 @@ export class TestPds { await server.start() - // we refresh label cache by hand in `processAll` instead of on a timer - if (!cfg.enableLabelsCache) server.ctx.labelCache.stop() return new TestPds(url, port, server) } @@ -129,7 +118,6 @@ export class TestPds { async processAll() { await this.ctx.backgroundQueue.processAll() - await this.ctx.labelCache.fullRefresh() } async close() { diff --git a/packages/dev-env/src/types.ts b/packages/dev-env/src/types.ts index d87e78a679a..a1642f4751c 100644 --- a/packages/dev-env/src/types.ts +++ b/packages/dev-env/src/types.ts @@ -10,8 +10,6 @@ export type PlcConfig = { export type PdsConfig = Partial & { plcUrl: string migration?: string - enableInProcessAppView?: boolean - enableLabelsCache?: boolean } export type BskyConfig = Partial & { diff --git a/packages/pds/package.json b/packages/pds/package.json index 833b016f4ef..a7d98578f35 100644 --- a/packages/pds/package.json +++ b/packages/pds/package.json @@ -56,7 +56,6 @@ "http-errors": "^2.0.0", "http-terminator": "^3.2.0", "ioredis": "^5.3.2", - "iso-datestring-validator": "^2.2.2", "jsonwebtoken": "^8.5.1", "kysely": "^0.22.0", "multiformats": "^9.9.0", diff --git a/packages/pds/src/api/com/atproto/admin/reverseModerationAction.ts b/packages/pds/src/api/com/atproto/admin/reverseModerationAction.ts index d652b501166..b0d1e149790 100644 --- a/packages/pds/src/api/com/atproto/admin/reverseModerationAction.ts +++ b/packages/pds/src/api/com/atproto/admin/reverseModerationAction.ts @@ -26,7 +26,6 @@ export default function (server: Server, ctx: AppContext) { const transact = db.transaction(async (dbTxn) => { const moderationTxn = services.moderation(dbTxn) - const labelTxn = services.appView.label(dbTxn) // reverse takedowns if (result.action === TAKEDOWN && isRepoRef(result.subject)) { await moderationTxn.reverseTakedownRepo({ @@ -38,18 +37,6 @@ export default function (server: Server, ctx: AppContext) { uri: new AtUri(result.subject.uri), }) } - // invert label creation & negations - const reverseLabels = (uri: string, cid: string | null) => - labelTxn.formatAndCreate(ctx.cfg.labelerDid, uri, cid, { - create: result.negateLabelVals, - negate: result.createLabelVals, - }) - if (isRepoRef(result.subject)) { - await reverseLabels(result.subject.did, null) - } - if (isStrongRef(result.subject)) { - await reverseLabels(result.subject.uri, result.subject.cid) - } }) try { @@ -72,7 +59,6 @@ export default function (server: Server, ctx: AppContext) { const moderationAction = await db.transaction(async (dbTxn) => { const moderationTxn = services.moderation(dbTxn) - const labelTxn = services.appView.label(dbTxn) const now = new Date() const existing = await moderationTxn.getAction(id) @@ -114,23 +100,6 @@ export default function (server: Server, ctx: AppContext) { reason, }) - // invert creates & negates - const { createLabelVals, negateLabelVals } = result - const negate = - createLabelVals && createLabelVals.length > 0 - ? createLabelVals.split(' ') - : undefined - const create = - negateLabelVals && negateLabelVals.length > 0 - ? negateLabelVals.split(' ') - : undefined - await labelTxn.formatAndCreate( - ctx.cfg.labelerDid, - result.subjectUri ?? result.subjectDid, - result.subjectCid, - { create, negate }, - ) - return result }) diff --git a/packages/pds/src/api/com/atproto/admin/takeModerationAction.ts b/packages/pds/src/api/com/atproto/admin/takeModerationAction.ts index 8a81245a9e8..e9fb9b7e043 100644 --- a/packages/pds/src/api/com/atproto/admin/takeModerationAction.ts +++ b/packages/pds/src/api/com/atproto/admin/takeModerationAction.ts @@ -29,7 +29,6 @@ export default function (server: Server, ctx: AppContext) { const transact = db.transaction(async (dbTxn) => { const authTxn = services.auth(dbTxn) const moderationTxn = services.moderation(dbTxn) - const labelTxn = services.appView.label(dbTxn) // perform takedowns if (result.action === TAKEDOWN && isRepoRef(result.subject)) { await authTxn.revokeRefreshTokensByDid(result.subject.did) @@ -45,18 +44,6 @@ export default function (server: Server, ctx: AppContext) { blobCids: result.subjectBlobCids.map((cid) => CID.parse(cid)), }) } - // apply label creation & negations - const applyLabels = (uri: string, cid: string | null) => - labelTxn.formatAndCreate(ctx.cfg.labelerDid, uri, cid, { - create: result.createLabelVals, - negate: result.negateLabelVals, - }) - if (isRepoRef(result.subject)) { - await applyLabels(result.subject.did, null) - } - if (isStrongRef(result.subject)) { - await applyLabels(result.subject.uri, result.subject.cid) - } }) try { @@ -113,7 +100,6 @@ export default function (server: Server, ctx: AppContext) { const moderationAction = await db.transaction(async (dbTxn) => { const authTxn = services.auth(dbTxn) const moderationTxn = services.moderation(dbTxn) - const labelTxn = services.appView.label(dbTxn) const result = await moderationTxn.logAction({ action: getAction(action), @@ -150,13 +136,6 @@ export default function (server: Server, ctx: AppContext) { }) } - await labelTxn.formatAndCreate( - ctx.cfg.labelerDid, - result.subjectUri ?? result.subjectDid, - result.subjectCid, - { create: createLabelVals, negate: negateLabelVals }, - ) - return result }) diff --git a/packages/pds/src/app-view/api/app/bsky/unspecced.ts b/packages/pds/src/app-view/api/app/bsky/unspecced.ts index 482adb3dbd8..8708b8fe92f 100644 --- a/packages/pds/src/app-view/api/app/bsky/unspecced.ts +++ b/packages/pds/src/app-view/api/app/bsky/unspecced.ts @@ -1,4 +1,4 @@ -import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' +import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import { GenericKeyset } from '../../../../db/pagination' import AppContext from '../../../../context' @@ -20,11 +20,6 @@ export default function (server: Server, ctx: AppContext) { encoding: 'application/json', body: res.data, } - - return { - encoding: 'application/json', - body: { feed: [] }, - } }, }) @@ -43,18 +38,6 @@ export default function (server: Server, ctx: AppContext) { } }, }) - - server.app.bsky.unspecced.applyLabels({ - auth: ctx.roleVerifier, - handler: async ({ auth, input }) => { - if (!auth.credentials.admin) { - throw new AuthRequiredError('Insufficient privileges') - } - const { services, db } = ctx - const { labels } = input.body - await services.appView.label(db).createLabels(labels) - }, - }) } type Result = { likeCount: number; cid: string } diff --git a/packages/pds/src/app-view/db/index.ts b/packages/pds/src/app-view/db/index.ts deleted file mode 100644 index 21edd709a7a..00000000000 --- a/packages/pds/src/app-view/db/index.ts +++ /dev/null @@ -1,38 +0,0 @@ -import * as duplicateRecords from './tables/duplicate-record' -import * as profile from './tables/profile' -import * as profileAgg from './tables/profile-agg' -import * as post from './tables/post' -import * as postAgg from './tables/post-agg' -import * as postEmbed from './tables/post-embed' -import * as repost from './tables/repost' -import * as feedItem from './tables/feed-item' -import * as follow from './tables/follow' -import * as list from './tables/list' -import * as listItem from './tables/list-item' -import * as actorBlock from './tables/actor-block' -import * as like from './tables/like' -import * as feedGenerator from './tables/feed-generator' -import * as subscription from './tables/subscription' -import * as algo from './tables/algo' -import * as viewParam from './tables/view-param' -import * as suggestedFollow from './tables/suggested-follow' - -// @NOTE app-view also shares did-handle, record, and repo-root tables w/ main pds -export type DatabaseSchemaType = duplicateRecords.PartialDB & - profile.PartialDB & - profileAgg.PartialDB & - post.PartialDB & - postAgg.PartialDB & - postEmbed.PartialDB & - repost.PartialDB & - feedItem.PartialDB & - follow.PartialDB & - list.PartialDB & - listItem.PartialDB & - actorBlock.PartialDB & - like.PartialDB & - feedGenerator.PartialDB & - subscription.PartialDB & - algo.PartialDB & - viewParam.PartialDB & - suggestedFollow.PartialDB diff --git a/packages/pds/src/app-view/db/tables/actor-block.ts b/packages/pds/src/app-view/db/tables/actor-block.ts deleted file mode 100644 index 8923e921071..00000000000 --- a/packages/pds/src/app-view/db/tables/actor-block.ts +++ /dev/null @@ -1,11 +0,0 @@ -export const tableName = 'actor_block' -export interface ActorBlock { - uri: string - cid: string - creator: string - subjectDid: string - createdAt: string - indexedAt: string -} - -export type PartialDB = { [tableName]: ActorBlock } diff --git a/packages/pds/src/app-view/db/tables/algo.ts b/packages/pds/src/app-view/db/tables/algo.ts deleted file mode 100644 index e38a64c4a69..00000000000 --- a/packages/pds/src/app-view/db/tables/algo.ts +++ /dev/null @@ -1,12 +0,0 @@ -// @NOTE postgres-only -export const whatsHotViewTableName = 'algo_whats_hot_view' - -export interface AlgoWhatsHotView { - uri: string - cid: string - score: number -} - -export type PartialDB = { - [whatsHotViewTableName]: AlgoWhatsHotView -} diff --git a/packages/pds/src/app-view/db/tables/duplicate-record.ts b/packages/pds/src/app-view/db/tables/duplicate-record.ts deleted file mode 100644 index 3cad0cd148b..00000000000 --- a/packages/pds/src/app-view/db/tables/duplicate-record.ts +++ /dev/null @@ -1,12 +0,0 @@ -export interface DuplicateRecord { - uri: string - cid: string - duplicateOf: string - indexedAt: string -} - -export const tableName = 'duplicate_record' - -export type PartialDB = { - [tableName]: DuplicateRecord -} diff --git a/packages/pds/src/app-view/db/tables/feed-generator.ts b/packages/pds/src/app-view/db/tables/feed-generator.ts deleted file mode 100644 index 0b33f05cb12..00000000000 --- a/packages/pds/src/app-view/db/tables/feed-generator.ts +++ /dev/null @@ -1,18 +0,0 @@ -export const tableName = 'feed_generator' - -export interface FeedGenerator { - uri: string - cid: string - creator: string - feedDid: string - displayName: string - description: string | null - descriptionFacets: string | null - avatarCid: string | null - createdAt: string - indexedAt: string -} - -export type PartialDB = { - [tableName]: FeedGenerator -} diff --git a/packages/pds/src/app-view/db/tables/feed-item.ts b/packages/pds/src/app-view/db/tables/feed-item.ts deleted file mode 100644 index 2fb7f867311..00000000000 --- a/packages/pds/src/app-view/db/tables/feed-item.ts +++ /dev/null @@ -1,12 +0,0 @@ -export const tableName = 'feed_item' - -export interface FeedItem { - uri: string - cid: string - type: 'post' | 'repost' - postUri: string - originatorDid: string - sortAt: string -} - -export type PartialDB = { [tableName]: FeedItem } diff --git a/packages/pds/src/app-view/db/tables/follow.ts b/packages/pds/src/app-view/db/tables/follow.ts deleted file mode 100644 index b6ba34aea43..00000000000 --- a/packages/pds/src/app-view/db/tables/follow.ts +++ /dev/null @@ -1,11 +0,0 @@ -export const tableName = 'follow' -export interface Follow { - uri: string - cid: string - creator: string - subjectDid: string - createdAt: string - indexedAt: string -} - -export type PartialDB = { [tableName]: Follow } diff --git a/packages/pds/src/app-view/db/tables/like.ts b/packages/pds/src/app-view/db/tables/like.ts deleted file mode 100644 index 48edac21fde..00000000000 --- a/packages/pds/src/app-view/db/tables/like.ts +++ /dev/null @@ -1,13 +0,0 @@ -export interface Like { - uri: string - cid: string - creator: string - subject: string - subjectCid: string - createdAt: string - indexedAt: string -} - -const tableName = 'like' - -export type PartialDB = { [tableName]: Like } diff --git a/packages/pds/src/app-view/db/tables/list-item.ts b/packages/pds/src/app-view/db/tables/list-item.ts deleted file mode 100644 index 7e70e718169..00000000000 --- a/packages/pds/src/app-view/db/tables/list-item.ts +++ /dev/null @@ -1,13 +0,0 @@ -export const tableName = 'list_item' - -export interface ListItem { - uri: string - cid: string - creator: string - subjectDid: string - listUri: string - createdAt: string - indexedAt: string -} - -export type PartialDB = { [tableName]: ListItem } diff --git a/packages/pds/src/app-view/db/tables/list.ts b/packages/pds/src/app-view/db/tables/list.ts deleted file mode 100644 index d2db17574d2..00000000000 --- a/packages/pds/src/app-view/db/tables/list.ts +++ /dev/null @@ -1,16 +0,0 @@ -export const tableName = 'list' - -export interface List { - uri: string - cid: string - creator: string - name: string - purpose: string - description: string | null - descriptionFacets: string | null - avatarCid: string | null - createdAt: string - indexedAt: string -} - -export type PartialDB = { [tableName]: List } diff --git a/packages/pds/src/app-view/db/tables/post-agg.ts b/packages/pds/src/app-view/db/tables/post-agg.ts deleted file mode 100644 index 5341347403d..00000000000 --- a/packages/pds/src/app-view/db/tables/post-agg.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { Generated } from 'kysely' - -export const tableName = 'post_agg' - -export interface PostAgg { - uri: string - likeCount: Generated - replyCount: Generated - repostCount: Generated -} - -export type PartialDB = { - [tableName]: PostAgg -} diff --git a/packages/pds/src/app-view/db/tables/post-embed.ts b/packages/pds/src/app-view/db/tables/post-embed.ts deleted file mode 100644 index 69c44efe718..00000000000 --- a/packages/pds/src/app-view/db/tables/post-embed.ts +++ /dev/null @@ -1,30 +0,0 @@ -export const imageTableName = 'post_embed_image' -export const externalTableName = 'post_embed_external' -export const recordTableName = 'post_embed_record' - -export interface PostEmbedImage { - postUri: string - position: number - imageCid: string - alt: string -} - -export interface PostEmbedExternal { - postUri: string - uri: string - title: string - description: string - thumbCid: string | null -} - -export interface PostEmbedRecord { - postUri: string - embedUri: string - embedCid: string -} - -export type PartialDB = { - [imageTableName]: PostEmbedImage - [externalTableName]: PostEmbedExternal - [recordTableName]: PostEmbedRecord -} diff --git a/packages/pds/src/app-view/db/tables/post.ts b/packages/pds/src/app-view/db/tables/post.ts deleted file mode 100644 index ce7c2b7acbe..00000000000 --- a/packages/pds/src/app-view/db/tables/post.ts +++ /dev/null @@ -1,18 +0,0 @@ -export const tableName = 'post' - -export interface Post { - uri: string - cid: string - creator: string - text: string - replyRoot: string | null - replyRootCid: string | null - replyParent: string | null - replyParentCid: string | null - createdAt: string - indexedAt: string -} - -export type PartialDB = { - [tableName]: Post -} diff --git a/packages/pds/src/app-view/db/tables/profile-agg.ts b/packages/pds/src/app-view/db/tables/profile-agg.ts deleted file mode 100644 index 333a5136461..00000000000 --- a/packages/pds/src/app-view/db/tables/profile-agg.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { Generated } from 'kysely' - -export const tableName = 'profile_agg' - -export interface ProfileAgg { - did: string - followersCount: Generated - followsCount: Generated - postsCount: Generated -} - -export type PartialDB = { - [tableName]: ProfileAgg -} diff --git a/packages/pds/src/app-view/db/tables/profile.ts b/packages/pds/src/app-view/db/tables/profile.ts deleted file mode 100644 index 6f8fac50aa5..00000000000 --- a/packages/pds/src/app-view/db/tables/profile.ts +++ /dev/null @@ -1,13 +0,0 @@ -export const tableName = 'profile' - -export interface Profile { - uri: string - cid: string - creator: string - displayName: string | null - description: string | null - avatarCid: string | null - bannerCid: string | null - indexedAt: string -} -export type PartialDB = { [tableName]: Profile } diff --git a/packages/pds/src/app-view/db/tables/repost.ts b/packages/pds/src/app-view/db/tables/repost.ts deleted file mode 100644 index 8a0fadd6611..00000000000 --- a/packages/pds/src/app-view/db/tables/repost.ts +++ /dev/null @@ -1,13 +0,0 @@ -export const tableName = 'repost' - -export interface Repost { - uri: string - cid: string - creator: string - subject: string - subjectCid: string - createdAt: string - indexedAt: string -} - -export type PartialDB = { [tableName]: Repost } diff --git a/packages/pds/src/app-view/db/tables/subscription.ts b/packages/pds/src/app-view/db/tables/subscription.ts deleted file mode 100644 index 62a49d0b29b..00000000000 --- a/packages/pds/src/app-view/db/tables/subscription.ts +++ /dev/null @@ -1,9 +0,0 @@ -export const tableName = 'subscription' - -export interface Subscription { - service: string - method: string - state: string -} - -export type PartialDB = { [tableName]: Subscription } diff --git a/packages/pds/src/app-view/db/tables/suggested-follow.ts b/packages/pds/src/app-view/db/tables/suggested-follow.ts deleted file mode 100644 index 6ad10f43203..00000000000 --- a/packages/pds/src/app-view/db/tables/suggested-follow.ts +++ /dev/null @@ -1,10 +0,0 @@ -export const tableName = 'suggested_follow' - -export interface SuggestedFollow { - did: string - order: number -} - -export type PartialDB = { - [tableName]: SuggestedFollow -} diff --git a/packages/pds/src/app-view/db/tables/view-param.ts b/packages/pds/src/app-view/db/tables/view-param.ts deleted file mode 100644 index 07e3b08f0cf..00000000000 --- a/packages/pds/src/app-view/db/tables/view-param.ts +++ /dev/null @@ -1,12 +0,0 @@ -// @NOTE postgres-only -export const tableName = 'view_param' - -// materialized views are difficult to change, -// so we parameterize them at runtime with contents of this table. -// its contents are set in migrations, available param names are static. -export interface ViewParam { - name: string - value: string -} - -export type PartialDB = { [tableName]: ViewParam } diff --git a/packages/pds/src/app-view/event-stream/consumers.ts b/packages/pds/src/app-view/event-stream/consumers.ts deleted file mode 100644 index 48fc963fb09..00000000000 --- a/packages/pds/src/app-view/event-stream/consumers.ts +++ /dev/null @@ -1,38 +0,0 @@ -import AppContext from '../../context' -import Database from '../../db' -import { - DeleteRecord, - IndexRecord, - DeleteRepo, -} from '../../event-stream/messages' - -// Used w/ in-process PDS as alternative to the repo subscription -export const listen = (ctx: AppContext) => { - ctx.messageDispatcher.listen('index_record', { - async listener(input: { db: Database; message: IndexRecord }) { - const { db, message } = input - const indexingService = ctx.services.appView.indexing(db) - await indexingService.indexRecord( - message.uri, - message.cid, - message.obj, - message.action, - message.timestamp, - ) - }, - }) - ctx.messageDispatcher.listen('delete_record', { - async listener(input: { db: Database; message: DeleteRecord }) { - const { db, message } = input - const indexingService = ctx.services.appView.indexing(db) - await indexingService.deleteRecord(message.uri, message.cascading) - }, - }) - ctx.messageDispatcher.listen('delete_repo', { - async listener(input: { db: Database; message: DeleteRepo }) { - const { db, message } = input - const indexingService = ctx.services.appView.indexing(db) - await indexingService.deleteForUser(message.did) - }, - }) -} diff --git a/packages/pds/src/app-view/services/indexing/index.ts b/packages/pds/src/app-view/services/indexing/index.ts deleted file mode 100644 index 346c31e83ea..00000000000 --- a/packages/pds/src/app-view/services/indexing/index.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { CID } from 'multiformats/cid' -import { WriteOpAction } from '@atproto/repo' -import { AtUri } from '@atproto/syntax' -import { ids } from '../../../lexicon/lexicons' -import Database from '../../../db' -import { BackgroundQueue } from '../../../event-stream/background-queue' -import { NoopProcessor } from './processor' -import * as Post from './plugins/post' -import * as Like from './plugins/like' -import * as Repost from './plugins/repost' -import * as Follow from './plugins/follow' -import * as Block from './plugins/block' -import * as List from './plugins/list' -import * as ListItem from './plugins/list-item' -import * as Profile from './plugins/profile' -import * as FeedGenerator from './plugins/feed-generator' - -export class IndexingService { - records: { - post: Post.PluginType - like: Like.PluginType - repost: Repost.PluginType - follow: Follow.PluginType - block: Block.PluginType - list: List.PluginType - listItem: ListItem.PluginType - listBlock: NoopProcessor - profile: Profile.PluginType - feedGenerator: FeedGenerator.PluginType - } - - constructor(public db: Database, public backgroundQueue: BackgroundQueue) { - this.records = { - post: Post.makePlugin(this.db, backgroundQueue), - like: Like.makePlugin(this.db, backgroundQueue), - repost: Repost.makePlugin(this.db, backgroundQueue), - follow: Follow.makePlugin(this.db, backgroundQueue), - block: Block.makePlugin(this.db, backgroundQueue), - list: List.makePlugin(this.db, backgroundQueue), - listItem: ListItem.makePlugin(this.db, backgroundQueue), - listBlock: new NoopProcessor( - ids.AppBskyGraphListblock, - this.db, - backgroundQueue, - ), - profile: Profile.makePlugin(this.db, backgroundQueue), - feedGenerator: FeedGenerator.makePlugin(this.db, backgroundQueue), - } - } - - static creator(backgroundQueue: BackgroundQueue) { - return (db: Database) => new IndexingService(db, backgroundQueue) - } - - async indexRecord( - uri: AtUri, - cid: CID, - obj: unknown, - action: WriteOpAction.Create | WriteOpAction.Update, - timestamp: string, - ) { - this.db.assertTransaction() - const indexer = this.findIndexerForCollection(uri.collection) - if (action === WriteOpAction.Create) { - await indexer.insertRecord(uri, cid, obj, timestamp) - } else { - await indexer.updateRecord(uri, cid, obj, timestamp) - } - } - - async deleteRecord(uri: AtUri, cascading = false) { - this.db.assertTransaction() - const indexer = this.findIndexerForCollection(uri.collection) - await indexer.deleteRecord(uri, cascading) - } - - findIndexerForCollection(collection: string) { - const found = Object.values(this.records).find( - (plugin) => plugin.collection === collection, - ) - if (!found) { - throw new Error('Could not find indexer for collection') - } - return found - } - - async deleteForUser(did: string) { - // Not done in transaction because it would be too long, prone to contention. - // Also, this can safely be run multiple times if it fails. - // Omitting updates to profile_agg and post_agg since it's expensive - // and they'll organically update themselves over time. - - const postByUser = (qb) => - qb - .selectFrom('post') - .where('post.creator', '=', did) - .select('post.uri as uri') - - await this.db.db - .deleteFrom('post_embed_image') - .where('post_embed_image.postUri', 'in', postByUser) - .execute() - await this.db.db - .deleteFrom('post_embed_external') - .where('post_embed_external.postUri', 'in', postByUser) - .execute() - await this.db.db - .deleteFrom('post_embed_record') - .where('post_embed_record.postUri', 'in', postByUser) - .execute() - await this.db.db - .deleteFrom('duplicate_record') - .where('duplicate_record.duplicateOf', 'in', (qb) => - // @TODO remove dependency on record table from app view - qb - .selectFrom('record') - .where('record.did', '=', did) - .select('record.uri as uri'), - ) - .execute() - await this.db.db - .deleteFrom('actor_block') - .where('creator', '=', did) - .execute() - await this.db.db.deleteFrom('list').where('creator', '=', did).execute() - await this.db.db - .deleteFrom('list_item') - .where('creator', '=', did) - .execute() - await this.db.db.deleteFrom('follow').where('creator', '=', did).execute() - await this.db.db.deleteFrom('post').where('creator', '=', did).execute() - await this.db.db.deleteFrom('profile').where('creator', '=', did).execute() - await this.db.db.deleteFrom('repost').where('creator', '=', did).execute() - await this.db.db.deleteFrom('like').where('creator', '=', did).execute() - } -} diff --git a/packages/pds/src/app-view/services/indexing/plugins/block.ts b/packages/pds/src/app-view/services/indexing/plugins/block.ts deleted file mode 100644 index bb30071c4d1..00000000000 --- a/packages/pds/src/app-view/services/indexing/plugins/block.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { AtUri } from '@atproto/syntax' -import { CID } from 'multiformats/cid' -import * as Block from '../../../../lexicon/types/app/bsky/graph/block' -import * as lex from '../../../../lexicon/lexicons' -import Database from '../../../../db' -import { - DatabaseSchema, - DatabaseSchemaType, -} from '../../../../db/database-schema' -import { BackgroundQueue } from '../../../../event-stream/background-queue' -import RecordProcessor from '../processor' -import { toSimplifiedISOSafe } from '../util' - -const lexId = lex.ids.AppBskyGraphBlock -type IndexedBlock = DatabaseSchemaType['actor_block'] - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: Block.Record, - timestamp: string, -): Promise => { - const inserted = await db - .insertInto('actor_block') - .values({ - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - subjectDid: obj.subject, - createdAt: toSimplifiedISOSafe(obj.createdAt), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst() - return inserted || null -} - -const findDuplicate = async ( - db: DatabaseSchema, - uri: AtUri, - obj: Block.Record, -): Promise => { - const found = await db - .selectFrom('actor_block') - .where('creator', '=', uri.host) - .where('subjectDid', '=', obj.subject) - .selectAll() - .executeTakeFirst() - return found ? new AtUri(found.uri) : null -} - -const notifsForInsert = () => { - return [] -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const deleted = await db - .deleteFrom('actor_block') - .where('uri', '=', uri.toString()) - .returningAll() - .executeTakeFirst() - return deleted || null -} - -const notifsForDelete = ( - deleted: IndexedBlock, - replacedBy: IndexedBlock | null, -) => { - const toDelete = replacedBy ? [] : [deleted.uri] - return { notifs: [], toDelete } -} - -const updateAggregates = async () => {} - -export type PluginType = RecordProcessor - -export const makePlugin = ( - db: Database, - backgroundQueue: BackgroundQueue, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - updateAggregates, - }) -} - -export default makePlugin diff --git a/packages/pds/src/app-view/services/indexing/plugins/feed-generator.ts b/packages/pds/src/app-view/services/indexing/plugins/feed-generator.ts deleted file mode 100644 index c18da08d51c..00000000000 --- a/packages/pds/src/app-view/services/indexing/plugins/feed-generator.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { AtUri } from '@atproto/syntax' -import { CID } from 'multiformats/cid' -import * as FeedGenerator from '../../../../lexicon/types/app/bsky/feed/generator' -import * as lex from '../../../../lexicon/lexicons' -import Database from '../../../../db' -import { - DatabaseSchema, - DatabaseSchemaType, -} from '../../../../db/database-schema' -import { BackgroundQueue } from '../../../../event-stream/background-queue' -import RecordProcessor from '../processor' -import { toSimplifiedISOSafe } from '../util' - -const lexId = lex.ids.AppBskyFeedGenerator -type IndexedFeedGenerator = DatabaseSchemaType['feed_generator'] - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: FeedGenerator.Record, - timestamp: string, -): Promise => { - const inserted = await db - .insertInto('feed_generator') - .values({ - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - feedDid: obj.did, - displayName: obj.displayName, - description: obj.description, - descriptionFacets: obj.descriptionFacets - ? JSON.stringify(obj.descriptionFacets) - : undefined, - avatarCid: obj.avatar?.ref.toString(), - createdAt: toSimplifiedISOSafe(obj.createdAt), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst() - return inserted || null -} - -const findDuplicate = async (): Promise => { - return null -} - -const notifsForInsert = () => { - return [] -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const deleted = await db - .deleteFrom('feed_generator') - .where('uri', '=', uri.toString()) - .returningAll() - .executeTakeFirst() - return deleted || null -} - -const notifsForDelete = () => { - return { notifs: [], toDelete: [] } -} - -export type PluginType = RecordProcessor< - FeedGenerator.Record, - IndexedFeedGenerator -> - -export const makePlugin = ( - db: Database, - backgroundQueue: BackgroundQueue, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - }) -} - -export default makePlugin diff --git a/packages/pds/src/app-view/services/indexing/plugins/follow.ts b/packages/pds/src/app-view/services/indexing/plugins/follow.ts deleted file mode 100644 index c405f288b76..00000000000 --- a/packages/pds/src/app-view/services/indexing/plugins/follow.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { AtUri } from '@atproto/syntax' -import { CID } from 'multiformats/cid' -import * as Follow from '../../../../lexicon/types/app/bsky/graph/follow' -import * as lex from '../../../../lexicon/lexicons' -import Database from '../../../../db' -import { - DatabaseSchema, - DatabaseSchemaType, -} from '../../../../db/database-schema' -import { countAll, excluded } from '../../../../db/util' -import { BackgroundQueue } from '../../../../event-stream/background-queue' -import RecordProcessor from '../processor' -import { toSimplifiedISOSafe } from '../util' - -const lexId = lex.ids.AppBskyGraphFollow -type IndexedFollow = DatabaseSchemaType['follow'] - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: Follow.Record, - timestamp: string, -): Promise => { - const inserted = await db - .insertInto('follow') - .values({ - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - subjectDid: obj.subject, - createdAt: toSimplifiedISOSafe(obj.createdAt), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst() - return inserted || null -} - -const findDuplicate = async ( - db: DatabaseSchema, - uri: AtUri, - obj: Follow.Record, -): Promise => { - const found = await db - .selectFrom('follow') - .where('creator', '=', uri.host) - .where('subjectDid', '=', obj.subject) - .selectAll() - .executeTakeFirst() - return found ? new AtUri(found.uri) : null -} - -const notifsForInsert = (obj: IndexedFollow) => { - return [ - { - userDid: obj.subjectDid, - author: obj.creator, - recordUri: obj.uri, - recordCid: obj.cid, - reason: 'follow' as const, - reasonSubject: null, - indexedAt: obj.indexedAt, - }, - ] -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const deleted = await db - .deleteFrom('follow') - .where('uri', '=', uri.toString()) - .returningAll() - .executeTakeFirst() - return deleted || null -} - -const notifsForDelete = ( - deleted: IndexedFollow, - replacedBy: IndexedFollow | null, -) => { - const toDelete = replacedBy ? [] : [deleted.uri] - return { notifs: [], toDelete } -} - -const updateAggregates = async (db: DatabaseSchema, follow: IndexedFollow) => { - const followersCountQb = db - .insertInto('profile_agg') - .values({ - did: follow.subjectDid, - followersCount: db - .selectFrom('follow') - .where('follow.subjectDid', '=', follow.subjectDid) - .select(countAll.as('count')), - }) - .onConflict((oc) => - oc.column('did').doUpdateSet({ - followersCount: excluded(db, 'followersCount'), - }), - ) - const followsCountQb = db - .insertInto('profile_agg') - .values({ - did: follow.creator, - followsCount: db - .selectFrom('follow') - .where('follow.creator', '=', follow.creator) - .select(countAll.as('count')), - }) - .onConflict((oc) => - oc.column('did').doUpdateSet({ - followsCount: excluded(db, 'followsCount'), - }), - ) - await Promise.all([followersCountQb.execute(), followsCountQb.execute()]) -} - -export type PluginType = RecordProcessor - -export const makePlugin = ( - db: Database, - backgroundQueue: BackgroundQueue, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - updateAggregates, - }) -} - -export default makePlugin diff --git a/packages/pds/src/app-view/services/indexing/plugins/like.ts b/packages/pds/src/app-view/services/indexing/plugins/like.ts deleted file mode 100644 index 32d87ebe177..00000000000 --- a/packages/pds/src/app-view/services/indexing/plugins/like.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { AtUri } from '@atproto/syntax' -import { CID } from 'multiformats/cid' -import * as Like from '../../../../lexicon/types/app/bsky/feed/like' -import * as lex from '../../../../lexicon/lexicons' -import Database from '../../../../db' -import { - DatabaseSchema, - DatabaseSchemaType, -} from '../../../../db/database-schema' -import { countAll, excluded } from '../../../../db/util' -import { BackgroundQueue } from '../../../../event-stream/background-queue' -import RecordProcessor from '../processor' -import { toSimplifiedISOSafe } from '../util' - -const lexId = lex.ids.AppBskyFeedLike -type IndexedLike = DatabaseSchemaType['like'] - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: Like.Record, - timestamp: string, -): Promise => { - const inserted = await db - .insertInto('like') - .values({ - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - subject: obj.subject.uri, - subjectCid: obj.subject.cid, - createdAt: toSimplifiedISOSafe(obj.createdAt), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst() - return inserted || null -} - -const findDuplicate = async ( - db: DatabaseSchema, - uri: AtUri, - obj: Like.Record, -): Promise => { - const found = await db - .selectFrom('like') - .where('creator', '=', uri.host) - .where('subject', '=', obj.subject.uri) - .selectAll() - .executeTakeFirst() - return found ? new AtUri(found.uri) : null -} - -const notifsForInsert = (obj: IndexedLike) => { - const subjectUri = new AtUri(obj.subject) - // prevent self-notifications - const isSelf = subjectUri.host === obj.creator - return isSelf - ? [] - : [ - { - userDid: subjectUri.host, - author: obj.creator, - recordUri: obj.uri, - recordCid: obj.cid, - reason: 'like' as const, - reasonSubject: subjectUri.toString(), - indexedAt: obj.indexedAt, - }, - ] -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const deleted = await db - .deleteFrom('like') - .where('uri', '=', uri.toString()) - .returningAll() - .executeTakeFirst() - return deleted || null -} - -const notifsForDelete = ( - deleted: IndexedLike, - replacedBy: IndexedLike | null, -) => { - const toDelete = replacedBy ? [] : [deleted.uri] - return { notifs: [], toDelete } -} - -const updateAggregates = async (db: DatabaseSchema, like: IndexedLike) => { - const likeCountQb = db - .insertInto('post_agg') - .values({ - uri: like.subject, - likeCount: db - .selectFrom('like') - .where('like.subject', '=', like.subject) - .select(countAll.as('count')), - }) - .onConflict((oc) => - oc.column('uri').doUpdateSet({ likeCount: excluded(db, 'likeCount') }), - ) - await likeCountQb.execute() -} - -export type PluginType = RecordProcessor - -export const makePlugin = ( - db: Database, - backgroundQueue: BackgroundQueue, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - updateAggregates, - }) -} - -export default makePlugin diff --git a/packages/pds/src/app-view/services/indexing/plugins/list-item.ts b/packages/pds/src/app-view/services/indexing/plugins/list-item.ts deleted file mode 100644 index 33d0c7e8152..00000000000 --- a/packages/pds/src/app-view/services/indexing/plugins/list-item.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { AtUri } from '@atproto/syntax' -import { CID } from 'multiformats/cid' -import * as ListItem from '../../../../lexicon/types/app/bsky/graph/listitem' -import * as lex from '../../../../lexicon/lexicons' -import Database from '../../../../db' -import { - DatabaseSchema, - DatabaseSchemaType, -} from '../../../../db/database-schema' -import { BackgroundQueue } from '../../../../event-stream/background-queue' -import RecordProcessor from '../processor' -import { InvalidRequestError } from '@atproto/xrpc-server' -import { toSimplifiedISOSafe } from '../util' - -const lexId = lex.ids.AppBskyGraphListitem -type IndexedListItem = DatabaseSchemaType['list_item'] - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: ListItem.Record, - timestamp: string, -): Promise => { - const listUri = new AtUri(obj.list) - if (listUri.hostname !== uri.hostname) { - throw new InvalidRequestError( - 'Creator of listitem does not match creator of list', - ) - } - const inserted = await db - .insertInto('list_item') - .values({ - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - subjectDid: obj.subject, - listUri: obj.list, - createdAt: toSimplifiedISOSafe(obj.createdAt), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst() - return inserted || null -} - -const findDuplicate = async ( - db: DatabaseSchema, - _uri: AtUri, - obj: ListItem.Record, -): Promise => { - const found = await db - .selectFrom('list_item') - .where('listUri', '=', obj.list) - .where('subjectDid', '=', obj.subject) - .selectAll() - .executeTakeFirst() - return found ? new AtUri(found.uri) : null -} - -const notifsForInsert = () => { - return [] -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const deleted = await db - .deleteFrom('list_item') - .where('uri', '=', uri.toString()) - .returningAll() - .executeTakeFirst() - return deleted || null -} - -const notifsForDelete = ( - deleted: IndexedListItem, - replacedBy: IndexedListItem | null, -) => { - const toDelete = replacedBy ? [] : [deleted.uri] - return { notifs: [], toDelete } -} - -export type PluginType = RecordProcessor - -export const makePlugin = ( - db: Database, - backgroundQueue: BackgroundQueue, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - }) -} - -export default makePlugin diff --git a/packages/pds/src/app-view/services/indexing/plugins/list.ts b/packages/pds/src/app-view/services/indexing/plugins/list.ts deleted file mode 100644 index b0250c5be39..00000000000 --- a/packages/pds/src/app-view/services/indexing/plugins/list.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { AtUri } from '@atproto/syntax' -import { CID } from 'multiformats/cid' -import * as List from '../../../../lexicon/types/app/bsky/graph/list' -import * as lex from '../../../../lexicon/lexicons' -import Database from '../../../../db' -import { - DatabaseSchema, - DatabaseSchemaType, -} from '../../../../db/database-schema' -import { BackgroundQueue } from '../../../../event-stream/background-queue' -import RecordProcessor from '../processor' -import { toSimplifiedISOSafe } from '../util' - -const lexId = lex.ids.AppBskyGraphList -type IndexedList = DatabaseSchemaType['list'] - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: List.Record, - timestamp: string, -): Promise => { - const inserted = await db - .insertInto('list') - .values({ - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - name: obj.name, - purpose: obj.purpose, - description: obj.description, - descriptionFacets: obj.descriptionFacets - ? JSON.stringify(obj.descriptionFacets) - : undefined, - avatarCid: obj.avatar?.ref.toString(), - createdAt: toSimplifiedISOSafe(obj.createdAt), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst() - return inserted || null -} - -const findDuplicate = async (): Promise => { - return null -} - -const notifsForInsert = () => { - return [] -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const deleted = await db - .deleteFrom('list') - .where('uri', '=', uri.toString()) - .returningAll() - .executeTakeFirst() - return deleted || null -} - -const notifsForDelete = () => { - return { notifs: [], toDelete: [] } -} - -export type PluginType = RecordProcessor - -export const makePlugin = ( - db: Database, - backgroundQueue: BackgroundQueue, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - }) -} - -export default makePlugin diff --git a/packages/pds/src/app-view/services/indexing/plugins/post.ts b/packages/pds/src/app-view/services/indexing/plugins/post.ts deleted file mode 100644 index efc1f2532c7..00000000000 --- a/packages/pds/src/app-view/services/indexing/plugins/post.ts +++ /dev/null @@ -1,326 +0,0 @@ -import { CID } from 'multiformats/cid' -import { AtUri } from '@atproto/syntax' -import { Record as PostRecord } from '../../../../lexicon/types/app/bsky/feed/post' -import { isMain as isEmbedImage } from '../../../../lexicon/types/app/bsky/embed/images' -import { isMain as isEmbedExternal } from '../../../../lexicon/types/app/bsky/embed/external' -import { isMain as isEmbedRecord } from '../../../../lexicon/types/app/bsky/embed/record' -import { isMain as isEmbedRecordWithMedia } from '../../../../lexicon/types/app/bsky/embed/recordWithMedia' -import { - isMention, - isLink, -} from '../../../../lexicon/types/app/bsky/richtext/facet' -import * as lex from '../../../../lexicon/lexicons' -import Database from '../../../../db' -import { - DatabaseSchema, - DatabaseSchemaType, -} from '../../../../db/database-schema' -import { BackgroundQueue } from '../../../../event-stream/background-queue' -import RecordProcessor from '../processor' -import { UserNotification } from '../../../../db/tables/user-notification' -import { countAll, excluded } from '../../../../db/util' -import { toSimplifiedISOSafe } from '../util' - -type Post = DatabaseSchemaType['post'] -type PostEmbedImage = DatabaseSchemaType['post_embed_image'] -type PostEmbedExternal = DatabaseSchemaType['post_embed_external'] -type PostEmbedRecord = DatabaseSchemaType['post_embed_record'] -type PostAncestor = { - uri: string - height: number -} -type IndexedPost = { - post: Post - facets: { type: 'mention' | 'link'; value: string }[] - embeds?: (PostEmbedImage[] | PostEmbedExternal | PostEmbedRecord)[] - ancestors?: PostAncestor[] -} - -const lexId = lex.ids.AppBskyFeedPost - -const REPLY_NOTIF_DEPTH = 5 - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: PostRecord, - timestamp: string, -): Promise => { - const post = { - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - text: obj.text, - createdAt: toSimplifiedISOSafe(obj.createdAt), - replyRoot: obj.reply?.root?.uri || null, - replyRootCid: obj.reply?.root?.cid || null, - replyParent: obj.reply?.parent?.uri || null, - replyParentCid: obj.reply?.parent?.cid || null, - indexedAt: timestamp, - } - const [insertedPost] = await Promise.all([ - db - .insertInto('post') - .values(post) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst(), - db - .insertInto('feed_item') - .values({ - type: 'post', - uri: post.uri, - cid: post.cid, - postUri: post.uri, - originatorDid: post.creator, - sortAt: - post.indexedAt < post.createdAt ? post.indexedAt : post.createdAt, - }) - .onConflict((oc) => oc.doNothing()) - .executeTakeFirst(), - ]) - if (!insertedPost) { - return null // Post already indexed - } - - const facets = (obj.facets || []) - .flatMap((facet) => facet.features) - .flatMap((feature) => { - if (isMention(feature)) { - return { - type: 'mention' as const, - value: feature.did, - } - } - if (isLink(feature)) { - return { - type: 'link' as const, - value: feature.uri, - } - } - return [] - }) - // Embed indices - const embeds: (PostEmbedImage[] | PostEmbedExternal | PostEmbedRecord)[] = [] - const postEmbeds = separateEmbeds(obj.embed) - for (const postEmbed of postEmbeds) { - if (isEmbedImage(postEmbed)) { - const { images } = postEmbed - const imagesEmbed = images.map((img, i) => ({ - postUri: uri.toString(), - position: i, - imageCid: img.image.ref.toString(), - alt: img.alt, - })) - embeds.push(imagesEmbed) - await db.insertInto('post_embed_image').values(imagesEmbed).execute() - } else if (isEmbedExternal(postEmbed)) { - const { external } = postEmbed - const externalEmbed = { - postUri: uri.toString(), - uri: external.uri, - title: external.title, - description: external.description, - thumbCid: external.thumb?.ref.toString() || null, - } - embeds.push(externalEmbed) - await db.insertInto('post_embed_external').values(externalEmbed).execute() - } else if (isEmbedRecord(postEmbed)) { - const { record } = postEmbed - const recordEmbed = { - postUri: uri.toString(), - embedUri: record.uri, - embedCid: record.cid, - } - embeds.push(recordEmbed) - await db.insertInto('post_embed_record').values(recordEmbed).execute() - } - } - return { post: insertedPost, facets, embeds } -} - -const findDuplicate = async (): Promise => { - return null -} - -const notifsForInsert = (obj: IndexedPost) => { - const notifs: UserNotification[] = [] - const notified = new Set([obj.post.creator]) - const maybeNotify = (notif: UserNotification) => { - if (!notified.has(notif.userDid)) { - notified.add(notif.userDid) - notifs.push(notif) - } - } - for (const facet of obj.facets) { - if (facet.type === 'mention') { - maybeNotify({ - userDid: facet.value, - reason: 'mention', - reasonSubject: null, - author: obj.post.creator, - recordUri: obj.post.uri, - recordCid: obj.post.cid, - indexedAt: obj.post.indexedAt, - }) - } - } - for (const embed of obj.embeds ?? []) { - if ('embedUri' in embed) { - const embedUri = new AtUri(embed.embedUri) - if (embedUri.collection === lex.ids.AppBskyFeedPost) { - maybeNotify({ - userDid: embedUri.host, - reason: 'quote', - reasonSubject: embedUri.toString(), - author: obj.post.creator, - recordUri: obj.post.uri, - recordCid: obj.post.cid, - indexedAt: obj.post.indexedAt, - }) - } - } - } - for (const ancestor of obj.ancestors ?? []) { - if (ancestor.uri === obj.post.uri) continue // no need to notify for own post - if (ancestor.height < REPLY_NOTIF_DEPTH) { - const ancestorUri = new AtUri(ancestor.uri) - maybeNotify({ - userDid: ancestorUri.host, - reason: 'reply', - reasonSubject: ancestorUri.toString(), - author: obj.post.creator, - recordUri: obj.post.uri, - recordCid: obj.post.cid, - indexedAt: obj.post.indexedAt, - }) - } - } - return notifs -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const uriStr = uri.toString() - const [deleted] = await Promise.all([ - db - .deleteFrom('post') - .where('uri', '=', uriStr) - .returningAll() - .executeTakeFirst(), - db.deleteFrom('feed_item').where('postUri', '=', uriStr).executeTakeFirst(), - ]) - const deletedEmbeds: ( - | PostEmbedImage[] - | PostEmbedExternal - | PostEmbedRecord - )[] = [] - const [deletedImgs, deletedExternals, deletedPosts] = await Promise.all([ - db - .deleteFrom('post_embed_image') - .where('postUri', '=', uriStr) - .returningAll() - .execute(), - db - .deleteFrom('post_embed_external') - .where('postUri', '=', uriStr) - .returningAll() - .executeTakeFirst(), - db - .deleteFrom('post_embed_record') - .where('postUri', '=', uriStr) - .returningAll() - .executeTakeFirst(), - ]) - if (deletedImgs.length) { - deletedEmbeds.push(deletedImgs) - } - if (deletedExternals) { - deletedEmbeds.push(deletedExternals) - } - if (deletedPosts) { - deletedEmbeds.push(deletedPosts) - } - return deleted - ? { - post: deleted, - facets: [], // Not used - embeds: deletedEmbeds, - } - : null -} - -const notifsForDelete = ( - deleted: IndexedPost, - replacedBy: IndexedPost | null, -) => { - const notifs = replacedBy ? notifsForInsert(replacedBy) : [] - return { - notifs, - toDelete: [deleted.post.uri], - } -} - -const updateAggregates = async (db: DatabaseSchema, postIdx: IndexedPost) => { - const replyCountQb = postIdx.post.replyParent - ? db - .insertInto('post_agg') - .values({ - uri: postIdx.post.replyParent, - replyCount: db - .selectFrom('post') - .where('post.replyParent', '=', postIdx.post.replyParent) - .select(countAll.as('count')), - }) - .onConflict((oc) => - oc - .column('uri') - .doUpdateSet({ replyCount: excluded(db, 'replyCount') }), - ) - : null - const postsCountQb = db - .insertInto('profile_agg') - .values({ - did: postIdx.post.creator, - postsCount: db - .selectFrom('post') - .where('post.creator', '=', postIdx.post.creator) - .select(countAll.as('count')), - }) - .onConflict((oc) => - oc.column('did').doUpdateSet({ postsCount: excluded(db, 'postsCount') }), - ) - await Promise.all([replyCountQb?.execute(), postsCountQb.execute()]) -} - -export type PluginType = RecordProcessor - -export const makePlugin = ( - db: Database, - backgroundQueue: BackgroundQueue, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - updateAggregates, - }) -} - -export default makePlugin - -function separateEmbeds(embed: PostRecord['embed']) { - if (!embed) { - return [] - } - if (isEmbedRecordWithMedia(embed)) { - return [{ $type: lex.ids.AppBskyEmbedRecord, ...embed.record }, embed.media] - } - return [embed] -} diff --git a/packages/pds/src/app-view/services/indexing/plugins/profile.ts b/packages/pds/src/app-view/services/indexing/plugins/profile.ts deleted file mode 100644 index 3b8e6db506c..00000000000 --- a/packages/pds/src/app-view/services/indexing/plugins/profile.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { AtUri } from '@atproto/syntax' -import { CID } from 'multiformats/cid' -import * as Profile from '../../../../lexicon/types/app/bsky/actor/profile' -import * as lex from '../../../../lexicon/lexicons' -import Database from '../../../../db' -import { - DatabaseSchema, - DatabaseSchemaType, -} from '../../../../db/database-schema' -import { BackgroundQueue } from '../../../../event-stream/background-queue' -import RecordProcessor from '../processor' - -const lexId = lex.ids.AppBskyActorProfile -type IndexedProfile = DatabaseSchemaType['profile'] - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: Profile.Record, - timestamp: string, -): Promise => { - if (uri.rkey !== 'self') return null - const inserted = await db - .insertInto('profile') - .values({ - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - displayName: obj.displayName, - description: obj.description, - avatarCid: obj.avatar?.ref.toString(), - bannerCid: obj.banner?.ref.toString(), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst() - return inserted || null -} - -const findDuplicate = async (): Promise => { - return null -} - -const notifsForInsert = () => { - return [] -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const deleted = await db - .deleteFrom('profile') - .where('uri', '=', uri.toString()) - .returningAll() - .executeTakeFirst() - return deleted || null -} - -const notifsForDelete = () => { - return { notifs: [], toDelete: [] } -} - -export type PluginType = RecordProcessor - -export const makePlugin = ( - db: Database, - backgroundQueue: BackgroundQueue, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - }) -} - -export default makePlugin diff --git a/packages/pds/src/app-view/services/indexing/plugins/repost.ts b/packages/pds/src/app-view/services/indexing/plugins/repost.ts deleted file mode 100644 index fb05ca57ffd..00000000000 --- a/packages/pds/src/app-view/services/indexing/plugins/repost.ts +++ /dev/null @@ -1,153 +0,0 @@ -import { CID } from 'multiformats/cid' -import { AtUri } from '@atproto/syntax' -import * as Repost from '../../../../lexicon/types/app/bsky/feed/repost' -import * as lex from '../../../../lexicon/lexicons' -import Database from '../../../../db' -import { - DatabaseSchema, - DatabaseSchemaType, -} from '../../../../db/database-schema' -import { BackgroundQueue } from '../../../../event-stream/background-queue' -import RecordProcessor from '../processor' -import { countAll, excluded } from '../../../../db/util' -import { toSimplifiedISOSafe } from '../util' - -const lexId = lex.ids.AppBskyFeedRepost -type IndexedRepost = DatabaseSchemaType['repost'] - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: Repost.Record, - timestamp: string, -): Promise => { - const repost = { - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - subject: obj.subject.uri, - subjectCid: obj.subject.cid, - createdAt: toSimplifiedISOSafe(obj.createdAt), - indexedAt: timestamp, - } - const [inserted] = await Promise.all([ - db - .insertInto('repost') - .values(repost) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst(), - db - .insertInto('feed_item') - .values({ - type: 'repost', - uri: repost.uri, - cid: repost.cid, - postUri: repost.subject, - originatorDid: repost.creator, - sortAt: - repost.indexedAt < repost.createdAt - ? repost.indexedAt - : repost.createdAt, - }) - .onConflict((oc) => oc.doNothing()) - .executeTakeFirst(), - ]) - - return inserted || null -} - -const findDuplicate = async ( - db: DatabaseSchema, - uri: AtUri, - obj: Repost.Record, -): Promise => { - const found = await db - .selectFrom('repost') - .where('creator', '=', uri.host) - .where('subject', '=', obj.subject.uri) - .selectAll() - .executeTakeFirst() - return found ? new AtUri(found.uri) : null -} - -const notifsForInsert = (obj: IndexedRepost) => { - const subjectUri = new AtUri(obj.subject) - // prevent self-notifications - const isSelf = subjectUri.host === obj.creator - return isSelf - ? [] - : [ - { - userDid: subjectUri.host, - author: obj.creator, - recordUri: obj.uri, - recordCid: obj.cid, - reason: 'repost' as const, - reasonSubject: subjectUri.toString(), - indexedAt: obj.indexedAt, - }, - ] -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const uriStr = uri.toString() - const [deleted] = await Promise.all([ - db - .deleteFrom('repost') - .where('uri', '=', uriStr) - .returningAll() - .executeTakeFirst(), - db.deleteFrom('feed_item').where('uri', '=', uriStr).executeTakeFirst(), - ]) - return deleted || null -} - -const notifsForDelete = ( - deleted: IndexedRepost, - replacedBy: IndexedRepost | null, -) => { - const toDelete = replacedBy ? [] : [deleted.uri] - return { notifs: [], toDelete } -} - -const updateAggregates = async (db: DatabaseSchema, repost: IndexedRepost) => { - const repostCountQb = db - .insertInto('post_agg') - .values({ - uri: repost.subject, - repostCount: db - .selectFrom('repost') - .where('repost.subject', '=', repost.subject) - .select(countAll.as('count')), - }) - .onConflict((oc) => - oc - .column('uri') - .doUpdateSet({ repostCount: excluded(db, 'repostCount') }), - ) - await repostCountQb.execute() -} - -export type PluginType = RecordProcessor - -export const makePlugin = ( - db: Database, - backgroundQueue: BackgroundQueue, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - updateAggregates, - }) -} - -export default makePlugin diff --git a/packages/pds/src/app-view/services/indexing/processor.ts b/packages/pds/src/app-view/services/indexing/processor.ts deleted file mode 100644 index 6b6712889f2..00000000000 --- a/packages/pds/src/app-view/services/indexing/processor.ts +++ /dev/null @@ -1,259 +0,0 @@ -import { CID } from 'multiformats/cid' -import { AtUri } from '@atproto/syntax' -import { cborToLexRecord } from '@atproto/repo' -import Database from '../../../db' -import DatabaseSchema from '../../../db/database-schema' -import { BackgroundQueue } from '../../../event-stream/background-queue' -import { lexicons } from '../../../lexicon/lexicons' -import { UserNotification } from '../../../db/tables/user-notification' - -// @NOTE re: insertions and deletions. Due to how record updates are handled, -// (insertFn) should have the same effect as (insertFn -> deleteFn -> insertFn). -type RecordProcessorParams = { - lexId: string - insertFn: ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: T, - timestamp: string, - ) => Promise - findDuplicate: ( - db: DatabaseSchema, - uri: AtUri, - obj: T, - ) => Promise - deleteFn: (db: DatabaseSchema, uri: AtUri) => Promise - notifsForInsert: (obj: S) => UserNotification[] - notifsForDelete: ( - prev: S, - replacedBy: S | null, - ) => { notifs: UserNotification[]; toDelete: string[] } - updateAggregates?: (db: DatabaseSchema, obj: S) => Promise -} - -export class RecordProcessor { - collection: string - db: DatabaseSchema - constructor( - private appDb: Database, - private backgroundQueue: BackgroundQueue, - private params: RecordProcessorParams, - ) { - this.db = appDb.db - this.collection = this.params.lexId - } - - matchesSchema(obj: unknown): obj is T { - try { - this.assertValidRecord(obj) - return true - } catch { - return false - } - } - - assertValidRecord(obj: unknown): void { - lexicons.assertValidRecord(this.params.lexId, obj) - } - - async insertRecord(uri: AtUri, cid: CID, obj: unknown, timestamp: string) { - if (!this.matchesSchema(obj)) { - throw new Error(`Record does not match schema: ${this.params.lexId}`) - } - const inserted = await this.params.insertFn( - this.db, - uri, - cid, - obj, - timestamp, - ) - // if this was a new record, return events - if (inserted) { - this.aggregateOnCommit(inserted) - await this.handleNotifs({ inserted }) - return - } - // if duplicate, insert into duplicates table with no events - const found = await this.params.findDuplicate(this.db, uri, obj) - if (found && found.toString() !== uri.toString()) { - await this.db - .insertInto('duplicate_record') - .values({ - uri: uri.toString(), - cid: cid.toString(), - duplicateOf: found.toString(), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .execute() - } - } - - // Currently using a very simple strategy for updates: purge the existing index - // for the uri then replace it. The main upside is that this allows the indexer - // for each collection to avoid bespoke logic for in-place updates, which isn't - // straightforward in the general case. We still get nice control over notifications. - async updateRecord(uri: AtUri, cid: CID, obj: unknown, timestamp: string) { - if (!this.matchesSchema(obj)) { - throw new Error(`Record does not match schema: ${this.params.lexId}`) - } - - // If the updated record was a dupe, update dupe info for it - const dupe = await this.params.findDuplicate(this.db, uri, obj) - if (dupe) { - await this.db - .updateTable('duplicate_record') - .where('uri', '=', uri.toString()) - .set({ - cid: cid.toString(), - duplicateOf: dupe.toString(), - indexedAt: timestamp, - }) - .execute() - } else { - await this.db - .deleteFrom('duplicate_record') - .where('uri', '=', uri.toString()) - .execute() - } - - const deleted = await this.params.deleteFn(this.db, uri) - if (!deleted) { - // If a record was updated but hadn't been indexed yet, treat it like a plain insert. - return this.insertRecord(uri, cid, obj, timestamp) - } - this.aggregateOnCommit(deleted) - const inserted = await this.params.insertFn( - this.db, - uri, - cid, - obj, - timestamp, - ) - if (!inserted) { - throw new Error( - 'Record update failed: removed from index but could not be replaced', - ) - } - this.aggregateOnCommit(inserted) - await this.handleNotifs({ inserted, deleted }) - } - - async deleteRecord(uri: AtUri, cascading = false) { - await this.db - .deleteFrom('duplicate_record') - .where('uri', '=', uri.toString()) - .execute() - const deleted = await this.params.deleteFn(this.db, uri) - if (!deleted) return - this.aggregateOnCommit(deleted) - if (cascading) { - await this.db - .deleteFrom('duplicate_record') - .where('duplicateOf', '=', uri.toString()) - .execute() - await this.handleNotifs({ deleted }) - return - } else { - const found = await this.db - .selectFrom('duplicate_record') - // @TODO remove ipld_block dependency from app-view - .innerJoin('ipld_block', (join) => - join - .onRef('ipld_block.cid', '=', 'duplicate_record.cid') - .on('ipld_block.creator', '=', uri.host), - ) - .where('duplicateOf', '=', uri.toString()) - .orderBy('duplicate_record.indexedAt', 'asc') - .limit(1) - .selectAll() - .executeTakeFirst() - - if (!found) { - return this.handleNotifs({ deleted }) - } - const record = cborToLexRecord(found.content) - if (!this.matchesSchema(record)) { - return this.handleNotifs({ deleted }) - } - const inserted = await this.params.insertFn( - this.db, - new AtUri(found.uri), - CID.parse(found.cid), - record, - found.indexedAt, - ) - if (inserted) { - this.aggregateOnCommit(inserted) - } - await this.handleNotifs({ deleted, inserted: inserted ?? undefined }) - } - } - - async handleNotifs(op: { deleted?: S; inserted?: S }) { - let notifs: UserNotification[] = [] - const runOnCommit: ((db: Database) => Promise)[] = [] - if (op.deleted) { - const forDelete = this.params.notifsForDelete( - op.deleted, - op.inserted ?? null, - ) - if (forDelete.toDelete.length > 0) { - // Notifs can be deleted in background: they are expensive to delete and - // listNotifications already excludes notifs with missing records. - runOnCommit.push(async (db) => { - await db.db - .deleteFrom('user_notification') - .where('recordUri', 'in', forDelete.toDelete) - .execute() - }) - } - notifs = forDelete.notifs - } else if (op.inserted) { - notifs = this.params.notifsForInsert(op.inserted) - } - if (notifs.length > 0) { - runOnCommit.push(async (db) => { - await db.db.insertInto('user_notification').values(notifs).execute() - }) - } - if (runOnCommit.length) { - // Need to ensure notif deletion always happens before creation, otherwise delete may clobber in a race. - this.appDb.onCommit(() => { - this.backgroundQueue.add(async (db) => { - for (const fn of runOnCommit) { - await fn(db) - } - }) - }) - } - } - - aggregateOnCommit(indexed: S) { - const { updateAggregates } = this.params - if (!updateAggregates) return - this.appDb.onCommit(() => { - this.backgroundQueue.add((db) => updateAggregates(db.db, indexed)) - }) - } -} - -export default RecordProcessor - -export class NoopProcessor extends RecordProcessor { - constructor( - lexId: string, - appDb: Database, - backgroundQueue: BackgroundQueue, - ) { - super(appDb, backgroundQueue, { - lexId, - insertFn: async () => null, - deleteFn: async () => null, - findDuplicate: async () => null, - notifsForInsert: () => [], - notifsForDelete: () => ({ notifs: [], toDelete: [] }), - }) - } -} diff --git a/packages/pds/src/app-view/services/indexing/util.ts b/packages/pds/src/app-view/services/indexing/util.ts deleted file mode 100644 index 5ce78959f8f..00000000000 --- a/packages/pds/src/app-view/services/indexing/util.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { isValidISODateString } from 'iso-datestring-validator' - -// Normalize date strings to simplified ISO so that the lexical sort preserves temporal sort. -// Rather than failing on an invalid date format, returns valid unix epoch. -export function toSimplifiedISOSafe(dateStr: string) { - const date = new Date(dateStr) - if (isNaN(date.getTime())) { - return new Date(0).toISOString() - } - const iso = date.toISOString() - if (!isValidISODateString(iso)) { - // Occurs in rare cases, e.g. where resulting UTC year is negative. These also don't preserve lexical sort. - return new Date(0).toISOString() - } - return iso // YYYY-MM-DDTHH:mm:ss.sssZ -} diff --git a/packages/pds/src/app-view/services/label/index.ts b/packages/pds/src/app-view/services/label/index.ts deleted file mode 100644 index a3534666d5d..00000000000 --- a/packages/pds/src/app-view/services/label/index.ts +++ /dev/null @@ -1,171 +0,0 @@ -import { sql } from 'kysely' -import { AtUri } from '@atproto/syntax' -import Database from '../../../db' -import { - Label, - isSelfLabels, -} from '../../../lexicon/types/com/atproto/label/defs' -import { ids } from '../../../lexicon/lexicons' -import { LabelCache } from '../../../label-cache' -import { toSimplifiedISOSafe } from '../indexing/util' - -export type Labels = Record - -export class LabelService { - constructor(public db: Database, public cache: LabelCache) {} - - static creator(cache: LabelCache) { - return (db: Database) => new LabelService(db, cache) - } - - async formatAndCreate( - src: string, - uri: string, - cid: string | null, - labels: { create?: string[]; negate?: string[] }, - ) { - const { create = [], negate = [] } = labels - const toCreate = create.map((val) => ({ - src, - uri, - cid: cid ?? undefined, - val, - neg: false, - cts: new Date().toISOString(), - })) - const toNegate = negate.map((val) => ({ - src, - uri, - cid: cid ?? undefined, - val, - neg: true, - cts: new Date().toISOString(), - })) - await this.createLabels([...toCreate, ...toNegate]) - } - - async createLabels(labels: Label[]) { - if (labels.length < 1) return - const dbVals = labels.map((l) => ({ - ...l, - cid: l.cid ?? '', - neg: (l.neg ? 1 : 0) as 1 | 0, - })) - const { ref } = this.db.db.dynamic - const excluded = (col: string) => ref(`excluded.${col}`) - await this.db.db - .insertInto('label') - .values(dbVals) - .onConflict((oc) => - oc.columns(['src', 'uri', 'cid', 'val']).doUpdateSet({ - neg: sql`${excluded('neg')}`, - cts: sql`${excluded('cts')}`, - }), - ) - .execute() - } - - async getLabelsForUris( - subjects: string[], - opts?: { - includeNeg?: boolean - skipCache?: boolean - }, - ): Promise { - if (subjects.length < 1) return {} - const res = opts?.skipCache - ? await this.db.db - .selectFrom('label') - .where('label.uri', 'in', subjects) - .if(!opts?.includeNeg, (qb) => qb.where('neg', '=', 0)) - .selectAll() - .execute() - : this.cache.forSubjects(subjects, opts?.includeNeg) - return res.reduce((acc, cur) => { - acc[cur.uri] ??= [] - acc[cur.uri].push({ - ...cur, - cid: cur.cid === '' ? undefined : cur.cid, - neg: cur.neg === 1, // @TODO update in appview - }) - return acc - }, {} as Labels) - } - - // gets labels for any record. when did is present, combine labels for both did & profile record. - async getLabelsForSubjects( - subjects: string[], - opts?: { - includeNeg?: boolean - skipCache?: boolean - }, - ): Promise { - if (subjects.length < 1) return {} - const expandedSubjects = subjects.flatMap((subject) => { - if (subject.startsWith('did:')) { - return [ - subject, - AtUri.make(subject, ids.AppBskyActorProfile, 'self').toString(), - ] - } - return subject - }) - const labels = await this.getLabelsForUris(expandedSubjects, opts) - return Object.keys(labels).reduce((acc, cur) => { - const uri = cur.startsWith('at://') ? new AtUri(cur) : null - if ( - uri && - uri.collection === ids.AppBskyActorProfile && - uri.rkey === 'self' - ) { - // combine labels for profile + did - const did = uri.hostname - acc[did] ??= [] - acc[did].push(...labels[cur]) - } - acc[cur] ??= [] - acc[cur].push(...labels[cur]) - return acc - }, {} as Labels) - } - - async getLabels( - subject: string, - opts?: { - includeNeg?: boolean - skipCache?: boolean - }, - ): Promise { - const labels = await this.getLabelsForUris([subject], opts) - return labels[subject] ?? [] - } - - async getLabelsForProfile( - did: string, - opts?: { - includeNeg?: boolean - skipCache?: boolean - }, - ): Promise { - const labels = await this.getLabelsForSubjects([did], opts) - return labels[did] ?? [] - } -} - -export function getSelfLabels(details: { - uri: string | null - cid: string | null - record: Record | null -}): Label[] { - const { uri, cid, record } = details - if (!uri || !cid || !record) return [] - if (!isSelfLabels(record.labels)) return [] - const src = new AtUri(uri).host // record creator - const cts = - typeof record.createdAt === 'string' - ? toSimplifiedISOSafe(record.createdAt) - : new Date(0).toISOString() - return record.labels.values.map(({ val }) => { - return { src, uri, cid, val, cts, neg: false } - }) -} diff --git a/packages/pds/src/event-stream/background-queue.ts b/packages/pds/src/background.ts similarity index 92% rename from packages/pds/src/event-stream/background-queue.ts rename to packages/pds/src/background.ts index aa7671ccc07..65d8cbd473b 100644 --- a/packages/pds/src/event-stream/background-queue.ts +++ b/packages/pds/src/background.ts @@ -1,6 +1,6 @@ import PQueue from 'p-queue' -import Database from '../db' -import { dbLogger } from '../logger' +import Database from './db' +import { dbLogger } from './logger' // A simple queue for in-process, out-of-band/backgrounded work diff --git a/packages/pds/src/config.ts b/packages/pds/src/config.ts index 816aa511873..f8d7b04d8fa 100644 --- a/packages/pds/src/config.ts +++ b/packages/pds/src/config.ts @@ -50,12 +50,6 @@ export interface ServerConfigValues { moderationEmailAddress?: string moderationEmailSmtpUrl?: string - hiveApiKey?: string - labelerDid: string - labelerKeywords: Record - - feedGenDid?: string - maxSubscriptionBuffer: number repoBackfillLimitMs: number sequencerLeaderLockId?: number @@ -179,12 +173,6 @@ export class ServerConfig { const moderationEmailSmtpUrl = process.env.MODERATION_EMAIL_SMTP_URL || undefined - const hiveApiKey = process.env.HIVE_API_KEY || undefined - const labelerDid = process.env.LABELER_DID || 'did:example:labeler' - const labelerKeywords = {} - - const feedGenDid = process.env.FEED_GEN_DID - const dbPostgresUrl = process.env.DB_POSTGRES_URL const dbPostgresSchema = process.env.DB_POSTGRES_SCHEMA @@ -271,10 +259,6 @@ export class ServerConfig { emailNoReplyAddress, moderationEmailAddress, moderationEmailSmtpUrl, - hiveApiKey, - labelerDid, - labelerKeywords, - feedGenDid, maxSubscriptionBuffer, repoBackfillLimitMs, sequencerLeaderLockId, @@ -467,22 +451,6 @@ export class ServerConfig { return this.cfg.moderationEmailSmtpUrl } - get hiveApiKey() { - return this.cfg.hiveApiKey - } - - get labelerDid() { - return this.cfg.labelerDid - } - - get labelerKeywords() { - return this.cfg.labelerKeywords - } - - get feedGenDid() { - return this.cfg.feedGenDid - } - get maxSubscriptionBuffer() { return this.cfg.maxSubscriptionBuffer } diff --git a/packages/pds/src/context.ts b/packages/pds/src/context.ts index ea0990955e2..791100c492b 100644 --- a/packages/pds/src/context.ts +++ b/packages/pds/src/context.ts @@ -11,13 +11,10 @@ import { ServerMailer } from './mailer' import { ModerationMailer } from './mailer/moderation' import { BlobStore } from '@atproto/repo' import { Services } from './services' -import { MessageDispatcher } from './event-stream/message-queue' import { Sequencer, SequencerLeader } from './sequencer' -import { Labeler } from './labeler' -import { BackgroundQueue } from './event-stream/background-queue' +import { BackgroundQueue } from './background' import DidSqlCache from './did-cache' import { Crawlers } from './crawlers' -import { LabelCache } from './label-cache' import { RuntimeFlags } from './runtime-flags' export class AppContext { @@ -35,11 +32,8 @@ export class AppContext { mailer: ServerMailer moderationMailer: ModerationMailer services: Services - messageDispatcher: MessageDispatcher sequencer: Sequencer sequencerLeader: SequencerLeader | null - labeler: Labeler - labelCache: LabelCache runtimeFlags: RuntimeFlags backgroundQueue: BackgroundQueue appviewAgent: AtpAgent @@ -115,10 +109,6 @@ export class AppContext { return this.opts.services } - get messageDispatcher(): MessageDispatcher { - return this.opts.messageDispatcher - } - get sequencer(): Sequencer { return this.opts.sequencer } @@ -127,14 +117,6 @@ export class AppContext { return this.opts.sequencerLeader } - get labeler(): Labeler { - return this.opts.labeler - } - - get labelCache(): LabelCache { - return this.opts.labelCache - } - get runtimeFlags(): RuntimeFlags { return this.opts.runtimeFlags } diff --git a/packages/pds/src/db/database-schema.ts b/packages/pds/src/db/database-schema.ts index f77df6b21ad..eda67a4e6fb 100644 --- a/packages/pds/src/db/database-schema.ts +++ b/packages/pds/src/db/database-schema.ts @@ -18,14 +18,11 @@ import * as deleteAccountToken from './tables/delete-account-token' import * as moderation from './tables/moderation' import * as mute from './tables/mute' import * as listMute from './tables/list-mute' -import * as label from './tables/label' import * as repoSeq from './tables/repo-seq' import * as appMigration from './tables/app-migration' import * as runtimeFlag from './tables/runtime-flag' -import * as appView from '../app-view/db' -export type DatabaseSchemaType = appView.DatabaseSchemaType & - runtimeFlag.PartialDB & +export type DatabaseSchemaType = runtimeFlag.PartialDB & appMigration.PartialDB & userAccount.PartialDB & userState.PartialDB & @@ -46,7 +43,6 @@ export type DatabaseSchemaType = appView.DatabaseSchemaType & moderation.PartialDB & mute.PartialDB & listMute.PartialDB & - label.PartialDB & repoSeq.PartialDB export type DatabaseSchema = Kysely diff --git a/packages/pds/src/db/migrations/20230922T033938477Z-remove-appview.ts b/packages/pds/src/db/migrations/20230922T033938477Z-remove-appview.ts new file mode 100644 index 00000000000..f66825fa722 --- /dev/null +++ b/packages/pds/src/db/migrations/20230922T033938477Z-remove-appview.ts @@ -0,0 +1,29 @@ +import { Kysely } from 'kysely' + +export async function up(db: Kysely): Promise { + await db.schema.dropView('algo_whats_hot_view').materialized().execute() + await db.schema.dropTable('actor_block').execute() + await db.schema.dropTable('duplicate_record').execute() + await db.schema.dropTable('feed_generator').execute() + await db.schema.dropTable('feed_item').execute() + await db.schema.dropTable('follow').execute() + await db.schema.dropTable('label').execute() + await db.schema.dropTable('like').execute() + await db.schema.dropTable('list_item').execute() + await db.schema.dropTable('list').execute() + await db.schema.dropTable('post_agg').execute() + await db.schema.dropTable('post_embed_image').execute() + await db.schema.dropTable('post_embed_external').execute() + await db.schema.dropTable('post_embed_record').execute() + await db.schema.dropTable('post').execute() + await db.schema.dropTable('profile_agg').execute() + await db.schema.dropTable('profile').execute() + await db.schema.dropTable('repost').execute() + await db.schema.dropTable('subscription').execute() + await db.schema.dropTable('suggested_follow').execute() + await db.schema.dropTable('view_param').execute() +} + +export async function down(_db: Kysely): Promise { + // Migration code +} diff --git a/packages/pds/src/db/migrations/index.ts b/packages/pds/src/db/migrations/index.ts index e7e521e986a..fde3ddd2398 100644 --- a/packages/pds/src/db/migrations/index.ts +++ b/packages/pds/src/db/migrations/index.ts @@ -65,3 +65,4 @@ export * as _20230818T134357818Z from './20230818T134357818Z-runtime-flags' export * as _20230824T182048120Z from './20230824T182048120Z-remove-post-hierarchy' export * as _20230825T142507884Z from './20230825T142507884Z-blob-tempkey-idx' export * as _20230828T153013575Z from './20230828T153013575Z-repo-history-rewrite' +export * as _20230922T033938477Z from './20230922T033938477Z-remove-appview' diff --git a/packages/pds/src/db/periodic-moderation-action-reversal.ts b/packages/pds/src/db/periodic-moderation-action-reversal.ts index 72730e61565..b3b631de71d 100644 --- a/packages/pds/src/db/periodic-moderation-action-reversal.ts +++ b/packages/pds/src/db/periodic-moderation-action-reversal.ts @@ -4,7 +4,6 @@ import { Leader } from './leader' import { dbLogger } from '../logger' import AppContext from '../context' import { ModerationActionRow } from '../services/moderation' -import { LabelService } from '../app-view/services/label' export const MODERATION_ACTION_REVERSAL_ID = 1011 @@ -14,39 +13,15 @@ export class PeriodicModerationActionReversal { constructor(private appContext: AppContext) {} - // invert label creation & negations - async reverseLabels(labelTxn: LabelService, actionRow: ModerationActionRow) { - let uri: string - let cid: string | null = null - - if (actionRow.subjectUri && actionRow.subjectCid) { - uri = actionRow.subjectUri - cid = actionRow.subjectCid - } else { - uri = actionRow.subjectDid - } - - await labelTxn.formatAndCreate(this.appContext.cfg.labelerDid, uri, cid, { - create: actionRow.negateLabelVals - ? actionRow.negateLabelVals.split(' ') - : undefined, - negate: actionRow.createLabelVals - ? actionRow.createLabelVals.split(' ') - : undefined, - }) - } - async revertAction(actionRow: ModerationActionRow) { return this.appContext.db.transaction(async (dbTxn) => { const moderationTxn = this.appContext.services.moderation(dbTxn) - const labelTxn = this.appContext.services.appView.label(dbTxn) await moderationTxn.revertAction({ id: actionRow.id, createdBy: actionRow.createdBy, createdAt: new Date(), reason: `[SCHEDULED_REVERSAL] Reverting action as originally scheduled`, }) - await this.reverseLabels(labelTxn, actionRow) }) } diff --git a/packages/pds/src/db/tables/label.ts b/packages/pds/src/db/tables/label.ts deleted file mode 100644 index 1837faab1c8..00000000000 --- a/packages/pds/src/db/tables/label.ts +++ /dev/null @@ -1,12 +0,0 @@ -export const tableName = 'label' - -export interface Label { - src: string - uri: string - cid: string - val: string - neg: 0 | 1 // @TODO convert to boolean in app-view - cts: string -} - -export type PartialDB = { [tableName]: Label } diff --git a/packages/pds/src/event-stream/message-queue.ts b/packages/pds/src/event-stream/message-queue.ts deleted file mode 100644 index 20aa5d5b862..00000000000 --- a/packages/pds/src/event-stream/message-queue.ts +++ /dev/null @@ -1,48 +0,0 @@ -import Database from '../db' -import { dbLogger as log } from '../logger' -import { MessageQueue, Listenable, Listener, MessageOfType } from './types' - -// @NOTE A message dispatcher for loose coupling within db transactions. -// Messages are handled immediately. This should not be around for long. -export class MessageDispatcher implements MessageQueue { - private destroyed = false - private listeners: Map = new Map() - - async send( - tx: Database, - message: MessageOfType | MessageOfType[], - ): Promise { - if (this.destroyed) return - const messages = Array.isArray(message) ? message : [message] - for (const msg of messages) { - await this.handleMessage(tx, msg) - } - } - - listen>( - topic: T, - listenable: Listenable, - ) { - const listeners = this.listeners.get(topic) ?? [] - listeners.push(listenable.listener as Listener) // @TODO avoid upcast - this.listeners.set(topic, listeners) - } - - destroy(): void { - this.destroyed = true - } - - private async handleMessage(db: Database, message: MessageOfType) { - const listeners = this.listeners.get(message.type) - if (!listeners?.length) { - return log.error({ message }, `no listeners for event: ${message.type}`) - } - for (const listener of listeners) { - await listener({ message, db }) - } - } - - // Unused by MessageDispatcher - async processNext(): Promise {} - async processAll(): Promise {} -} diff --git a/packages/pds/src/event-stream/messages.ts b/packages/pds/src/event-stream/messages.ts deleted file mode 100644 index 9c5f00baca6..00000000000 --- a/packages/pds/src/event-stream/messages.ts +++ /dev/null @@ -1,51 +0,0 @@ -// Below specific to message dispatcher - -import { CID } from 'multiformats/cid' -import { AtUri } from '@atproto/syntax' -import { WriteOpAction } from '@atproto/repo' - -export type IndexRecord = { - type: 'index_record' - action: WriteOpAction.Create | WriteOpAction.Update - uri: AtUri - cid: CID - obj: unknown - timestamp: string -} - -export type DeleteRecord = { - type: 'delete_record' - uri: AtUri - cascading: boolean -} - -export type DeleteRepo = { - type: 'delete_repo' - did: string -} - -export const indexRecord = ( - uri: AtUri, - cid: CID, - obj: unknown, - action: WriteOpAction.Create | WriteOpAction.Update, - timestamp: string, -): IndexRecord => ({ - type: 'index_record', - uri, - cid, - obj, - action, - timestamp, -}) - -export const deleteRecord = (uri: AtUri, cascading: boolean): DeleteRecord => ({ - type: 'delete_record', - uri, - cascading, -}) - -export const deleteRepo = (did: string): DeleteRepo => ({ - type: 'delete_repo', - did, -}) diff --git a/packages/pds/src/event-stream/types.ts b/packages/pds/src/event-stream/types.ts deleted file mode 100644 index aad8fc5f01b..00000000000 --- a/packages/pds/src/event-stream/types.ts +++ /dev/null @@ -1,38 +0,0 @@ -import Database from '../db' - -export type MessageOfType = { - type: T - [s: string]: unknown -} - -export type Listener = (ctx: { - message: M - db: Database -}) => Promise - -export interface Listenable { - listener: Listener -} - -export abstract class Consumer - implements Listenable -{ - abstract dispatch(ctx: { - db: Database - message: M - }): Promise - get listener() { - return this.dispatch.bind(this) - } -} - -export interface MessageQueue { - send(tx: Database, message: MessageOfType | MessageOfType[]): Promise - listen>( - topic: T, - listenable: Listenable, - ): void - processNext(): Promise - processAll(): Promise - destroy(): void -} diff --git a/packages/pds/src/index.ts b/packages/pds/src/index.ts index 36f15adae8e..fb08ef375e9 100644 --- a/packages/pds/src/index.ts +++ b/packages/pds/src/index.ts @@ -21,7 +21,6 @@ import { Options as XrpcServerOptions, } from '@atproto/xrpc-server' import { DAY, HOUR, MINUTE } from '@atproto/common' -import * as appviewConsumers from './app-view/event-stream/consumers' import inProcessAppView from './app-view/api' import API from './api' import * as basicRoutes from './basic-routes' @@ -35,16 +34,13 @@ import { ServerConfig } from './config' import { ServerMailer } from './mailer' import { ModerationMailer } from './mailer/moderation' import { createServer } from './lexicon' -import { MessageDispatcher } from './event-stream/message-queue' import { createServices } from './services' import { createHttpTerminator, HttpTerminator } from 'http-terminator' import AppContext from './context' import { Sequencer, SequencerLeader } from './sequencer' -import { Labeler, HiveLabeler, KeywordLabeler } from './labeler' -import { BackgroundQueue } from './event-stream/background-queue' +import { BackgroundQueue } from './background' import DidSqlCache from './did-cache' import { Crawlers } from './crawlers' -import { LabelCache } from './label-cache' import { getRedisClient } from './redis' import { RuntimeFlags } from './runtime-flags' @@ -95,7 +91,6 @@ export class PDS { backupNameservers: config.handleResolveNameservers, }) - const messageDispatcher = new MessageDispatcher() const sequencer = new Sequencer(db) const sequencerLeader = config.sequencerLeaderEnabled ? new SequencerLeader(db, config.sequencerLeaderLockId) @@ -129,35 +124,11 @@ export class PDS { config.crawlersToNotify ?? [], ) - let labeler: Labeler - if (config.hiveApiKey) { - labeler = new HiveLabeler({ - db, - blobstore, - backgroundQueue, - labelerDid: config.labelerDid, - hiveApiKey: config.hiveApiKey, - keywords: config.labelerKeywords, - }) - } else { - labeler = new KeywordLabeler({ - db, - blobstore, - backgroundQueue, - labelerDid: config.labelerDid, - keywords: config.labelerKeywords, - }) - } - - const labelCache = new LabelCache(db) const appviewAgent = new AtpAgent({ service: config.bskyAppViewEndpoint }) const services = createServices({ repoSigningKey, - messageDispatcher, blobstore, - labeler, - labelCache, appviewAgent, appviewDid: config.bskyAppViewDid, appviewCdnUrlPattern: config.bskyAppViewCdnUrlPattern, @@ -185,11 +156,8 @@ export class PDS { didCache, cfg: config, auth, - messageDispatcher, sequencer, sequencerLeader, - labeler, - labelCache, runtimeFlags, services, mailer, @@ -293,11 +261,9 @@ export class PDS { } } }, 500) - appviewConsumers.listen(this.ctx) this.ctx.sequencerLeader?.run() await this.ctx.sequencer.start() await this.ctx.db.startListeningToChannels() - this.ctx.labelCache.start() await this.ctx.runtimeFlags.start() const server = this.app.listen(this.ctx.cfg.port) this.server = server @@ -309,7 +275,6 @@ export class PDS { async destroy(): Promise { await this.ctx.runtimeFlags.destroy() - this.ctx.labelCache.stop() await this.ctx.sequencerLeader?.destroy() await this.terminator?.terminate() await this.ctx.backgroundQueue.destroy() diff --git a/packages/pds/src/label-cache.ts b/packages/pds/src/label-cache.ts deleted file mode 100644 index e4f23daa599..00000000000 --- a/packages/pds/src/label-cache.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { wait } from '@atproto/common' -import Database from './db' -import { Label } from './db/tables/label' -import { labelerLogger as log } from './logger' - -export class LabelCache { - bySubject: Record = {} - latestLabel = '' - refreshes = 0 - - destroyed = false - - constructor(public db: Database) {} - - start() { - this.poll() - } - - async fullRefresh() { - const allLabels = await this.db.db.selectFrom('label').selectAll().execute() - this.wipeCache() - this.processLabels(allLabels) - } - - async partialRefresh() { - const labels = await this.db.db - .selectFrom('label') - .selectAll() - .where('cts', '>', this.latestLabel) - .execute() - this.processLabels(labels) - } - - async poll() { - try { - if (this.destroyed) return - if (this.refreshes >= 120) { - await this.fullRefresh() - this.refreshes = 0 - } else { - await this.partialRefresh() - this.refreshes++ - } - } catch (err) { - log.error( - { err, latestLabel: this.latestLabel, refreshes: this.refreshes }, - 'label cache failed to refresh', - ) - } - await wait(500) - this.poll() - } - - processLabels(labels: Label[]) { - for (const label of labels) { - if (label.cts > this.latestLabel) { - this.latestLabel = label.cts - } - this.bySubject[label.uri] ??= [] - this.bySubject[label.uri].push(label) - } - } - - wipeCache() { - this.bySubject = {} - } - - stop() { - this.destroyed = true - } - - forSubject(subject: string, includeNeg = false): Label[] { - const labels = this.bySubject[subject] ?? [] - return includeNeg ? labels : labels.filter((l) => l.neg === 0) - } - - forSubjects(subjects: string[], includeNeg?: boolean): Label[] { - let labels: Label[] = [] - const alreadyAdded = new Set() - for (const subject of subjects) { - if (alreadyAdded.has(subject)) { - continue - } - const subLabels = this.forSubject(subject, includeNeg) - labels = [...labels, ...subLabels] - alreadyAdded.add(subject) - } - return labels - } -} diff --git a/packages/pds/src/labeler/base.ts b/packages/pds/src/labeler/base.ts deleted file mode 100644 index 8a52ed2c717..00000000000 --- a/packages/pds/src/labeler/base.ts +++ /dev/null @@ -1,72 +0,0 @@ -import Database from '../db' -import { BlobStore, cidForRecord } from '@atproto/repo' -import { dedupe, getFieldsFromRecord } from './util' -import { AtUri } from '@atproto/syntax' -import { labelerLogger as log } from '../logger' -import { BackgroundQueue } from '../event-stream/background-queue' -import { CID } from 'multiformats/cid' - -export abstract class Labeler { - public db: Database - public blobstore: BlobStore - public labelerDid: string - public backgroundQueue: BackgroundQueue - constructor(opts: { - db: Database - blobstore: BlobStore - labelerDid: string - backgroundQueue: BackgroundQueue - }) { - this.db = opts.db - this.blobstore = opts.blobstore - this.labelerDid = opts.labelerDid - this.backgroundQueue = opts.backgroundQueue - } - - processRecord(uri: AtUri, obj: unknown) { - this.backgroundQueue.add(() => - this.createAndStoreLabels(uri, obj).catch((err) => { - log.error( - { err, uri: uri.toString(), record: obj }, - 'failed to label record', - ) - }), - ) - } - - async createAndStoreLabels(uri: AtUri, obj: unknown): Promise { - const labels = await this.labelRecord(obj) - if (labels.length < 1) return - const cid = await cidForRecord(obj) - const rows = labels.map((val) => ({ - src: this.labelerDid, - uri: uri.toString(), - cid: cid.toString(), - val, - neg: 0 as const, - cts: new Date().toISOString(), - })) - - await this.db.db - .insertInto('label') - .values(rows) - .onConflict((oc) => oc.doNothing()) - .execute() - } - - async labelRecord(obj: unknown): Promise { - const { text, imgs } = getFieldsFromRecord(obj) - const txtLabels = await this.labelText(text.join(' ')) - const imgLabels = await Promise.all( - imgs.map(async (cid) => this.labelImg(cid)), - ) - return dedupe([...txtLabels, ...imgLabels.flat()]) - } - - abstract labelText(text: string): Promise - abstract labelImg(cid: CID): Promise - - async processAll() { - await this.backgroundQueue.processAll() - } -} diff --git a/packages/pds/src/labeler/hive.ts b/packages/pds/src/labeler/hive.ts deleted file mode 100644 index e4a0bc48045..00000000000 --- a/packages/pds/src/labeler/hive.ts +++ /dev/null @@ -1,191 +0,0 @@ -import stream from 'stream' -import axios from 'axios' -import FormData from 'form-data' -import { Labeler } from './base' -import Database from '../db' -import { BlobStore } from '@atproto/repo' -import { keywordLabeling } from './util' -import { BackgroundQueue } from '../event-stream/background-queue' -import { CID } from 'multiformats/cid' - -const HIVE_ENDPOINT = 'https://api.thehive.ai/api/v2/task/sync' - -export class HiveLabeler extends Labeler { - hiveApiKey: string - keywords: Record - - constructor(opts: { - db: Database - blobstore: BlobStore - backgroundQueue: BackgroundQueue - labelerDid: string - hiveApiKey: string - keywords: Record - }) { - const { db, blobstore, backgroundQueue, labelerDid, hiveApiKey, keywords } = - opts - super({ db, blobstore, backgroundQueue, labelerDid }) - this.hiveApiKey = hiveApiKey - this.keywords = keywords - } - - async labelText(text: string): Promise { - return keywordLabeling(this.keywords, text) - } - - async labelImg(cid: CID): Promise { - const stream = await this.blobstore.getStream(cid) - return labelBlob(stream, this.hiveApiKey) - } -} - -export const labelBlob = async ( - blob: stream.Readable, - hiveApiKey: string, -): Promise => { - const classes = await makeHiveReq(blob, hiveApiKey) - return summarizeLabels(classes) -} - -export const makeHiveReq = async ( - blob: stream.Readable, - hiveApiKey: string, -): Promise => { - const form = new FormData() - form.append('media', blob) - const res = await axios.post(HIVE_ENDPOINT, form, { - headers: { - 'Content-Type': 'multipart/form-data', - authorization: `token ${hiveApiKey}`, - accept: 'application/json', - }, - }) - return respToClasses(res.data) -} - -export const respToClasses = (res: HiveResp): HiveRespClass[] => { - const classes: HiveRespClass[] = [] - for (const status of res.status) { - for (const out of status.response.output) { - for (const cls of out.classes) { - classes.push(cls) - } - } - } - return classes -} - -// Matches only one (or none) of: porn, sexual, nudity -// -// porn: sexual and nudity. including both explicit activity or full-frontal and suggestive/intent -// sexual: sexually suggestive, not explicit; may include some forms of nudity -// nudity: non-sexual nudity (eg, artistic, possibly some photographic) -// -// hive docs/definitions: https://docs.thehive.ai/docs/sexual-content -export const sexualLabels = (classes: HiveRespClass[]): string[] => { - const scores = {} - - for (const cls of classes) { - scores[cls.class] = cls.score - } - - // first check if porn... - for (const pornClass of [ - 'yes_sexual_activity', - 'animal_genitalia_and_human', - 'yes_realistic_nsfw', - ]) { - if (scores[pornClass] >= 0.9) { - return ['porn'] - } - } - if (scores['general_nsfw'] >= 0.9) { - // special case for some anime examples - if (scores['animated_animal_genitalia'] >= 0.5) { - return ['porn'] - } - // special case for some pornographic/explicit classic drawings - if (scores['yes_undressed'] >= 0.9 && scores['yes_sexual_activity'] > 0.9) { - return ['porn'] - } - } - - // then check for sexual suggestive (which may include nudity)... - for (const sexualClass of ['yes_sexual_intent', 'yes_sex_toy']) { - if (scores[sexualClass] >= 0.9) { - return ['sexual'] - } - } - if (scores['yes_undressed'] >= 0.9) { - // special case for bondage examples - if (scores['yes_sex_toy'] > 0.75) { - return ['sexual'] - } - } - - // then non-sexual nudity... - for (const nudityClass of [ - 'yes_male_nudity', - 'yes_female_nudity', - 'yes_undressed', - ]) { - if (scores[nudityClass] >= 0.9) { - return ['nudity'] - } - } - - // then finally flag remaining "underwear" images in to sexually suggestive - // (after non-sexual content already labeled above) - for (const nudityClass of ['yes_male_underwear', 'yes_female_underwear']) { - if (scores[nudityClass] >= 0.9) { - return ['sexual'] - } - } - - return [] -} - -// gore and violence: https://docs.thehive.ai/docs/class-descriptions-violence-gore -const labelForClass = { - very_bloody: 'gore', - human_corpse: 'corpse', - hanging: 'corpse', -} -const labelForClassLessSensitive = { - yes_self_harm: 'self-harm', -} - -export const summarizeLabels = (classes: HiveRespClass[]): string[] => { - const labels: string[] = sexualLabels(classes) - for (const cls of classes) { - if (labelForClass[cls.class] && cls.score >= 0.9) { - labels.push(labelForClass[cls.class]) - } - } - for (const cls of classes) { - if (labelForClassLessSensitive[cls.class] && cls.score >= 0.96) { - labels.push(labelForClassLessSensitive[cls.class]) - } - } - return labels -} - -type HiveResp = { - status: HiveRespStatus[] -} - -type HiveRespStatus = { - response: { - output: HiveRespOutput[] - } -} - -type HiveRespOutput = { - time: number - classes: HiveRespClass[] -} - -type HiveRespClass = { - class: string - score: number -} diff --git a/packages/pds/src/labeler/index.ts b/packages/pds/src/labeler/index.ts deleted file mode 100644 index cd6d2a64345..00000000000 --- a/packages/pds/src/labeler/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './base' -export * from './hive' -export * from './keyword' diff --git a/packages/pds/src/labeler/keyword.ts b/packages/pds/src/labeler/keyword.ts deleted file mode 100644 index 8838e87486a..00000000000 --- a/packages/pds/src/labeler/keyword.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { BlobStore } from '@atproto/repo' -import Database from '../db' -import { Labeler } from './base' -import { keywordLabeling } from './util' -import { BackgroundQueue } from '../event-stream/background-queue' - -export class KeywordLabeler extends Labeler { - keywords: Record - - constructor(opts: { - db: Database - blobstore: BlobStore - backgroundQueue: BackgroundQueue - labelerDid: string - keywords: Record - }) { - const { db, blobstore, backgroundQueue, labelerDid, keywords } = opts - super({ db, blobstore, backgroundQueue, labelerDid }) - this.keywords = keywords - } - - async labelText(text: string): Promise { - return keywordLabeling(this.keywords, text) - } - - async labelImg(): Promise { - return [] - } -} diff --git a/packages/pds/src/labeler/util.ts b/packages/pds/src/labeler/util.ts deleted file mode 100644 index 4175886a542..00000000000 --- a/packages/pds/src/labeler/util.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { CID } from 'multiformats/cid' -import * as lex from '../lexicon/lexicons' -import { Record as PostRecord } from '../lexicon/types/app/bsky/feed/post' -import { Record as ProfileRecord } from '../lexicon/types/app/bsky/actor/profile' -import { isMain as isEmbedImage } from '../lexicon/types/app/bsky/embed/images' -import { isMain as isEmbedExternal } from '../lexicon/types/app/bsky/embed/external' -import { isMain as isEmbedRecordWithMedia } from '../lexicon/types/app/bsky/embed/recordWithMedia' - -type RecordFields = { - text: string[] - imgs: CID[] -} - -export const getFieldsFromRecord = (record: unknown): RecordFields => { - if (isPost(record)) { - return getFieldsFromPost(record) - // @TODO add back in profile labeling - // } else if (isProfile(record)) { - // return getFieldsFromProfile(record) - } else { - return { text: [], imgs: [] } - } -} - -export const getFieldsFromPost = (record: PostRecord): RecordFields => { - const text: string[] = [] - const imgs: CID[] = [] - text.push(record.text) - const embeds = separateEmbeds(record.embed) - for (const embed of embeds) { - if (isEmbedImage(embed)) { - for (const img of embed.images) { - imgs.push(img.image.ref) - text.push(img.alt) - } - } else if (isEmbedExternal(embed)) { - if (embed.external.thumb) { - imgs.push(embed.external.thumb.ref) - } - text.push(embed.external.title) - text.push(embed.external.description) - } - } - return { text, imgs } -} - -export const getFieldsFromProfile = (record: ProfileRecord): RecordFields => { - const text: string[] = [] - const imgs: CID[] = [] - if (record.displayName) { - text.push(record.displayName) - } - if (record.description) { - text.push(record.description) - } - if (record.avatar) { - imgs.push(record.avatar.ref) - } - if (record.banner) { - imgs.push(record.banner.ref) - } - return { text, imgs } -} - -export const dedupe = (str: string[]): string[] => { - const set = new Set(str) - return [...set] -} - -export const isPost = (obj: unknown): obj is PostRecord => { - return isRecordType(obj, 'app.bsky.feed.post') -} - -export const isProfile = (obj: unknown): obj is ProfileRecord => { - return isRecordType(obj, 'app.bsky.actor.profile') -} - -export const isRecordType = (obj: unknown, lexId: string): boolean => { - try { - lex.lexicons.assertValidRecord(lexId, obj) - return true - } catch { - return false - } -} - -export const keywordLabeling = ( - keywords: Record, - text: string, -): string[] => { - const lowerText = text.toLowerCase() - const labels: string[] = [] - for (const word of Object.keys(keywords)) { - if (lowerText.includes(word)) { - labels.push(keywords[word]) - } - } - return labels -} - -const separateEmbeds = (embed: PostRecord['embed']) => { - if (!embed) { - return [] - } - if (isEmbedRecordWithMedia(embed)) { - return [{ $type: lex.ids.AppBskyEmbedRecord, ...embed.record }, embed.media] - } - return [embed] -} diff --git a/packages/pds/src/lexicon/index.ts b/packages/pds/src/lexicon/index.ts index 3ec69a3503f..3dd2b5104cc 100644 --- a/packages/pds/src/lexicon/index.ts +++ b/packages/pds/src/lexicon/index.ts @@ -107,7 +107,6 @@ import * as AppBskyNotificationGetUnreadCount from './types/app/bsky/notificatio import * as AppBskyNotificationListNotifications from './types/app/bsky/notification/listNotifications' import * as AppBskyNotificationRegisterPush from './types/app/bsky/notification/registerPush' import * as AppBskyNotificationUpdateSeen from './types/app/bsky/notification/updateSeen' -import * as AppBskyUnspeccedApplyLabels from './types/app/bsky/unspecced/applyLabels' import * as AppBskyUnspeccedGetPopular from './types/app/bsky/unspecced/getPopular' import * as AppBskyUnspeccedGetPopularFeedGenerators from './types/app/bsky/unspecced/getPopularFeedGenerators' import * as AppBskyUnspeccedGetTimelineSkeleton from './types/app/bsky/unspecced/getTimelineSkeleton' @@ -1402,17 +1401,6 @@ export class UnspeccedNS { this._server = server } - applyLabels( - cfg: ConfigOf< - AV, - AppBskyUnspeccedApplyLabels.Handler>, - AppBskyUnspeccedApplyLabels.HandlerReqCtx> - >, - ) { - const nsid = 'app.bsky.unspecced.applyLabels' // @ts-ignore - return this._server.xrpc.method(nsid, cfg) - } - getPopular( cfg: ConfigOf< AV, diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index b624ce335ed..177b63808f4 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -6905,32 +6905,6 @@ export const schemaDict = { }, }, }, - AppBskyUnspeccedApplyLabels: { - lexicon: 1, - id: 'app.bsky.unspecced.applyLabels', - defs: { - main: { - type: 'procedure', - description: 'Allow a labeler to apply labels directly.', - input: { - encoding: 'application/json', - schema: { - type: 'object', - required: ['labels'], - properties: { - labels: { - type: 'array', - items: { - type: 'ref', - ref: 'lex:com.atproto.label.defs#label', - }, - }, - }, - }, - }, - }, - }, - }, AppBskyUnspeccedDefs: { lexicon: 1, id: 'app.bsky.unspecced.defs', @@ -7359,7 +7333,6 @@ export const ids = { AppBskyNotificationRegisterPush: 'app.bsky.notification.registerPush', AppBskyNotificationUpdateSeen: 'app.bsky.notification.updateSeen', AppBskyRichtextFacet: 'app.bsky.richtext.facet', - AppBskyUnspeccedApplyLabels: 'app.bsky.unspecced.applyLabels', AppBskyUnspeccedDefs: 'app.bsky.unspecced.defs', AppBskyUnspeccedGetPopular: 'app.bsky.unspecced.getPopular', AppBskyUnspeccedGetPopularFeedGenerators: diff --git a/packages/pds/src/lexicon/types/app/bsky/unspecced/applyLabels.ts b/packages/pds/src/lexicon/types/app/bsky/unspecced/applyLabels.ts deleted file mode 100644 index 1d359a9547d..00000000000 --- a/packages/pds/src/lexicon/types/app/bsky/unspecced/applyLabels.ts +++ /dev/null @@ -1,39 +0,0 @@ -/** - * GENERATED CODE - DO NOT MODIFY - */ -import express from 'express' -import { ValidationResult, BlobRef } from '@atproto/lexicon' -import { lexicons } from '../../../../lexicons' -import { isObj, hasProp } from '../../../../util' -import { CID } from 'multiformats/cid' -import { HandlerAuth } from '@atproto/xrpc-server' -import * as ComAtprotoLabelDefs from '../../../com/atproto/label/defs' - -export interface QueryParams {} - -export interface InputSchema { - labels: ComAtprotoLabelDefs.Label[] - [k: string]: unknown -} - -export interface HandlerInput { - encoding: 'application/json' - body: InputSchema -} - -export interface HandlerError { - status: number - message?: string -} - -export type HandlerOutput = HandlerError | void -export type HandlerReqCtx = { - auth: HA - params: QueryParams - input: HandlerInput - req: express.Request - res: express.Response -} -export type Handler = ( - ctx: HandlerReqCtx, -) => Promise | HandlerOutput diff --git a/packages/pds/src/services/account/index.ts b/packages/pds/src/services/account/index.ts index 8173d9160af..0cb293ff26c 100644 --- a/packages/pds/src/services/account/index.ts +++ b/packages/pds/src/services/account/index.ts @@ -1,18 +1,17 @@ -import { WhereInterface, sql } from 'kysely' +import { sql } from 'kysely' import { dbLogger as log } from '../../logger' import Database from '../../db' import * as scrypt from '../../db/scrypt' import { UserAccountEntry } from '../../db/tables/user-account' import { DidHandle } from '../../db/tables/did-handle' import { RepoRoot } from '../../db/tables/repo-root' -import { DbRef, countAll, notSoftDeletedClause } from '../../db/util' +import { countAll, notSoftDeletedClause } from '../../db/util' import { getUserSearchQueryPg, getUserSearchQuerySqlite } from '../util/search' import { paginate, TimeCidKeyset } from '../../db/pagination' import * as sequencer from '../../sequencer' import { AppPassword } from '../../lexicon/types/com/atproto/server/createAppPassword' import { randomStr } from '@atproto/crypto' import { InvalidRequestError } from '@atproto/xrpc-server' -import { NotEmptyArray } from '@atproto/common' export class AccountService { constructor(public db: Database) {} @@ -355,27 +354,6 @@ export class AccountService { .execute() } - whereNotMuted>( - qb: W, - requester: string, - refs: NotEmptyArray, - ) { - const subjectRefs = sql.join(refs) - const actorMute = this.db.db - .selectFrom('mute') - .where('mutedByDid', '=', requester) - .where('did', 'in', sql`(${subjectRefs})`) - .select('did as muted') - const listMute = this.db.db - .selectFrom('list_item') - .innerJoin('list_mute', 'list_mute.listUri', 'list_item.listUri') - .where('list_mute.mutedByDid', '=', requester) - .whereRef('list_item.subjectDid', 'in', sql`(${subjectRefs})`) - .select('list_item.subjectDid as muted') - // Splitting the mute from list-mute checks seems to be more flexible for the query-planner and often quicker - return qb.whereNotExists(actorMute).whereNotExists(listMute) - } - async search(opts: { searchField?: 'did' | 'handle' term: string @@ -397,13 +375,9 @@ export class AccountService { const builder = this.db.dialect === 'pg' ? getUserSearchQueryPg(this.db, opts) - .innerJoin('repo_root', 'repo_root.did', 'did_handle.did') .selectAll('did_handle') .selectAll('repo_root') - .select('results.distance as distance') : getUserSearchQuerySqlite(this.db, opts) - .leftJoin('profile', 'profile.creator', 'did_handle.did') // @TODO leaky, for getUserSearchQuerySqlite() - .innerJoin('repo_root', 'repo_root.did', 'did_handle.did') .selectAll('did_handle') .selectAll('repo_root') .select(sql`0`.as('distance')) diff --git a/packages/pds/src/services/index.ts b/packages/pds/src/services/index.ts index a60451785ec..3dd376a8dd6 100644 --- a/packages/pds/src/services/index.ts +++ b/packages/pds/src/services/index.ts @@ -2,26 +2,18 @@ import { AtpAgent } from '@atproto/api' import * as crypto from '@atproto/crypto' import { BlobStore } from '@atproto/repo' import Database from '../db' -import { MessageDispatcher } from '../event-stream/message-queue' import { AccountService } from './account' import { AuthService } from './auth' import { RecordService } from './record' import { RepoService } from './repo' import { ModerationService } from './moderation' -import { IndexingService } from '../app-view/services/indexing' -import { Labeler } from '../labeler' -import { LabelService } from '../app-view/services/label' -import { BackgroundQueue } from '../event-stream/background-queue' +import { BackgroundQueue } from '../background' import { Crawlers } from '../crawlers' -import { LabelCache } from '../label-cache' import { LocalService } from './local' export function createServices(resources: { repoSigningKey: crypto.Keypair - messageDispatcher: MessageDispatcher blobstore: BlobStore - labeler: Labeler - labelCache: LabelCache appviewAgent?: AtpAgent appviewDid?: string appviewCdnUrlPattern?: string @@ -30,10 +22,7 @@ export function createServices(resources: { }): Services { const { repoSigningKey, - messageDispatcher, blobstore, - labeler, - labelCache, appviewAgent, appviewDid, appviewCdnUrlPattern, @@ -43,14 +32,12 @@ export function createServices(resources: { return { account: AccountService.creator(), auth: AuthService.creator(), - record: RecordService.creator(messageDispatcher), + record: RecordService.creator(), repo: RepoService.creator( repoSigningKey, - messageDispatcher, blobstore, backgroundQueue, crawlers, - labeler, ), local: LocalService.creator( repoSigningKey, @@ -58,11 +45,7 @@ export function createServices(resources: { appviewDid, appviewCdnUrlPattern, ), - moderation: ModerationService.creator(messageDispatcher, blobstore), - appView: { - indexing: IndexingService.creator(backgroundQueue), - label: LabelService.creator(labelCache), - }, + moderation: ModerationService.creator(blobstore), } } @@ -73,10 +56,6 @@ export type Services = { repo: FromDb local: FromDb moderation: FromDb - appView: { - indexing: FromDb - label: FromDb - } } type FromDb = (db: Database) => T diff --git a/packages/pds/src/services/moderation/index.ts b/packages/pds/src/services/moderation/index.ts index 6104ffe728b..9e46332cf33 100644 --- a/packages/pds/src/services/moderation/index.ts +++ b/packages/pds/src/services/moderation/index.ts @@ -4,7 +4,6 @@ import { BlobStore } from '@atproto/repo' import { AtUri } from '@atproto/syntax' import { InvalidRequestError } from '@atproto/xrpc-server' import Database from '../../db' -import { MessageQueue } from '../../event-stream/types' import { ModerationAction, ModerationReport } from '../../db/tables/moderation' import { RecordService } from '../record' import { ModerationViews } from './views' @@ -13,21 +12,16 @@ import { TAKEDOWN } from '../../lexicon/types/com/atproto/admin/defs' import { addHoursToDate } from '../../util/date' export class ModerationService { - constructor( - public db: Database, - public messageDispatcher: MessageQueue, - public blobstore: BlobStore, - ) {} - - static creator(messageDispatcher: MessageQueue, blobstore: BlobStore) { - return (db: Database) => - new ModerationService(db, messageDispatcher, blobstore) + constructor(public db: Database, public blobstore: BlobStore) {} + + static creator(blobstore: BlobStore) { + return (db: Database) => new ModerationService(db, blobstore) } - views = new ModerationViews(this.db, this.messageDispatcher) + views = new ModerationViews(this.db) services = { - record: RecordService.creator(this.messageDispatcher), + record: RecordService.creator(), } async getAction(id: number): Promise { diff --git a/packages/pds/src/services/moderation/views.ts b/packages/pds/src/services/moderation/views.ts index 2403de729fb..e8d89620d73 100644 --- a/packages/pds/src/services/moderation/views.ts +++ b/packages/pds/src/services/moderation/views.ts @@ -2,7 +2,6 @@ import { Selectable } from 'kysely' import { ArrayEl, cborBytesToRecord } from '@atproto/common' import { AtUri } from '@atproto/syntax' import Database from '../../db' -import { MessageQueue } from '../../event-stream/types' import { DidHandle } from '../../db/tables/did-handle' import { RepoRoot } from '../../db/tables/repo-root' import { @@ -17,19 +16,18 @@ import { BlobView, } from '../../lexicon/types/com/atproto/admin/defs' import { OutputSchema as ReportOutput } from '../../lexicon/types/com/atproto/moderation/createReport' -import { Label } from '../../lexicon/types/com/atproto/label/defs' import { ModerationAction } from '../../db/tables/moderation' import { AccountService } from '../account' import { RecordService } from '../record' import { ModerationReportRowWithHandle } from '.' -import { getSelfLabels } from '../../app-view/services/label' +import { ids } from '../../lexicon/lexicons' export class ModerationViews { - constructor(private db: Database, private messageDispatcher: MessageQueue) {} + constructor(private db: Database) {} services = { account: AccountService.creator(), - record: RecordService.creator(this.messageDispatcher), + record: RecordService.creator(), } repo(result: RepoResult, opts: ModViewOptions): Promise @@ -45,10 +43,15 @@ export class ModerationViews { await this.db.db .selectFrom('did_handle') .leftJoin('user_account', 'user_account.did', 'did_handle.did') - .leftJoin('profile', 'profile.creator', 'did_handle.did') + .leftJoin('record as profile_record', (join) => + join + .onRef('profile_record.did', '=', 'did_handle.did') + .on('profile_record.collection', '=', ids.AppBskyActorProfile) + .on('profile_record.rkey', '=', 'self'), + ) .leftJoin('ipld_block as profile_block', (join) => join - .onRef('profile_block.cid', '=', 'profile.cid') + .onRef('profile_block.cid', '=', 'profile_record.cid') .onRef('profile_block.creator', '=', 'did_handle.did'), ) .where( @@ -143,10 +146,9 @@ export class ModerationViews { .execute(), this.services.account(this.db).getAccountInviteCodes(repo.did), ]) - const [reports, actions, labels] = await Promise.all([ + const [reports, actions] = await Promise.all([ this.report(reportResults), this.action(actionResults), - this.labels(repo.did), ]) return { ...repo, @@ -156,7 +158,6 @@ export class ModerationViews { actions, }, invites: inviteCodes, - labels, } } @@ -270,17 +271,11 @@ export class ModerationViews { .selectAll() .execute(), ]) - const [reports, actions, blobs, labels] = await Promise.all([ + const [reports, actions, blobs] = await Promise.all([ this.report(reportResults), this.action(actionResults), this.blob(record.blobCids), - this.labels(record.uri), ]) - const selfLabels = getSelfLabels({ - uri: result.uri, - cid: result.cid, - record: result.value as Record, - }) return { ...record, blobs, @@ -289,7 +284,6 @@ export class ModerationViews { reports, actions, }, - labels: [...labels, ...selfLabels], } } @@ -609,21 +603,6 @@ export class ModerationViews { } }) } - - // @TODO: call into label service instead on AppView - async labels(subject: string, includeNeg?: boolean): Promise { - const res = await this.db.db - .selectFrom('label') - .where('label.uri', '=', subject) - .if(!includeNeg, (qb) => qb.where('neg', '=', 0)) - .selectAll() - .execute() - return res.map((l) => ({ - ...l, - cid: l.cid === '' ? undefined : l.cid, - neg: l.neg === 1, - })) - } } type RepoResult = DidHandle & RepoRoot diff --git a/packages/pds/src/services/record/index.ts b/packages/pds/src/services/record/index.ts index 857ae7f1d95..3cdb7bbc05e 100644 --- a/packages/pds/src/services/record/index.ts +++ b/packages/pds/src/services/record/index.ts @@ -6,19 +6,13 @@ import { dbLogger as log } from '../../logger' import Database from '../../db' import { notSoftDeletedClause } from '../../db/util' import { Backlink } from '../../db/tables/backlink' -import { MessageQueue } from '../../event-stream/types' -import { - indexRecord, - deleteRecord, - deleteRepo, -} from '../../event-stream/messages' import { ids } from '../../lexicon/lexicons' export class RecordService { - constructor(public db: Database, public messageDispatcher: MessageQueue) {} + constructor(public db: Database) {} - static creator(messageDispatcher: MessageQueue) { - return (db: Database) => new RecordService(db, messageDispatcher) + static creator() { + return (db: Database) => new RecordService(db) } async indexRecord( @@ -70,30 +64,19 @@ export class RecordService { } await this.addBacklinks(backlinks) - // Send to indexers - await this.messageDispatcher.send( - this.db, - indexRecord(uri, cid, obj, action, record.indexedAt), - ) - log.info({ uri }, 'indexed record') } - async deleteRecord(uri: AtUri, cascading = false) { + async deleteRecord(uri: AtUri) { this.db.assertTransaction() log.debug({ uri }, 'deleting indexed record') const deleteQuery = this.db.db .deleteFrom('record') .where('uri', '=', uri.toString()) - .execute() - await this.db.db + const backlinkQuery = this.db.db .deleteFrom('backlink') .where('uri', '=', uri.toString()) - .execute() - await Promise.all([ - this.messageDispatcher.send(this.db, deleteRecord(uri, cascading)), - deleteQuery, - ]) + await Promise.all([deleteQuery.execute(), backlinkQuery.execute()]) log.info({ uri }, 'deleted indexed record') } @@ -233,7 +216,6 @@ export class RecordService { async deleteForActor(did: string) { // Not done in transaction because it would be too long, prone to contention. // Also, this can safely be run multiple times if it fails. - await this.messageDispatcher.send(this.db, deleteRepo(did)) // Needs record table await this.db.db.deleteFrom('record').where('did', '=', did).execute() await this.db.db .deleteFrom('user_notification') diff --git a/packages/pds/src/services/repo/blobs.ts b/packages/pds/src/services/repo/blobs.ts index 7078f66a76e..a366f4def1f 100644 --- a/packages/pds/src/services/repo/blobs.ts +++ b/packages/pds/src/services/repo/blobs.ts @@ -13,7 +13,7 @@ import { Blob as BlobTable } from '../../db/tables/blob' import * as img from '../../image' import { BlobRef } from '@atproto/lexicon' import { PreparedDelete, PreparedUpdate } from '../../repo' -import { BackgroundQueue } from '../../event-stream/background-queue' +import { BackgroundQueue } from '../../background' export class RepoBlobs { constructor( diff --git a/packages/pds/src/services/repo/index.ts b/packages/pds/src/services/repo/index.ts index 8b8db8eb6be..406635b736d 100644 --- a/packages/pds/src/services/repo/index.ts +++ b/packages/pds/src/services/repo/index.ts @@ -4,7 +4,6 @@ import { BlobStore, CommitData, Repo, WriteOpAction } from '@atproto/repo' import { InvalidRequestError } from '@atproto/xrpc-server' import { AtUri } from '@atproto/syntax' import Database from '../../db' -import { MessageQueue } from '../../event-stream/types' import SqlRepoStorage from '../../sql-repo-storage' import { BadCommitSwapError, @@ -16,9 +15,8 @@ import { RepoBlobs } from './blobs' import { createWriteToOp, writeToOp } from '../../repo' import { RecordService } from '../record' import * as sequencer from '../../sequencer' -import { Labeler } from '../../labeler' import { wait } from '@atproto/common' -import { BackgroundQueue } from '../../event-stream/background-queue' +import { BackgroundQueue } from '../../background' import { Crawlers } from '../../crawlers' export class RepoService { @@ -27,37 +25,25 @@ export class RepoService { constructor( public db: Database, public repoSigningKey: crypto.Keypair, - public messageDispatcher: MessageQueue, public blobstore: BlobStore, public backgroundQueue: BackgroundQueue, public crawlers: Crawlers, - public labeler: Labeler, ) { this.blobs = new RepoBlobs(db, blobstore, backgroundQueue) } static creator( keypair: crypto.Keypair, - messageDispatcher: MessageQueue, blobstore: BlobStore, backgroundQueue: BackgroundQueue, crawlers: Crawlers, - labeler: Labeler, ) { return (db: Database) => - new RepoService( - db, - keypair, - messageDispatcher, - blobstore, - backgroundQueue, - crawlers, - labeler, - ) + new RepoService(db, keypair, blobstore, backgroundQueue, crawlers) } services = { - record: RecordService.creator(this.messageDispatcher), + record: RecordService.creator(), } private async serviceTx( @@ -68,11 +54,9 @@ export class RepoService { const srvc = new RepoService( dbTxn, this.repoSigningKey, - this.messageDispatcher, this.blobstore, this.backgroundQueue, this.crawlers, - this.labeler, ) return fn(srvc) }) @@ -294,15 +278,6 @@ export class RepoService { this.backgroundQueue.add(async () => { await this.crawlers.notifyOfUpdate() }) - writes.forEach((write) => { - if ( - write.action === WriteOpAction.Create || - write.action === WriteOpAction.Update - ) { - // @TODO move to appview - this.labeler.processRecord(write.uri, write.record) - } - }) }) const seqEvt = await sequencer.formatSeqCommit(did, commitData, writes) diff --git a/packages/pds/src/services/util/search.ts b/packages/pds/src/services/util/search.ts index 007052e707c..26b3e81bf06 100644 --- a/packages/pds/src/services/util/search.ts +++ b/packages/pds/src/services/util/search.ts @@ -1,7 +1,7 @@ import { sql } from 'kysely' import { InvalidRequestError } from '@atproto/xrpc-server' import Database from '../../db' -import { notSoftDeletedClause, DbRef, AnyQb } from '../../db/util' +import { notSoftDeletedClause, DbRef } from '../../db/util' import { GenericKeyset, paginate } from '../../db/pagination' // @TODO utilized in both pds and app-view @@ -19,58 +19,13 @@ export const getUserSearchQueryPg = ( const { term, limit, cursor, includeSoftDeleted } = opts // Matching user accounts based on handle const distanceAccount = distance(term, ref('handle')) - let accountsQb = getMatchingAccountsQb(db, { term, includeSoftDeleted }) - accountsQb = paginate(accountsQb, { + const accountsQb = getMatchingAccountsQb(db, { term, includeSoftDeleted }) + return paginate(accountsQb, { limit, cursor, direction: 'asc', keyset: new SearchKeyset(distanceAccount, ref('handle')), }) - // Matching profiles based on display name - const distanceProfile = distance(term, ref('displayName')) - let profilesQb = getMatchingProfilesQb(db, { term, includeSoftDeleted }) - profilesQb = paginate( - profilesQb.innerJoin('did_handle', 'did_handle.did', 'profile.creator'), // for handle pagination - { - limit, - cursor, - direction: 'asc', - keyset: new SearchKeyset(distanceProfile, ref('handle')), - }, - ) - // Combine and paginate result set - return paginate(combineAccountsAndProfilesQb(db, accountsQb, profilesQb), { - limit, - cursor, - direction: 'asc', - keyset: new SearchKeyset(ref('distance'), ref('handle')), - }) -} - -// Takes maximal advantage of trigram index at the expense of ability to paginate. -export const getUserSearchQuerySimplePg = ( - db: Database, - opts: { - term: string - limit: number - }, -) => { - const { ref } = db.db.dynamic - const { term, limit } = opts - // Matching user accounts based on handle - const accountsQb = getMatchingAccountsQb(db, { term }) - .orderBy('distance', 'asc') - .limit(limit) - // Matching profiles based on display name - const profilesQb = getMatchingProfilesQb(db, { term }) - .orderBy('distance', 'asc') - .limit(limit) - // Combine and paginate result set - return paginate(combineAccountsAndProfilesQb(db, accountsQb, profilesQb), { - limit, - direction: 'asc', - keyset: new SearchKeyset(ref('distance'), ref('handle')), - }) } // Matching user accounts based on handle @@ -91,51 +46,6 @@ const getMatchingAccountsQb = ( .select(['did_handle.did as did', distanceAccount.as('distance')]) } -// Matching profiles based on display name -const getMatchingProfilesQb = ( - db: Database, - opts: { term: string; includeSoftDeleted?: boolean }, -) => { - const { ref } = db.db.dynamic - const { term, includeSoftDeleted } = opts - const distanceProfile = distance(term, ref('displayName')) - return db.db - .selectFrom('profile') - .innerJoin('repo_root', 'repo_root.did', 'profile.creator') - .if(!includeSoftDeleted, (qb) => - qb.where(notSoftDeletedClause(ref('repo_root'))), - ) - .where(similar(term, ref('displayName'))) // Coarse filter engaging trigram index - .select(['profile.creator as did', distanceProfile.as('distance')]) -} - -// Combine profile and account result sets -const combineAccountsAndProfilesQb = ( - db: Database, - accountsQb: AnyQb, - profilesQb: AnyQb, -) => { - // Combine user account and profile results, taking best matches from each - const emptyQb = db.db - .selectFrom('user_account') - .where(sql`1 = 0`) - .select([sql.literal('').as('did'), sql`0`.as('distance')]) - const resultsQb = db.db - .selectFrom( - emptyQb - .unionAll(sql`${accountsQb}`) // The sql`` is adding parens - .unionAll(sql`${profilesQb}`) - .as('accounts_and_profiles'), - ) - .selectAll() - .distinctOn('did') // Per did, take whichever of account and profile distance is best - .orderBy('did') - .orderBy('distance') - return db.db - .selectFrom(resultsQb.as('results')) - .innerJoin('did_handle', 'did_handle.did', 'results.did') -} - export const getUserSearchQuerySqlite = ( db: Database, opts: { @@ -160,7 +70,10 @@ export const getUserSearchQuerySqlite = ( if (!safeWords.length) { // Return no results. This could happen with weird input like ' % _ '. - return db.db.selectFrom('did_handle').where(sql`1 = 0`) + return db.db + .selectFrom('did_handle') + .innerJoin('repo_root', 'repo_root.did', 'did_handle.did') + .where(sql`1 = 0`) } // We'll ensure there's a space before each word in both textForMatch and in safeWords, @@ -174,9 +87,9 @@ export const getUserSearchQuerySqlite = ( return db.db .selectFrom('did_handle') - .innerJoin('repo_root as _repo_root', '_repo_root.did', 'did_handle.did') + .innerJoin('repo_root', 'repo_root.did', 'did_handle.did') .if(!includeSoftDeleted, (qb) => - qb.where(notSoftDeletedClause(ref('_repo_root'))), + qb.where(notSoftDeletedClause(ref('repo_root'))), ) .where((q) => { safeWords.forEach((word) => { diff --git a/packages/pds/tests/_util.ts b/packages/pds/tests/_util.ts index 16061b8af51..b8135836c15 100644 --- a/packages/pds/tests/_util.ts +++ b/packages/pds/tests/_util.ts @@ -98,9 +98,6 @@ export const runTestServer = async ( dbPostgresUrl: process.env.DB_POSTGRES_URL, blobstoreLocation: `${blobstoreLoc}/blobs`, blobstoreTmp: `${blobstoreLoc}/tmp`, - labelerDid: 'did:example:labeler', - labelerKeywords: { label_me: 'test-label', label_me_2: 'test-label-2' }, - feedGenDid: 'did:example:feedGen', maxSubscriptionBuffer: 200, repoBackfillLimitMs: HOUR, sequencerLeaderLockId: uniqueLockId(), @@ -153,9 +150,6 @@ export const runTestServer = async ( const pdsServer = await pds.start() const pdsPort = (pdsServer.address() as AddressInfo).port - // we refresh label cache by hand in `processAll` instead of on a timer - pds.ctx.labelCache.stop() - return { url: `http://localhost:${pdsPort}`, ctx: pds.ctx, @@ -165,7 +159,6 @@ export const runTestServer = async ( }, processAll: async () => { await pds.ctx.backgroundQueue.processAll() - await pds.ctx.labelCache.fullRefresh() }, } } diff --git a/packages/pds/tests/account-deletion.test.ts b/packages/pds/tests/account-deletion.test.ts index b99807ec6d5..d6b4aa101ce 100644 --- a/packages/pds/tests/account-deletion.test.ts +++ b/packages/pds/tests/account-deletion.test.ts @@ -11,24 +11,12 @@ import { BlobNotFoundError, BlobStore } from '@atproto/repo' import { RepoRoot } from '../src/db/tables/repo-root' import { UserAccount } from '../src/db/tables/user-account' import { IpldBlock } from '../src/db/tables/ipld-block' -import { Post } from '../src/app-view/db/tables/post' -import { Like } from '../src/app-view/db/tables/like' -import { Repost } from '../src/app-view/db/tables/repost' -import { Follow } from '../src/app-view/db/tables/follow' import { RepoBlob } from '../src/db/tables/repo-blob' import { Blob } from '../src/db/tables/blob' -import { - PostEmbedImage, - PostEmbedExternal, - PostEmbedRecord, -} from '../src/app-view/db/tables/post-embed' import { Record } from '../src/db/tables/record' import { RepoSeq } from '../src/db/tables/repo-seq' import { ACKNOWLEDGE } from '../src/lexicon/types/com/atproto/admin/defs' import { UserState } from '../src/db/tables/user-state' -import { ActorBlock } from '../src/app-view/db/tables/actor-block' -import { List } from '../src/app-view/db/tables/list' -import { ListItem } from '../src/app-view/db/tables/list-item' describe('account deletion', () => { let server: util.TestServerInfo @@ -179,43 +167,10 @@ describe('account deletion', () => { (row) => row.did === carol.did && row.eventType === 'tombstone', ).length, ).toEqual(1) - }) - it('no longer stores indexed records from the user', async () => { expect(updatedDbContents.records).toEqual( initialDbContents.records.filter((row) => row.did !== carol.did), ) - expect(updatedDbContents.posts).toEqual( - initialDbContents.posts.filter((row) => row.creator !== carol.did), - ) - expect(updatedDbContents.likes).toEqual( - initialDbContents.likes.filter((row) => row.creator !== carol.did), - ) - expect(updatedDbContents.actorBlocks).toEqual( - initialDbContents.actorBlocks.filter((row) => row.creator !== carol.did), - ) - expect(updatedDbContents.lists).toEqual( - initialDbContents.lists.filter((row) => row.creator !== carol.did), - ) - expect(updatedDbContents.listItems).toEqual( - initialDbContents.listItems.filter((row) => row.creator !== carol.did), - ) - expect(updatedDbContents.reposts).toEqual( - initialDbContents.reposts.filter((row) => row.creator !== carol.did), - ) - expect(updatedDbContents.follows).toEqual( - initialDbContents.follows.filter((row) => row.creator !== carol.did), - ) - expect(updatedDbContents.postImages).toEqual( - initialDbContents.postImages.filter( - (row) => !row.postUri.includes(carol.did), - ), - ) - expect(updatedDbContents.postExternals).toEqual( - initialDbContents.postExternals.filter( - (row) => !row.postUri.includes(carol.did), - ), - ) }) it('deletes relevant blobs', async () => { @@ -269,82 +224,32 @@ type DbContents = { blocks: IpldBlock[] seqs: Selectable[] records: Record[] - posts: Post[] - postImages: PostEmbedImage[] - postExternals: PostEmbedExternal[] - postRecords: PostEmbedRecord[] - likes: Like[] - reposts: Repost[] - follows: Follow[] - actorBlocks: ActorBlock[] - lists: List[] - listItems: ListItem[] repoBlobs: RepoBlob[] blobs: Blob[] } const getDbContents = async (db: Database): Promise => { - const [ - roots, - users, - userState, - blocks, - seqs, - records, - posts, - postImages, - postExternals, - postRecords, - likes, - reposts, - follows, - actorBlocks, - lists, - listItems, - repoBlobs, - blobs, - ] = await Promise.all([ - db.db.selectFrom('repo_root').orderBy('did').selectAll().execute(), - db.db.selectFrom('user_account').orderBy('did').selectAll().execute(), - db.db.selectFrom('user_state').orderBy('did').selectAll().execute(), - db.db - .selectFrom('ipld_block') - .orderBy('creator') - .orderBy('cid') - .selectAll() - .execute(), - db.db.selectFrom('repo_seq').orderBy('id').selectAll().execute(), - db.db.selectFrom('record').orderBy('uri').selectAll().execute(), - db.db.selectFrom('post').orderBy('uri').selectAll().execute(), - db.db - .selectFrom('post_embed_image') - .orderBy('postUri') - .selectAll() - .execute(), - db.db - .selectFrom('post_embed_external') - .orderBy('postUri') - .selectAll() - .execute(), - db.db - .selectFrom('post_embed_record') - .orderBy('postUri') - .selectAll() - .execute(), - db.db.selectFrom('like').orderBy('uri').selectAll().execute(), - db.db.selectFrom('repost').orderBy('uri').selectAll().execute(), - db.db.selectFrom('follow').orderBy('uri').selectAll().execute(), - db.db.selectFrom('actor_block').orderBy('uri').selectAll().execute(), - db.db.selectFrom('list').orderBy('uri').selectAll().execute(), - db.db.selectFrom('list_item').orderBy('uri').selectAll().execute(), - db.db - .selectFrom('repo_blob') - .orderBy('did') - .orderBy('cid') - .selectAll() - .execute(), - db.db.selectFrom('blob').orderBy('cid').selectAll().execute(), - ]) + const [roots, users, userState, blocks, seqs, records, repoBlobs, blobs] = + await Promise.all([ + db.db.selectFrom('repo_root').orderBy('did').selectAll().execute(), + db.db.selectFrom('user_account').orderBy('did').selectAll().execute(), + db.db.selectFrom('user_state').orderBy('did').selectAll().execute(), + db.db + .selectFrom('ipld_block') + .orderBy('creator') + .orderBy('cid') + .selectAll() + .execute(), + db.db.selectFrom('repo_seq').orderBy('id').selectAll().execute(), + db.db.selectFrom('record').orderBy('uri').selectAll().execute(), + db.db + .selectFrom('repo_blob') + .orderBy('did') + .orderBy('cid') + .selectAll() + .execute(), + db.db.selectFrom('blob').orderBy('cid').selectAll().execute(), + ]) return { roots, @@ -353,16 +258,6 @@ const getDbContents = async (db: Database): Promise => { blocks, seqs, records, - posts, - postImages, - postExternals, - postRecords, - likes, - reposts, - follows, - actorBlocks, - lists, - listItems, repoBlobs, blobs, } diff --git a/packages/pds/tests/admin/__snapshots__/get-record.test.ts.snap b/packages/pds/tests/admin/__snapshots__/get-record.test.ts.snap index 3a816975459..00fbc5bda1c 100644 --- a/packages/pds/tests/admin/__snapshots__/get-record.test.ts.snap +++ b/packages/pds/tests/admin/__snapshots__/get-record.test.ts.snap @@ -6,16 +6,6 @@ Object { "blobs": Array [], "cid": "cids(0)", "indexedAt": "1970-01-01T00:00:00.000Z", - "labels": Array [ - Object { - "cid": "cids(0)", - "cts": "1970-01-01T00:00:00.000Z", - "neg": false, - "src": "user(0)", - "uri": "record(0)", - "val": "self-label", - }, - ], "moderation": Object { "actions": Array [ Object { @@ -143,169 +133,6 @@ Object { "blobs": Array [], "cid": "cids(0)", "indexedAt": "1970-01-01T00:00:00.000Z", - "labels": Array [ - Object { - "cid": "cids(0)", - "cts": "1970-01-01T00:00:00.000Z", - "neg": false, - "src": "user(0)", - "uri": "record(0)", - "val": "self-label", - }, - ], - "moderation": Object { - "actions": Array [ - Object { - "action": "com.atproto.admin.defs#takedown", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 3, - "reason": "X", - "resolvedReportIds": Array [], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectBlobCids": Array [], - }, - Object { - "action": "com.atproto.admin.defs#acknowledge", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 2, - "reason": "X", - "resolvedReportIds": Array [], - "reversal": Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "reason": "X", - }, - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectBlobCids": Array [], - }, - ], - "currentAction": Object { - "action": "com.atproto.admin.defs#takedown", - "id": 3, - }, - "reports": Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 2, - "reason": "defamation", - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(1)", - "resolvedByActionIds": Array [], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectRepoHandle": "alice.test", - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(2)", - "resolvedByActionIds": Array [], - "subject": Object { - "$type": "com.atproto.repo.strongRef", - "cid": "cids(0)", - "uri": "record(0)", - }, - "subjectRepoHandle": "alice.test", - }, - ], - }, - "repo": Object { - "did": "user(0)", - "email": "alice@test.com", - "handle": "alice.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitesDisabled": false, - "moderation": Object {}, - "relatedRecords": Array [ - Object { - "$type": "app.bsky.actor.profile", - "avatar": Object { - "$type": "blob", - "mimeType": "image/jpeg", - "ref": Object { - "$link": "cids(1)", - }, - "size": 3976, - }, - "description": "its me!", - "displayName": "ali", - "labels": Object { - "$type": "com.atproto.label.defs#selfLabels", - "values": Array [ - Object { - "val": "self-label-a", - }, - Object { - "val": "self-label-b", - }, - ], - }, - }, - ], - }, - "uri": "record(0)", - "value": Object { - "$type": "app.bsky.feed.post", - "createdAt": "1970-01-01T00:00:00.000Z", - "labels": Object { - "$type": "com.atproto.label.defs#selfLabels", - "values": Array [ - Object { - "val": "self-label", - }, - ], - }, - "text": "hey there", - }, -} -`; - -exports[`pds admin get record view serves labels. 1`] = ` -Object { - "blobCids": Array [], - "blobs": Array [], - "cid": "cids(0)", - "indexedAt": "1970-01-01T00:00:00.000Z", - "labels": Array [ - Object { - "cid": "cids(0)", - "cts": "1970-01-01T00:00:00.000Z", - "neg": false, - "src": "did:example:labeler", - "uri": "record(0)", - "val": "kittens", - }, - Object { - "cid": "cids(0)", - "cts": "1970-01-01T00:00:00.000Z", - "neg": false, - "src": "did:example:labeler", - "uri": "record(0)", - "val": "puppies", - }, - Object { - "cid": "cids(0)", - "cts": "1970-01-01T00:00:00.000Z", - "neg": false, - "src": "user(0)", - "uri": "record(0)", - "val": "self-label", - }, - ], "moderation": Object { "actions": Array [ Object { diff --git a/packages/pds/tests/admin/__snapshots__/get-repo.test.ts.snap b/packages/pds/tests/admin/__snapshots__/get-repo.test.ts.snap index 7234789c2c1..c90b1a070b2 100644 --- a/packages/pds/tests/admin/__snapshots__/get-repo.test.ts.snap +++ b/packages/pds/tests/admin/__snapshots__/get-repo.test.ts.snap @@ -8,124 +8,6 @@ Object { "indexedAt": "1970-01-01T00:00:00.000Z", "invites": Array [], "invitesDisabled": false, - "labels": Array [], - "moderation": Object { - "actions": Array [ - Object { - "action": "com.atproto.admin.defs#takedown", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 3, - "reason": "X", - "resolvedReportIds": Array [], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - "subjectBlobCids": Array [], - }, - Object { - "action": "com.atproto.admin.defs#acknowledge", - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "id": 2, - "reason": "X", - "resolvedReportIds": Array [], - "reversal": Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "createdBy": "did:example:admin", - "reason": "X", - }, - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - "subjectBlobCids": Array [], - }, - ], - "currentAction": Object { - "action": "com.atproto.admin.defs#takedown", - "id": 3, - }, - "reports": Array [ - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 2, - "reason": "defamation", - "reasonType": "com.atproto.moderation.defs#reasonOther", - "reportedBy": "user(1)", - "resolvedByActionIds": Array [], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - }, - Object { - "createdAt": "1970-01-01T00:00:00.000Z", - "id": 1, - "reasonType": "com.atproto.moderation.defs#reasonSpam", - "reportedBy": "user(2)", - "resolvedByActionIds": Array [], - "subject": Object { - "$type": "com.atproto.admin.defs#repoRef", - "did": "user(0)", - }, - }, - ], - }, - "relatedRecords": Array [ - Object { - "$type": "app.bsky.actor.profile", - "avatar": Object { - "$type": "blob", - "mimeType": "image/jpeg", - "ref": Object { - "$link": "cids(0)", - }, - "size": 3976, - }, - "description": "its me!", - "displayName": "ali", - "labels": Object { - "$type": "com.atproto.label.defs#selfLabels", - "values": Array [ - Object { - "val": "self-label-a", - }, - Object { - "val": "self-label-b", - }, - ], - }, - }, - ], -} -`; - -exports[`pds admin get repo view serves labels. 1`] = ` -Object { - "did": "user(0)", - "email": "alice@test.com", - "handle": "alice.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invites": Array [], - "invitesDisabled": false, - "labels": Array [ - Object { - "cts": "1970-01-01T00:00:00.000Z", - "neg": false, - "src": "did:example:labeler", - "uri": "user(0)", - "val": "kittens", - }, - Object { - "cts": "1970-01-01T00:00:00.000Z", - "neg": false, - "src": "did:example:labeler", - "uri": "user(0)", - "val": "puppies", - }, - ], "moderation": Object { "actions": Array [ Object { diff --git a/packages/pds/tests/admin/get-record.test.ts b/packages/pds/tests/admin/get-record.test.ts index d70707b2b70..6c38419612e 100644 --- a/packages/pds/tests/admin/get-record.test.ts +++ b/packages/pds/tests/admin/get-record.test.ts @@ -120,29 +120,4 @@ describe('pds admin get record view', () => { ) await expect(promise).rejects.toThrow('Record not found') }) - - it('serves labels.', async () => { - const { ctx } = server - const labelingService = ctx.services.appView.label(ctx.db) - await labelingService.formatAndCreate( - ctx.cfg.labelerDid, - sc.posts[sc.dids.alice][0].ref.uriStr, - sc.posts[sc.dids.alice][0].ref.cidStr, - { create: ['kittens', 'puppies', 'birds'] }, - ) - await labelingService.formatAndCreate( - ctx.cfg.labelerDid, - sc.posts[sc.dids.alice][0].ref.uriStr, - sc.posts[sc.dids.alice][0].ref.cidStr, - { negate: ['birds'] }, - ) - const result = await agent.api.com.atproto.admin.getRecord( - { - uri: sc.posts[sc.dids.alice][0].ref.uriStr, - cid: sc.posts[sc.dids.alice][0].ref.cidStr, - }, - { headers: { authorization: adminAuth() } }, - ) - expect(forSnapshot(result.data)).toMatchSnapshot() - }) }) diff --git a/packages/pds/tests/admin/get-repo.test.ts b/packages/pds/tests/admin/get-repo.test.ts index 3cb997f6ff2..9cd38ae101f 100644 --- a/packages/pds/tests/admin/get-repo.test.ts +++ b/packages/pds/tests/admin/get-repo.test.ts @@ -108,26 +108,4 @@ describe('pds admin get repo view', () => { ) await expect(promise).rejects.toThrow('Repo not found') }) - - it('serves labels.', async () => { - const { ctx } = server - const labelingService = ctx.services.appView.label(ctx.db) - await labelingService.formatAndCreate( - ctx.cfg.labelerDid, - sc.dids.alice, - null, - { create: ['kittens', 'puppies', 'birds'] }, - ) - await labelingService.formatAndCreate( - ctx.cfg.labelerDid, - sc.dids.alice, - null, - { negate: ['birds'] }, - ) - const result = await agent.api.com.atproto.admin.getRepo( - { did: sc.dids.alice }, - { headers: { authorization: adminAuth() } }, - ) - expect(forSnapshot(result.data)).toMatchSnapshot() - }) }) diff --git a/packages/pds/tests/admin/repo-search.test.ts b/packages/pds/tests/admin/repo-search.test.ts index 3f25c893901..e3fcdef2d80 100644 --- a/packages/pds/tests/admin/repo-search.test.ts +++ b/packages/pds/tests/admin/repo-search.test.ts @@ -1,19 +1,11 @@ import AtpAgent from '@atproto/api' import { TAKEDOWN } from '@atproto/api/src/client/types/com/atproto/admin/defs' -import { - runTestServer, - forSnapshot, - CloseFn, - paginateAll, - adminAuth, -} from '../_util' +import { runTestServer, CloseFn, paginateAll, adminAuth } from '../_util' import { SeedClient } from '../seeds/client' import usersBulkSeed from '../seeds/users-bulk' -import { Database } from '../../src' describe('pds admin repo search view', () => { let agent: AtpAgent - let db: Database let close: CloseFn let sc: SeedClient let headers: { [s: string]: string } @@ -23,7 +15,6 @@ describe('pds admin repo search view', () => { dbPostgresSchema: 'views_admin_repo_search', }) close = server.close - db = server.ctx.db agent = new AtpAgent({ service: server.url }) sc = new SeedClient(agent) await usersBulkSeed(sc) @@ -54,21 +45,12 @@ describe('pds admin repo search view', () => { const shouldContain = [ 'cara-wiegand69.test', // Present despite repo takedown - 'eudora-dietrich4.test', // Carol Littel - 'shane-torphy52.test', // Sadie Carter - 'aliya-hodkiewicz.test', // Carlton Abernathy IV 'carlos6.test', 'carolina-mcdermott77.test', ] shouldContain.forEach((handle) => expect(handles).toContain(handle)) - if (db.dialect === 'pg') { - expect(handles).toContain('cayla-marquardt39.test') // Fuzzy match supported by postgres - } else { - expect(handles).not.toContain('cayla-marquardt39.test') - } - const shouldNotContain = [ 'sven70.test', 'hilario84.test', @@ -80,18 +62,12 @@ describe('pds admin repo search view', () => { ] shouldNotContain.forEach((handle) => expect(handles).not.toContain(handle)) - - if (db.dialect === 'pg') { - expect(forSnapshot(result.data.repos)).toMatchInlineSnapshot(snapPg) - } else { - expect(forSnapshot(result.data.repos)).toMatchInlineSnapshot(snapSqlite) - } }) it('finds repo by did', async () => { const term = sc.dids['cara-wiegand69.test'] const res = await agent.api.com.atproto.admin.searchRepos( - { term, limit: 1 }, + { term }, { headers }, ) @@ -99,6 +75,19 @@ describe('pds admin repo search view', () => { expect(res.data.repos[0].did).toEqual(term) }) + it('finds repo by email', async () => { + const did = sc.dids['cara-wiegand69.test'] + const { email } = sc.accounts[did] + const res = await agent.api.com.atproto.admin.searchRepos( + { term: email }, + { headers }, + ) + + expect(res.data.repos.length).toEqual(1) + expect(res.data.repos[0].did).toEqual(did) + expect(res.data.repos[0].email).toEqual(email) + }) + it('paginates with term', async () => { const results = (results) => results.flatMap((res) => res.users) const paginator = async (cursor?: string) => { @@ -119,7 +108,7 @@ describe('pds admin repo search view', () => { { headers }, ) - expect(full.data.repos.length).toBeGreaterThan(5) + expect(full.data.repos.length).toBeGreaterThan(3) expect(results(paginatedAll)).toEqual(results([full.data])) }) @@ -147,269 +136,3 @@ describe('pds admin repo search view', () => { expect(results(paginatedAll)).toEqual(results([full.data])) }) }) - -// Not using jest snapshots because it doesn't handle the conditional pg/sqlite very well: -// you can achieve it using named snapshots, but when you run the tests for pg the test suite fails -// since the sqlite snapshots appear obsolete to jest (and vice-versa when you run the sqlite suite). - -const snapPg = ` -Array [ - Object { - "did": "user(0)", - "email": "aliya-hodkiewicz.test@bsky.app", - "handle": "aliya-hodkiewicz.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitesDisabled": false, - "moderation": Object {}, - "relatedRecords": Array [ - Object { - "$type": "app.bsky.actor.profile", - "avatar": Object { - "$type": "blob", - "mimeType": "image/jpeg", - "ref": Object { - "$link": "cids(0)", - }, - "size": 3976, - }, - "description": "", - "displayName": "Carlton Abernathy IV", - }, - ], - }, - Object { - "did": "user(1)", - "email": "cara-wiegand69.test@bsky.app", - "handle": "cara-wiegand69.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitesDisabled": false, - "moderation": Object { - "currentAction": Object { - "action": "com.atproto.admin.defs#takedown", - "id": 1, - }, - }, - "relatedRecords": Array [], - }, - Object { - "did": "user(2)", - "email": "carlos6.test@bsky.app", - "handle": "carlos6.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitesDisabled": false, - "moderation": Object {}, - "relatedRecords": Array [], - }, - Object { - "did": "user(3)", - "email": "carolina-mcdermott77.test@bsky.app", - "handle": "carolina-mcdermott77.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitesDisabled": false, - "moderation": Object {}, - "relatedRecords": Array [ - Object { - "$type": "app.bsky.actor.profile", - "avatar": Object { - "$type": "blob", - "mimeType": "image/jpeg", - "ref": Object { - "$link": "cids(0)", - }, - "size": 3976, - }, - "description": "", - "displayName": "Latoya Windler", - }, - ], - }, - Object { - "did": "user(4)", - "email": "eudora-dietrich4.test@bsky.app", - "handle": "eudora-dietrich4.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitesDisabled": false, - "moderation": Object {}, - "relatedRecords": Array [ - Object { - "$type": "app.bsky.actor.profile", - "avatar": Object { - "$type": "blob", - "mimeType": "image/jpeg", - "ref": Object { - "$link": "cids(0)", - }, - "size": 3976, - }, - "description": "", - "displayName": "Carol Littel", - }, - ], - }, - Object { - "did": "user(5)", - "email": "shane-torphy52.test@bsky.app", - "handle": "shane-torphy52.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitesDisabled": false, - "moderation": Object {}, - "relatedRecords": Array [ - Object { - "$type": "app.bsky.actor.profile", - "avatar": Object { - "$type": "blob", - "mimeType": "image/jpeg", - "ref": Object { - "$link": "cids(0)", - }, - "size": 3976, - }, - "description": "", - "displayName": "Sadie Carter", - }, - ], - }, - Object { - "did": "user(6)", - "email": "cayla-marquardt39.test@bsky.app", - "handle": "cayla-marquardt39.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitesDisabled": false, - "moderation": Object {}, - "relatedRecords": Array [ - Object { - "$type": "app.bsky.actor.profile", - "avatar": Object { - "$type": "blob", - "mimeType": "image/jpeg", - "ref": Object { - "$link": "cids(0)", - }, - "size": 3976, - }, - "description": "", - "displayName": "Rachel Kshlerin", - }, - ], - }, -] -` -const snapSqlite = ` -Array [ - Object { - "did": "user(0)", - "email": "aliya-hodkiewicz.test@bsky.app", - "handle": "aliya-hodkiewicz.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitesDisabled": false, - "moderation": Object {}, - "relatedRecords": Array [ - Object { - "$type": "app.bsky.actor.profile", - "avatar": Object { - "$type": "blob", - "mimeType": "image/jpeg", - "ref": Object { - "$link": "cids(0)", - }, - "size": 3976, - }, - "description": "", - "displayName": "Carlton Abernathy IV", - }, - ], - }, - Object { - "did": "user(1)", - "email": "cara-wiegand69.test@bsky.app", - "handle": "cara-wiegand69.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitesDisabled": false, - "moderation": Object { - "currentAction": Object { - "action": "com.atproto.admin.defs#takedown", - "id": 1, - }, - }, - "relatedRecords": Array [], - }, - Object { - "did": "user(2)", - "email": "carlos6.test@bsky.app", - "handle": "carlos6.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitesDisabled": false, - "moderation": Object {}, - "relatedRecords": Array [], - }, - Object { - "did": "user(3)", - "email": "carolina-mcdermott77.test@bsky.app", - "handle": "carolina-mcdermott77.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitesDisabled": false, - "moderation": Object {}, - "relatedRecords": Array [ - Object { - "$type": "app.bsky.actor.profile", - "avatar": Object { - "$type": "blob", - "mimeType": "image/jpeg", - "ref": Object { - "$link": "cids(0)", - }, - "size": 3976, - }, - "description": "", - "displayName": "Latoya Windler", - }, - ], - }, - Object { - "did": "user(4)", - "email": "eudora-dietrich4.test@bsky.app", - "handle": "eudora-dietrich4.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitesDisabled": false, - "moderation": Object {}, - "relatedRecords": Array [ - Object { - "$type": "app.bsky.actor.profile", - "avatar": Object { - "$type": "blob", - "mimeType": "image/jpeg", - "ref": Object { - "$link": "cids(0)", - }, - "size": 3976, - }, - "description": "", - "displayName": "Carol Littel", - }, - ], - }, - Object { - "did": "user(5)", - "email": "shane-torphy52.test@bsky.app", - "handle": "shane-torphy52.test", - "indexedAt": "1970-01-01T00:00:00.000Z", - "invitesDisabled": false, - "moderation": Object {}, - "relatedRecords": Array [ - Object { - "$type": "app.bsky.actor.profile", - "avatar": Object { - "$type": "blob", - "mimeType": "image/jpeg", - "ref": Object { - "$link": "cids(0)", - }, - "size": 3976, - }, - "description": "", - "displayName": "Sadie Carter", - }, - ], - }, -] -` diff --git a/packages/pds/tests/duplicate-records.test.ts b/packages/pds/tests/duplicate-records.test.ts deleted file mode 100644 index b435dca32f3..00000000000 --- a/packages/pds/tests/duplicate-records.test.ts +++ /dev/null @@ -1,179 +0,0 @@ -import { CID } from 'multiformats/cid' -import { AtUri } from '@atproto/syntax' -import { cidForCbor, TID, cborEncode } from '@atproto/common' -import { CloseFn, runTestServer } from './_util' -import { Database } from '../src' -import * as lex from '../src/lexicon/lexicons' -import { Services } from '../src/services' - -describe('duplicate record', () => { - let close: CloseFn - let did: string - let db: Database - let services: Services - - beforeAll(async () => { - const server = await runTestServer({ - dbPostgresSchema: 'duplicates', - }) - db = server.ctx.db - services = server.ctx.services - close = server.close - did = 'did:example:alice' - }) - - afterAll(async () => { - await close() - }) - - const countRecords = async (db: Database, table: string) => { - const got = await db.db - .selectFrom(table as any) - .selectAll() - .where('creator', '=', did) - .execute() - return got.length - } - - const putBlock = async ( - db: Database, - creator: string, - data: object, - ): Promise => { - const cid = await cidForCbor(data) - const bytes = await cborEncode(data) - await db.db - .insertInto('ipld_block') - .values({ - cid: cid.toString(), - creator, - size: bytes.length, - content: bytes, - }) - .onConflict((oc) => oc.doNothing()) - .execute() - return cid - } - - it('dedupes reposts', async () => { - const subject = AtUri.make(did, lex.ids.AppBskyFeedPost, TID.nextStr()) - const subjectCid = await putBlock(db, did, { test: 'blah' }) - const coll = lex.ids.AppBskyFeedRepost - const uris: AtUri[] = [] - await db.transaction(async (tx) => { - for (let i = 0; i < 5; i++) { - const repost = { - $type: coll, - subject: { - uri: subject.toString(), - cid: subjectCid.toString(), - }, - createdAt: new Date().toISOString(), - } - const uri = AtUri.make(did, coll, TID.nextStr()) - const cid = await putBlock(tx, did, repost) - await services.record(tx).indexRecord(uri, cid, repost) - uris.push(uri) - } - }) - - let count = await countRecords(db, 'repost') - expect(count).toBe(1) - - await db.transaction(async (tx) => { - await services.record(tx).deleteRecord(uris[0], false) - }) - - count = await countRecords(db, 'repost') - expect(count).toBe(1) - - await db.transaction(async (tx) => { - await services.record(tx).deleteRecord(uris[1], true) - }) - - count = await countRecords(db, 'repost') - expect(count).toBe(0) - }) - - it('dedupes likes', async () => { - const subject = AtUri.make(did, lex.ids.AppBskyFeedPost, TID.nextStr()) - const subjectCid = await putBlock(db, did, { test: 'blah' }) - const coll = lex.ids.AppBskyFeedLike - const uris: AtUri[] = [] - await db.transaction(async (tx) => { - for (let i = 0; i < 5; i++) { - const like = { - $type: coll, - subject: { - uri: subject.toString(), - cid: subjectCid.toString(), - }, - createdAt: new Date().toISOString(), - } - const uri = AtUri.make(did, coll, TID.nextStr()) - const cid = await putBlock(tx, did, like) - await services.record(tx).indexRecord(uri, cid, like) - uris.push(uri) - } - }) - - let count = await countRecords(db, 'like') - expect(count).toBe(1) - - await db.transaction(async (tx) => { - await services.record(tx).deleteRecord(uris[0], false) - }) - - count = await countRecords(db, 'like') - expect(count).toBe(1) - - const got = await db.db - .selectFrom('like') - .where('creator', '=', did) - .selectAll() - .executeTakeFirst() - expect(got?.uri).toEqual(uris[1].toString()) - - await db.transaction(async (tx) => { - await services.record(tx).deleteRecord(uris[1], true) - }) - - count = await countRecords(db, 'like') - expect(count).toBe(0) - }) - - it('dedupes follows', async () => { - const coll = lex.ids.AppBskyGraphFollow - const uris: AtUri[] = [] - await db.transaction(async (tx) => { - for (let i = 0; i < 5; i++) { - const follow = { - $type: coll, - subject: 'did:example:bob', - createdAt: new Date().toISOString(), - } - const uri = AtUri.make(did, coll, TID.nextStr()) - const cid = await putBlock(tx, did, follow) - await services.record(tx).indexRecord(uri, cid, follow) - uris.push(uri) - } - }) - - let count = await countRecords(db, 'follow') - expect(count).toBe(1) - - await db.transaction(async (tx) => { - await services.record(tx).deleteRecord(uris[0], false) - }) - - count = await countRecords(db, 'follow') - expect(count).toBe(1) - - await db.transaction(async (tx) => { - await services.record(tx).deleteRecord(uris[1], true) - }) - - count = await countRecords(db, 'follow') - expect(count).toBe(0) - }) -}) diff --git a/packages/pds/tests/labeler/apply-labels.test.ts b/packages/pds/tests/labeler/apply-labels.test.ts deleted file mode 100644 index 80fedc459d5..00000000000 --- a/packages/pds/tests/labeler/apply-labels.test.ts +++ /dev/null @@ -1,180 +0,0 @@ -import AtpAgent from '@atproto/api' -import { TestNetworkNoAppView } from '@atproto/dev-env' -import { adminAuth, moderatorAuth } from '../_util' -import { SeedClient } from '../seeds/client' -import basicSeed from '../seeds/basic' - -describe('unspecced.applyLabels', () => { - let network: TestNetworkNoAppView - let agent: AtpAgent - let sc: SeedClient - - beforeAll(async () => { - network = await TestNetworkNoAppView.create({ - dbPostgresSchema: 'apply_labels', - }) - agent = network.pds.getClient() - sc = new SeedClient(agent) - await basicSeed(sc) - }) - - afterAll(async () => { - await network.close() - }) - - it('requires admin auth.', async () => { - const tryToLabel = agent.api.app.bsky.unspecced.applyLabels( - { - labels: [ - { - src: network.pds.ctx.cfg.labelerDid, - uri: sc.dids.carol, - val: 'cats', - neg: false, - cts: new Date().toISOString(), - }, - ], - }, - { - encoding: 'application/json', - headers: { authorization: moderatorAuth() }, - }, - ) - await expect(tryToLabel).rejects.toThrow('Insufficient privileges') - }) - - it('adds and removes labels on record as though applied by the labeler.', async () => { - const post = sc.posts[sc.dids.bob][1].ref - await agent.api.app.bsky.unspecced.applyLabels( - { - labels: [ - { - src: network.pds.ctx.cfg.labelerDid, - uri: post.uriStr, - cid: post.cidStr, - val: 'birds', - neg: false, - cts: new Date().toISOString(), - }, - { - src: network.pds.ctx.cfg.labelerDid, - uri: post.uriStr, - cid: post.cidStr, - val: 'bats', - neg: false, - cts: new Date().toISOString(), - }, - ], - }, - { - encoding: 'application/json', - headers: { authorization: adminAuth() }, - }, - ) - await expect(getRecordLabels(post.uriStr)).resolves.toEqual([ - 'birds', - 'bats', - ]) - await agent.api.app.bsky.unspecced.applyLabels( - { - labels: [ - { - src: network.pds.ctx.cfg.labelerDid, - uri: post.uriStr, - cid: post.cidStr, - val: 'birds', - neg: true, - cts: new Date().toISOString(), - }, - { - src: network.pds.ctx.cfg.labelerDid, - uri: post.uriStr, - cid: post.cidStr, - val: 'bats', - neg: true, - cts: new Date().toISOString(), - }, - ], - }, - { - encoding: 'application/json', - headers: { authorization: adminAuth() }, - }, - ) - await expect(getRecordLabels(post.uriStr)).resolves.toEqual([]) - }) - - it('adds and removes labels on repo as though applied by the labeler.', async () => { - await agent.api.app.bsky.unspecced.applyLabels( - { - labels: [ - { - src: network.pds.ctx.cfg.labelerDid, - uri: sc.dids.carol, - val: 'birds', - neg: false, - cts: new Date().toISOString(), - }, - { - src: network.pds.ctx.cfg.labelerDid, - uri: sc.dids.carol, - val: 'bats', - neg: false, - cts: new Date().toISOString(), - }, - ], - }, - { - encoding: 'application/json', - headers: { authorization: adminAuth() }, - }, - ) - await expect(getRepoLabels(sc.dids.carol)).resolves.toEqual([ - 'birds', - 'bats', - ]) - await agent.api.app.bsky.unspecced.applyLabels( - { - labels: [ - { - src: network.pds.ctx.cfg.labelerDid, - uri: sc.dids.carol, - val: 'birds', - neg: true, - cts: new Date().toISOString(), - }, - { - src: network.pds.ctx.cfg.labelerDid, - uri: sc.dids.carol, - val: 'bats', - neg: true, - cts: new Date().toISOString(), - }, - ], - }, - { - encoding: 'application/json', - headers: { authorization: adminAuth() }, - }, - ) - await expect(getRepoLabels(sc.dids.carol)).resolves.toEqual([]) - }) - - async function getRecordLabels(uri: string) { - const result = await agent.api.com.atproto.admin.getRecord( - { uri }, - { headers: { authorization: adminAuth() } }, - ) - const labels = result.data.labels ?? [] - return labels.map((l) => l.val) - } - - async function getRepoLabels(did: string) { - const result = await agent.api.com.atproto.admin.getRepo( - { did }, - { headers: { authorization: adminAuth() } }, - ) - const labels = result.data.labels ?? [] - return labels.map((l) => l.val) - } -}) diff --git a/packages/pds/tests/labeler/fixtures/hiveai_resp_example.json b/packages/pds/tests/labeler/fixtures/hiveai_resp_example.json deleted file mode 100644 index 2315fa9d0c0..00000000000 --- a/packages/pds/tests/labeler/fixtures/hiveai_resp_example.json +++ /dev/null @@ -1,401 +0,0 @@ -{ - "id": "02122580-c37f-11ed-81d2-000000000000", - "code": 200, - "project_id": 12345, - "user_id": 12345, - "created_on": "2023-03-15T22:16:18.408Z", - "status": [ - { - "status": { - "code": "0", - "message": "SUCCESS" - }, - "response": { - "input": { - "id": "02122580-c37f-11ed-81d2-000000000000", - "charge": 0.003, - "model": "mod55_dense", - "model_version": 1, - "model_type": "CATEGORIZATION", - "created_on": "2023-03-15T22:16:18.136Z", - "media": { - "url": null, - "filename": "bafkreiam7k6mvkyuoybq4ynhljvj5xa75sdbhjbolzjf5j2udx7vj5gnsy", - "type": "PHOTO", - "mime_type": "jpeg", - "mimetype": "image/jpeg", - "width": 800, - "height": 800, - "num_frames": 1, - "duration": 0 - }, - "user_id": 12345, - "project_id": 12345, - "config_version": 1, - "config_tag": "default" - }, - "output": [ - { - "time": 0, - "classes": [ - { - "class": "general_not_nsfw_not_suggestive", - "score": 0.9998097218132356 - }, - { - "class": "general_nsfw", - "score": 8.857344804177162e-5 - }, - { - "class": "general_suggestive", - "score": 0.00010170473872266839 - }, - { - "class": "no_female_underwear", - "score": 0.9999923079040384 - }, - { - "class": "yes_female_underwear", - "score": 7.692095961599136e-6 - }, - { - "class": "no_male_underwear", - "score": 0.9999984904867634 - }, - { - "class": "yes_male_underwear", - "score": 1.5095132367094679e-6 - }, - { - "class": "no_sex_toy", - "score": 0.9999970970762551 - }, - { - "class": "yes_sex_toy", - "score": 2.9029237450490604e-6 - }, - { - "class": "no_female_nudity", - "score": 0.9999739028909301 - }, - { - "class": "yes_female_nudity", - "score": 2.60971090699536e-5 - }, - { - "class": "no_male_nudity", - "score": 0.9999711373083747 - }, - { - "class": "yes_male_nudity", - "score": 2.8862691625255323e-5 - }, - { - "class": "no_female_swimwear", - "score": 0.9999917609899659 - }, - { - "class": "yes_female_swimwear", - "score": 8.239010034025379e-6 - }, - { - "class": "no_male_shirtless", - "score": 0.9999583350744331 - }, - { - "class": "yes_male_shirtless", - "score": 4.166492556688088e-5 - }, - { - "class": "no_text", - "score": 0.9958378716447616 - }, - { - "class": "text", - "score": 0.0041621283552384265 - }, - { - "class": "animated", - "score": 0.46755478950048235 - }, - { - "class": "hybrid", - "score": 0.0011440363434524984 - }, - { - "class": "natural", - "score": 0.5313011741560651 - }, - { - "class": "animated_gun", - "score": 2.0713000782979496e-5 - }, - { - "class": "gun_in_hand", - "score": 1.5844730446534659e-6 - }, - { - "class": "gun_not_in_hand", - "score": 1.0338973818006654e-6 - }, - { - "class": "no_gun", - "score": 0.9999766686287906 - }, - { - "class": "culinary_knife_in_hand", - "score": 3.8063500083369785e-6 - }, - { - "class": "culinary_knife_not_in_hand", - "score": 7.94057948996249e-7 - }, - { - "class": "knife_in_hand", - "score": 4.5578955723278505e-7 - }, - { - "class": "knife_not_in_hand", - "score": 3.842124714748908e-7 - }, - { - "class": "no_knife", - "score": 0.999994559590014 - }, - { - "class": "a_little_bloody", - "score": 2.1317745626539786e-7 - }, - { - "class": "no_blood", - "score": 0.9999793341236429 - }, - { - "class": "other_blood", - "score": 2.0322054269591763e-5 - }, - { - "class": "very_bloody", - "score": 1.306446309561673e-7 - }, - { - "class": "no_pills", - "score": 0.9999989592376954 - }, - { - "class": "yes_pills", - "score": 1.0407623044588633e-6 - }, - { - "class": "no_smoking", - "score": 0.9999939101969173 - }, - { - "class": "yes_smoking", - "score": 6.089803082758281e-6 - }, - { - "class": "illicit_injectables", - "score": 6.925695592003094e-7 - }, - { - "class": "medical_injectables", - "score": 8.587808234452378e-7 - }, - { - "class": "no_injectables", - "score": 0.9999984486496174 - }, - { - "class": "no_nazi", - "score": 0.9999987449628097 - }, - { - "class": "yes_nazi", - "score": 1.2550371902234279e-6 - }, - { - "class": "no_kkk", - "score": 0.999999762417549 - }, - { - "class": "yes_kkk", - "score": 2.3758245111050425e-7 - }, - { - "class": "no_middle_finger", - "score": 0.9999881515231847 - }, - { - "class": "yes_middle_finger", - "score": 1.184847681536747e-5 - }, - { - "class": "no_terrorist", - "score": 0.9999998870793229 - }, - { - "class": "yes_terrorist", - "score": 1.1292067715380635e-7 - }, - { - "class": "no_overlay_text", - "score": 0.9996453363440359 - }, - { - "class": "yes_overlay_text", - "score": 0.0003546636559640924 - }, - { - "class": "no_sexual_activity", - "score": 0.9999563580374798 - }, - { - "class": "yes_sexual_activity", - "score": 0.99, - "realScore": 4.364196252012032e-5 - }, - { - "class": "hanging", - "score": 3.6435135762510905e-7 - }, - { - "class": "no_hanging_no_noose", - "score": 0.9999980779196416 - }, - { - "class": "noose", - "score": 1.5577290007796094e-6 - }, - { - "class": "no_realistic_nsfw", - "score": 0.9999944341007805 - }, - { - "class": "yes_realistic_nsfw", - "score": 5.565899219571182e-6 - }, - { - "class": "animated_corpse", - "score": 5.276802046755426e-7 - }, - { - "class": "human_corpse", - "score": 2.5449360984211012e-8 - }, - { - "class": "no_corpse", - "score": 0.9999994468704343 - }, - { - "class": "no_self_harm", - "score": 0.9999994515625507 - }, - { - "class": "yes_self_harm", - "score": 5.484374493605692e-7 - }, - { - "class": "no_drawing", - "score": 0.9978276028816608 - }, - { - "class": "yes_drawing", - "score": 0.0021723971183392485 - }, - { - "class": "no_emaciated_body", - "score": 0.9999998146500432 - }, - { - "class": "yes_emaciated_body", - "score": 1.853499568724518e-7 - }, - { - "class": "no_child_present", - "score": 0.9999970498515446 - }, - { - "class": "yes_child_present", - "score": 2.950148455380443e-6 - }, - { - "class": "no_sexual_intent", - "score": 0.9999963861546292 - }, - { - "class": "yes_sexual_intent", - "score": 3.613845370766111e-6 - }, - { - "class": "animal_genitalia_and_human", - "score": 2.255472023465222e-8 - }, - { - "class": "animal_genitalia_only", - "score": 4.6783185199931176e-7 - }, - { - "class": "animated_animal_genitalia", - "score": 6.707857419436447e-7 - }, - { - "class": "no_animal_genitalia", - "score": 0.9999988388276858 - }, - { - "class": "no_gambling", - "score": 0.9999960939687145 - }, - { - "class": "yes_gambling", - "score": 3.906031285604864e-6 - }, - { - "class": "no_undressed", - "score": 0.99999923356218 - }, - { - "class": "yes_undressed", - "score": 7.664378199789045e-7 - }, - { - "class": "no_confederate", - "score": 0.9999925456900376 - }, - { - "class": "yes_confederate", - "score": 7.454309962453175e-6 - }, - { - "class": "animated_alcohol", - "score": 1.8109949948066074e-6 - }, - { - "class": "no_alcohol", - "score": 0.9999916620957963 - }, - { - "class": "yes_alcohol", - "score": 5.88781463445443e-6 - }, - { - "class": "yes_drinking_alcohol", - "score": 6.390945746578106e-7 - }, - { - "class": "no_religious_icon", - "score": 0.9999862158580689 - }, - { - "class": "yes_religious_icon", - "score": 1.3784141931119298e-5 - } - ] - } - ] - } - } - ], - "from_cache": false -} diff --git a/packages/pds/tests/labeler/hive.test.ts b/packages/pds/tests/labeler/hive.test.ts deleted file mode 100644 index 3213d794e30..00000000000 --- a/packages/pds/tests/labeler/hive.test.ts +++ /dev/null @@ -1,16 +0,0 @@ -import fs from 'fs/promises' -import * as hive from '../../src/labeler/hive' - -describe('labeling', () => { - it('correctly parses hive responses', async () => { - const exampleRespBytes = await fs.readFile( - 'tests/labeler/fixtures/hiveai_resp_example.json', - ) - const exampleResp = JSON.parse(exampleRespBytes.toString()) - const classes = hive.respToClasses(exampleResp) - expect(classes.length).toBeGreaterThan(10) - - const labels = hive.summarizeLabels(classes) - expect(labels).toEqual(['porn']) - }) -}) diff --git a/packages/pds/tests/labeler/labeler.test.ts b/packages/pds/tests/labeler/labeler.test.ts deleted file mode 100644 index 646e5d621d9..00000000000 --- a/packages/pds/tests/labeler/labeler.test.ts +++ /dev/null @@ -1,178 +0,0 @@ -import { AtUri, BlobRef } from '@atproto/api' -import { runTestServer, CloseFn, TestServerInfo } from '../_util' -import { Labeler } from '../../src/labeler' -import { AppContext, Database } from '../../src' -import { BlobStore, cidForRecord } from '@atproto/repo' -import { keywordLabeling } from '../../src/labeler/util' -import { cidForCbor, TID } from '@atproto/common' -import { LabelService } from '../../src/app-view/services/label' -import { BackgroundQueue } from '../../src/event-stream/background-queue' -import { CID } from 'multiformats/cid' - -// outside of test suite so that TestLabeler can access them -let badCid1: CID | undefined = undefined -let badCid2: CID | undefined = undefined - -describe('labeler', () => { - let server: TestServerInfo - let close: CloseFn - let labeler: Labeler - let labelSrvc: LabelService - let ctx: AppContext - let labelerDid: string - let badBlob1: BlobRef - let badBlob2: BlobRef - let goodBlob: BlobRef - - beforeAll(async () => { - server = await runTestServer({ - dbPostgresSchema: 'labeler', - }) - close = server.close - ctx = server.ctx - labelerDid = ctx.cfg.labelerDid - labeler = new TestLabeler({ - db: ctx.db, - blobstore: ctx.blobstore, - backgroundQueue: ctx.backgroundQueue, - labelerDid, - keywords: { label_me: 'test-label', another_label: 'another-label' }, - }) - labelSrvc = ctx.services.appView.label(ctx.db) - const bytes1 = new Uint8Array([1, 2, 3, 4]) - const bytes2 = new Uint8Array([5, 6, 7, 8]) - const bytes3 = new Uint8Array([4, 3, 2, 1]) - const cid1 = await cidForCbor(bytes1) - const cid2 = await cidForCbor(bytes2) - const cid3 = await cidForCbor(bytes3) - ctx.blobstore.putPermanent(cid1, bytes1) - ctx.blobstore.putPermanent(cid2, bytes2) - ctx.blobstore.putPermanent(cid3, bytes3) - badBlob1 = new BlobRef(cid1, 'image/jpeg', 4) - badBlob2 = new BlobRef(cid2, 'image/jpeg', 4) - goodBlob = new BlobRef(cid3, 'image/jpeg', 4) - badCid1 = badBlob1.ref - badCid2 = badBlob2.ref - }) - - afterAll(async () => { - await close() - }) - - it('labels text in posts', async () => { - const post = { - $type: 'app.bsky.feed.post', - text: 'blah blah label_me', - createdAt: new Date().toISOString(), - } - const cid = await cidForRecord(post) - const uri = postUri() - labeler.processRecord(uri, post) - await labeler.processAll() - await server.processAll() - const labels = await labelSrvc.getLabels(uri.toString()) - expect(labels.length).toBe(1) - expect(labels[0]).toMatchObject({ - src: labelerDid, - uri: uri.toString(), - cid: cid.toString(), - val: 'test-label', - neg: false, - }) - }) - - it('labels embeds in posts', async () => { - const post = { - $type: 'app.bsky.feed.post', - text: 'blah blah', - embed: { - $type: 'app.bsky.embed.images', - images: [ - { - image: badBlob1, - alt: 'img', - }, - { - image: badBlob2, - alt: 'another_label', - }, - { - image: goodBlob, - alt: 'img', - }, - ], - }, - createdAt: new Date().toISOString(), - } - const uri = postUri() - labeler.processRecord(uri, post) - await labeler.processAll() - await server.processAll() - const dbLabels = await labelSrvc.getLabels(uri.toString()) - const labels = dbLabels.map((row) => row.val).sort() - expect(labels).toEqual( - ['another-label', 'img-label', 'other-img-label'].sort(), - ) - }) - - it('retrieves repo labels on profile views', async () => { - await ctx.db.db - .insertInto('label') - .values({ - src: labelerDid, - uri: aliceDid, - cid: '', - val: 'repo-label', - neg: 0, - cts: new Date().toISOString(), - }) - .execute() - await server.processAll() - - const labels = await labelSrvc.getLabelsForProfile('did:example:alice') - // 4 from earlier & then just added one - expect(labels.length).toBe(1) - expect(labels[0]).toMatchObject({ - src: labelerDid, - uri: aliceDid, - val: 'repo-label', - neg: false, - }) - }) -}) - -const aliceDid = 'did:example:alice' - -const postUri = () => AtUri.make(aliceDid, 'app.bsky.feed.post', TID.nextStr()) - -class TestLabeler extends Labeler { - hiveApiKey: string - keywords: Record - - constructor(opts: { - db: Database - blobstore: BlobStore - backgroundQueue: BackgroundQueue - labelerDid: string - keywords: Record - }) { - const { db, blobstore, backgroundQueue, labelerDid, keywords } = opts - super({ db, blobstore, backgroundQueue, labelerDid }) - this.keywords = keywords - } - - async labelText(text: string): Promise { - return keywordLabeling(this.keywords, text) - } - - async labelImg(cid: CID): Promise { - if (cid.equals(badCid1)) { - return ['img-label'] - } - - if (cid.equals(badCid2)) { - return ['other-img-label'] - } - return [] - } -} diff --git a/packages/pds/tests/moderation.test.ts b/packages/pds/tests/moderation.test.ts index 34e52a156a0..edbb23c6578 100644 --- a/packages/pds/tests/moderation.test.ts +++ b/packages/pds/tests/moderation.test.ts @@ -1,4 +1,4 @@ -import AtpAgent, { ComAtprotoAdminTakeModerationAction } from '@atproto/api' +import AtpAgent from '@atproto/api' import { AtUri } from '@atproto/syntax' import { BlobNotFoundError } from '@atproto/repo' import { @@ -827,164 +827,6 @@ describe('moderation', () => { ) }) - it('negates an existing label and reverses.', async () => { - const { ctx } = server - const post = sc.posts[sc.dids.bob][0].ref - const labelingService = ctx.services.appView.label(ctx.db) - await labelingService.formatAndCreate( - ctx.cfg.labelerDid, - post.uriStr, - post.cidStr, - { create: ['kittens'] }, - ) - const action = await actionWithLabels({ - negateLabelVals: ['kittens'], - subject: { - $type: 'com.atproto.repo.strongRef', - uri: post.uriStr, - cid: post.cidStr, - }, - }) - await expect(getRecordLabels(post.uriStr)).resolves.toEqual([]) - await reverse(action.id) - await expect(getRecordLabels(post.uriStr)).resolves.toEqual(['kittens']) - // Cleanup - await labelingService.formatAndCreate( - ctx.cfg.labelerDid, - post.uriStr, - post.cidStr, - { negate: ['kittens'] }, - ) - }) - - it('no-ops when negating an already-negated label and reverses.', async () => { - const { ctx } = server - const post = sc.posts[sc.dids.bob][0].ref - const labelingService = ctx.services.appView.label(ctx.db) - const action = await actionWithLabels({ - negateLabelVals: ['bears'], - subject: { - $type: 'com.atproto.repo.strongRef', - uri: post.uriStr, - cid: post.cidStr, - }, - }) - await expect(getRecordLabels(post.uriStr)).resolves.toEqual([]) - await reverse(action.id) - await expect(getRecordLabels(post.uriStr)).resolves.toEqual(['bears']) - // Cleanup - await labelingService.formatAndCreate( - ctx.cfg.labelerDid, - post.uriStr, - post.cidStr, - { negate: ['bears'] }, - ) - }) - - it('creates non-existing labels and reverses.', async () => { - const post = sc.posts[sc.dids.bob][0].ref - const action = await actionWithLabels({ - createLabelVals: ['puppies', 'doggies'], - negateLabelVals: [], - subject: { - $type: 'com.atproto.repo.strongRef', - uri: post.uriStr, - cid: post.cidStr, - }, - }) - await expect(getRecordLabels(post.uriStr)).resolves.toEqual([ - 'puppies', - 'doggies', - ]) - await reverse(action.id) - await expect(getRecordLabels(post.uriStr)).resolves.toEqual([]) - }) - - it('no-ops when creating an existing label and reverses.', async () => { - const { ctx } = server - const post = sc.posts[sc.dids.bob][0].ref - const labelingService = ctx.services.appView.label(ctx.db) - await labelingService.formatAndCreate( - ctx.cfg.labelerDid, - post.uriStr, - post.cidStr, - { create: ['birds'] }, - ) - const action = await actionWithLabels({ - createLabelVals: ['birds'], - subject: { - $type: 'com.atproto.repo.strongRef', - uri: post.uriStr, - cid: post.cidStr, - }, - }) - await expect(getRecordLabels(post.uriStr)).resolves.toEqual(['birds']) - await reverse(action.id) - await expect(getRecordLabels(post.uriStr)).resolves.toEqual([]) - }) - - it('creates labels on a repo and reverses.', async () => { - const action = await actionWithLabels({ - createLabelVals: ['puppies', 'doggies'], - negateLabelVals: [], - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, - }) - await expect(getRepoLabels(sc.dids.bob)).resolves.toEqual([ - 'puppies', - 'doggies', - ]) - await reverse(action.id) - await expect(getRepoLabels(sc.dids.bob)).resolves.toEqual([]) - }) - - it('creates and negates labels on a repo and reverses.', async () => { - const { ctx } = server - const labelingService = ctx.services.appView.label(ctx.db) - await labelingService.formatAndCreate( - ctx.cfg.labelerDid, - sc.dids.bob, - null, - { create: ['kittens'] }, - ) - const action = await actionWithLabels({ - createLabelVals: ['puppies'], - negateLabelVals: ['kittens'], - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, - }) - await expect(getRepoLabels(sc.dids.bob)).resolves.toEqual(['puppies']) - await reverse(action.id) - await expect(getRepoLabels(sc.dids.bob)).resolves.toEqual(['kittens']) - }) - - it('does not allow triage moderators to label.', async () => { - const attemptLabel = agent.api.com.atproto.admin.takeModerationAction( - { - action: ACKNOWLEDGE, - createdBy: 'did:example:moderator', - reason: 'Y', - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: sc.dids.bob, - }, - negateLabelVals: ['a'], - createLabelVals: ['b', 'c'], - }, - { - encoding: 'application/json', - headers: { authorization: triageAuth() }, - }, - ) - await expect(attemptLabel).rejects.toThrow( - 'Must be a full moderator to label content', - ) - }) - it('allows full moderators to takedown.', async () => { const { data: action } = await agent.api.com.atproto.admin.takeModerationAction( @@ -1017,7 +859,6 @@ describe('moderation', () => { $type: 'com.atproto.admin.defs#repoRef', did: sc.dids.bob, }, - createLabelVals: ['takendown'], // Use negative value to set the expiry time in the past so that the action is automatically reversed // right away without having to wait n number of hours for a successful assertion durationInHours: -1, @@ -1028,8 +869,6 @@ describe('moderation', () => { }, ) - const labelsAfterTakedown = await getRepoLabels(sc.dids.bob) - expect(labelsAfterTakedown).toContain('takendown') // In the actual app, this will be instantiated and run on server startup const periodicReversal = new PeriodicModerationActionReversal(server.ctx) await periodicReversal.findAndRevertDueActions() @@ -1046,10 +885,6 @@ describe('moderation', () => { createdBy: action.createdBy, reason: '[SCHEDULED_REVERSAL] Reverting action as originally scheduled', }) - - // Verify that labels are also reversed when takedown action is reversed - const labelsAfterReversal = await getRepoLabels(sc.dids.bob) - expect(labelsAfterReversal).not.toContain('takendown') }) it('does not allow non-full moderators to takedown.', async () => { @@ -1074,26 +909,6 @@ describe('moderation', () => { ) }) - async function actionWithLabels( - opts: Partial & { - subject: ComAtprotoAdminTakeModerationAction.InputSchema['subject'] - }, - ) { - const result = await agent.api.com.atproto.admin.takeModerationAction( - { - action: FLAG, - createdBy: 'did:example:admin', - reason: 'Y', - ...opts, - }, - { - encoding: 'application/json', - headers: { authorization: adminAuth() }, - }, - ) - return result.data - } - async function reverse(actionId: number) { await agent.api.com.atproto.admin.reverseModerationAction( { @@ -1107,24 +922,6 @@ describe('moderation', () => { }, ) } - - async function getRecordLabels(uri: string) { - const result = await agent.api.com.atproto.admin.getRecord( - { uri }, - { headers: { authorization: adminAuth() } }, - ) - const labels = result.data.labels ?? [] - return labels.map((l) => l.val) - } - - async function getRepoLabels(did: string) { - const result = await agent.api.com.atproto.admin.getRepo( - { did }, - { headers: { authorization: adminAuth() } }, - ) - const labels = result.data.labels ?? [] - return labels.map((l) => l.val) - } }) describe('blob takedown', () => { diff --git a/packages/pds/tests/proxied/admin.test.ts b/packages/pds/tests/proxied/admin.test.ts index 127717225a6..98ec8f1f70c 100644 --- a/packages/pds/tests/proxied/admin.test.ts +++ b/packages/pds/tests/proxied/admin.test.ts @@ -24,7 +24,6 @@ describe('proxies admin requests', () => { dbPostgresSchema: 'proxy_admin', pds: { // @NOTE requires admin pass be the same on pds and appview, which TestNetwork is handling for us. - enableInProcessAppView: true, bskyAppViewModeration: true, inviteRequired: true, }, @@ -246,7 +245,7 @@ describe('proxies admin requests', () => { }) it('takesdown and labels repos, and reverts.', async () => { - const { db, services } = network.pds.ctx + const { db, services } = network.bsky.ctx // takedown repo const { data: action } = await agent.api.com.atproto.admin.takeModerationAction( @@ -276,8 +275,8 @@ describe('proxies admin requests', () => { await expect(tryGetProfileAppview).rejects.toThrow( 'Account has been taken down', ) - const labelsA = await services.appView - .label(db) + const labelsA = await services + .label(db.getPrimary()) .getLabels(sc.dids.alice, { includeNeg: false, skipCache: true }) expect(labelsA.map((l) => l.val)).toEqual(['dogs']) // reverse action @@ -298,14 +297,14 @@ describe('proxies admin requests', () => { expect(profileAppview).toEqual( expect.objectContaining({ did: sc.dids.alice, handle: 'alice.test' }), ) - const labelsB = await services.appView - .label(db) + const labelsB = await services + .label(db.getPrimary()) .getLabels(sc.dids.alice, { includeNeg: false, skipCache: true }) expect(labelsB.map((l) => l.val)).toEqual(['cats']) }) it('takesdown and labels records, and reverts.', async () => { - const { db, services } = network.pds.ctx + const { db, services } = network.bsky.ctx const post = sc.posts[sc.dids.alice][0] // takedown post const { data: action } = @@ -335,8 +334,8 @@ describe('proxies admin requests', () => { }, ) await expect(tryGetPostAppview).rejects.toThrow(NotFoundError) - const labelsA = await services.appView - .label(db) + const labelsA = await services + .label(db.getPrimary()) .getLabels(post.ref.uriStr, { includeNeg: false, skipCache: true }) expect(labelsA.map((l) => l.val)).toEqual(['dogs']) // reverse action @@ -357,8 +356,8 @@ describe('proxies admin requests', () => { expect(threadAppview.thread.post).toEqual( expect.objectContaining({ uri: post.ref.uriStr, cid: post.ref.cidStr }), ) - const labelsB = await services.appView - .label(db) + const labelsB = await services + .label(db.getPrimary()) .getLabels(post.ref.uriStr, { includeNeg: false, skipCache: true }) expect(labelsB.map((l) => l.val)).toEqual(['cats']) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 843e3acd27e..fd9ef8cdbad 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -213,9 +213,6 @@ importers: ioredis: specifier: ^5.3.2 version: 5.3.2 - iso-datestring-validator: - specifier: ^2.2.2 - version: 2.2.2 kysely: specifier: ^0.22.0 version: 0.22.0 @@ -292,6 +289,9 @@ importers: cbor-x: specifier: ^1.5.1 version: 1.5.1 + iso-datestring-validator: + specifier: ^2.2.2 + version: 2.2.2 multiformats: specifier: ^9.9.0 version: 9.9.0 @@ -531,9 +531,6 @@ importers: ioredis: specifier: ^5.3.2 version: 5.3.2 - iso-datestring-validator: - specifier: ^2.2.2 - version: 2.2.2 jsonwebtoken: specifier: ^8.5.1 version: 8.5.1