Disable pds appview indexing (#1645)

* rm indexing service * remove message queue & refactor background queue * wip * remove all canProxyReadc * finish cleanup * clean up tests * fix up tests * fix api tests * fix build * fix compression test * update image tests * fix dev envs * build branch * wip - removing labeler * fix service file * remove kysely tables * re-enable getPopular * format * cleaning up tests * rm unused sharp code * rm pds build * clean up tests * fix build * fix build * migration * tidy * build branch * tidy * build branch * small tidy * dont build
bluesky-social · Sep 26, 2023 · 1db3d72 · 1db3d72
1 parent f8a9946
commit 1db3d72
Show file tree

Hide file tree

Showing 112 changed files with 177 additions and 5,496 deletions.
diff --git a/lexicons/app/bsky/unspecced/applyLabels.json b/lexicons/app/bsky/unspecced/applyLabels.json
diff --git a/packages/api/src/client/index.ts b/packages/api/src/client/index.ts
@@ -133,7 +133,6 @@ import * as AppBskyNotificationListNotifications from './types/app/bsky/notifica
 import * as AppBskyNotificationRegisterPush from './types/app/bsky/notification/registerPush'
 import * as AppBskyNotificationUpdateSeen from './types/app/bsky/notification/updateSeen'
 import * as AppBskyRichtextFacet from './types/app/bsky/richtext/facet'
-import * as AppBskyUnspeccedApplyLabels from './types/app/bsky/unspecced/applyLabels'
 import * as AppBskyUnspeccedDefs from './types/app/bsky/unspecced/defs'
 import * as AppBskyUnspeccedGetPopular from './types/app/bsky/unspecced/getPopular'
 import * as AppBskyUnspeccedGetPopularFeedGenerators from './types/app/bsky/unspecced/getPopularFeedGenerators'
@@ -267,7 +266,6 @@ export * as AppBskyNotificationListNotifications from './types/app/bsky/notifica
 export * as AppBskyNotificationRegisterPush from './types/app/bsky/notification/registerPush'
 export * as AppBskyNotificationUpdateSeen from './types/app/bsky/notification/updateSeen'
 export * as AppBskyRichtextFacet from './types/app/bsky/richtext/facet'
-export * as AppBskyUnspeccedApplyLabels from './types/app/bsky/unspecced/applyLabels'
 export * as AppBskyUnspeccedDefs from './types/app/bsky/unspecced/defs'
 export * as AppBskyUnspeccedGetPopular from './types/app/bsky/unspecced/getPopular'
 export * as AppBskyUnspeccedGetPopularFeedGenerators from './types/app/bsky/unspecced/getPopularFeedGenerators'
@@ -2305,17 +2303,6 @@ export class UnspeccedNS {
     this._service = service
   }
 
-  applyLabels(
-    data?: AppBskyUnspeccedApplyLabels.InputSchema,
-    opts?: AppBskyUnspeccedApplyLabels.CallOptions,
-  ): Promise<AppBskyUnspeccedApplyLabels.Response> {
-    return this._service.xrpc
-      .call('app.bsky.unspecced.applyLabels', opts?.qp, data, opts)
-      .catch((e) => {
-        throw AppBskyUnspeccedApplyLabels.toKnownErr(e)
-      })
-  }
-
   getPopular(
     params?: AppBskyUnspeccedGetPopular.QueryParams,
     opts?: AppBskyUnspeccedGetPopular.CallOptions,

diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts
@@ -7022,32 +7022,6 @@ export const schemaDict = {
       },
     },
   },
-  AppBskyUnspeccedApplyLabels: {
-    lexicon: 1,
-    id: 'app.bsky.unspecced.applyLabels',
-    defs: {
-      main: {
-        type: 'procedure',
-        description: 'Allow a labeler to apply labels directly.',
-        input: {
-          encoding: 'application/json',
-          schema: {
-            type: 'object',
-            required: ['labels'],
-            properties: {
-              labels: {
-                type: 'array',
-                items: {
-                  type: 'ref',
-                  ref: 'lex:com.atproto.label.defs#label',
-                },
-              },
-            },
-          },
-        },
-      },
-    },
-  },
   AppBskyUnspeccedDefs: {
     lexicon: 1,
     id: 'app.bsky.unspecced.defs',
@@ -7481,7 +7455,6 @@ export const ids = {
   AppBskyNotificationRegisterPush: 'app.bsky.notification.registerPush',
   AppBskyNotificationUpdateSeen: 'app.bsky.notification.updateSeen',
   AppBskyRichtextFacet: 'app.bsky.richtext.facet',
-  AppBskyUnspeccedApplyLabels: 'app.bsky.unspecced.applyLabels',
   AppBskyUnspeccedDefs: 'app.bsky.unspecced.defs',
   AppBskyUnspeccedGetPopular: 'app.bsky.unspecced.getPopular',
   AppBskyUnspeccedGetPopularFeedGenerators:

diff --git a/packages/api/src/client/types/app/bsky/unspecced/applyLabels.ts b/packages/api/src/client/types/app/bsky/unspecced/applyLabels.ts
diff --git a/packages/bsky/package.json b/packages/bsky/package.json
@@ -50,7 +50,6 @@
     "http-errors": "^2.0.0",
     "http-terminator": "^3.2.0",
     "ioredis": "^5.3.2",
-    "iso-datestring-validator": "^2.2.2",
     "kysely": "^0.22.0",
     "multiformats": "^9.9.0",
     "p-queue": "^6.6.2",

diff --git a/packages/bsky/src/auto-moderator/index.ts b/packages/bsky/src/auto-moderator/index.ts
@@ -280,28 +280,12 @@ export class AutoModerator {
   async storeLabels(uri: AtUri, cid: CID, labels: string[]): Promise<void> {
     if (labels.length < 1) return
     const labelSrvc = this.services.label(this.ctx.db)
-    const formatted = await labelSrvc.formatAndCreate(
+    await labelSrvc.formatAndCreate(
       this.ctx.cfg.labelerDid,
       uri.toString(),
       cid.toString(),
       { create: labels },
     )
-    if (this.pushAgent) {
-      const agent = this.pushAgent
-      try {
-        await agent.api.app.bsky.unspecced.applyLabels({ labels: formatted })
-      } catch (err) {
-        log.error(
-          {
-            err,
-            uri: uri.toString(),
-            labels,
-            receiver: agent.service.toString(),
-          },
-          'failed to push labels',
-        )
-      }
-    }
   }
 
   async processAll() {

diff --git a/packages/bsky/src/lexicon/index.ts b/packages/bsky/src/lexicon/index.ts
@@ -111,7 +111,6 @@ import * as AppBskyNotificationGetUnreadCount from './types/app/bsky/notificatio
 import * as AppBskyNotificationListNotifications from './types/app/bsky/notification/listNotifications'
 import * as AppBskyNotificationRegisterPush from './types/app/bsky/notification/registerPush'
 import * as AppBskyNotificationUpdateSeen from './types/app/bsky/notification/updateSeen'
-import * as AppBskyUnspeccedApplyLabels from './types/app/bsky/unspecced/applyLabels'
 import * as AppBskyUnspeccedGetPopular from './types/app/bsky/unspecced/getPopular'
 import * as AppBskyUnspeccedGetPopularFeedGenerators from './types/app/bsky/unspecced/getPopularFeedGenerators'
 import * as AppBskyUnspeccedGetTimelineSkeleton from './types/app/bsky/unspecced/getTimelineSkeleton'
@@ -1450,17 +1449,6 @@ export class UnspeccedNS {
     this._server = server
   }
 
-  applyLabels<AV extends AuthVerifier>(
-    cfg: ConfigOf<
-      AV,
-      AppBskyUnspeccedApplyLabels.Handler<ExtractAuth<AV>>,
-      AppBskyUnspeccedApplyLabels.HandlerReqCtx<ExtractAuth<AV>>
-    >,
-  ) {
-    const nsid = 'app.bsky.unspecced.applyLabels' // @ts-ignore
-    return this._server.xrpc.method(nsid, cfg)
-  }
-
   getPopular<AV extends AuthVerifier>(
     cfg: ConfigOf<
       AV,

diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts
@@ -7022,32 +7022,6 @@ export const schemaDict = {
       },
     },
   },
-  AppBskyUnspeccedApplyLabels: {
-    lexicon: 1,
-    id: 'app.bsky.unspecced.applyLabels',
-    defs: {
-      main: {
-        type: 'procedure',
-        description: 'Allow a labeler to apply labels directly.',
-        input: {
-          encoding: 'application/json',
-          schema: {
-            type: 'object',
-            required: ['labels'],
-            properties: {
-              labels: {
-                type: 'array',
-                items: {
-                  type: 'ref',
-                  ref: 'lex:com.atproto.label.defs#label',
-                },
-              },
-            },
-          },
-        },
-      },
-    },
-  },
   AppBskyUnspeccedDefs: {
     lexicon: 1,
     id: 'app.bsky.unspecced.defs',
@@ -7481,7 +7455,6 @@ export const ids = {
   AppBskyNotificationRegisterPush: 'app.bsky.notification.registerPush',
   AppBskyNotificationUpdateSeen: 'app.bsky.notification.updateSeen',
   AppBskyRichtextFacet: 'app.bsky.richtext.facet',
-  AppBskyUnspeccedApplyLabels: 'app.bsky.unspecced.applyLabels',
   AppBskyUnspeccedDefs: 'app.bsky.unspecced.defs',
   AppBskyUnspeccedGetPopular: 'app.bsky.unspecced.getPopular',
   AppBskyUnspeccedGetPopularFeedGenerators:

diff --git a/packages/bsky/src/lexicon/types/app/bsky/unspecced/applyLabels.ts b/packages/bsky/src/lexicon/types/app/bsky/unspecced/applyLabels.ts
diff --git a/packages/bsky/src/services/indexing/plugins/block.ts b/packages/bsky/src/services/indexing/plugins/block.ts
@@ -1,11 +1,11 @@
 import { Selectable } from 'kysely'
 import { AtUri } from '@atproto/syntax'
+import { toSimplifiedISOSafe } from '@atproto/common'
 import { CID } from 'multiformats/cid'
 import * as Block from '../../../lexicon/types/app/bsky/graph/block'
 import * as lex from '../../../lexicon/lexicons'
 import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
 import RecordProcessor from '../processor'
-import { toSimplifiedISOSafe } from '../util'
 import { PrimaryDatabase } from '../../../db'
 import { BackgroundQueue } from '../../../background'
 import { NotificationServer } from '../../../notifications'

diff --git a/packages/bsky/src/services/indexing/plugins/feed-generator.ts b/packages/bsky/src/services/indexing/plugins/feed-generator.ts
@@ -1,13 +1,13 @@
 import { Selectable } from 'kysely'
 import { AtUri } from '@atproto/syntax'
+import { toSimplifiedISOSafe } from '@atproto/common'
 import { CID } from 'multiformats/cid'
 import * as FeedGenerator from '../../../lexicon/types/app/bsky/feed/generator'
 import * as lex from '../../../lexicon/lexicons'
 import { PrimaryDatabase } from '../../../db'
 import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
 import { BackgroundQueue } from '../../../background'
 import RecordProcessor from '../processor'
-import { toSimplifiedISOSafe } from '../util'
 import { NotificationServer } from '../../../notifications'
 
 const lexId = lex.ids.AppBskyFeedGenerator

diff --git a/packages/bsky/src/services/indexing/plugins/follow.ts b/packages/bsky/src/services/indexing/plugins/follow.ts
@@ -1,11 +1,11 @@
 import { Selectable } from 'kysely'
 import { AtUri } from '@atproto/syntax'
+import { toSimplifiedISOSafe } from '@atproto/common'
 import { CID } from 'multiformats/cid'
 import * as Follow from '../../../lexicon/types/app/bsky/graph/follow'
 import * as lex from '../../../lexicon/lexicons'
 import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
 import RecordProcessor from '../processor'
-import { toSimplifiedISOSafe } from '../util'
 import { PrimaryDatabase } from '../../../db'
 import { countAll, excluded } from '../../../db/util'
 import { BackgroundQueue } from '../../../background'

diff --git a/packages/bsky/src/services/indexing/plugins/like.ts b/packages/bsky/src/services/indexing/plugins/like.ts
@@ -1,11 +1,11 @@
 import { Selectable } from 'kysely'
 import { AtUri } from '@atproto/syntax'
+import { toSimplifiedISOSafe } from '@atproto/common'
 import { CID } from 'multiformats/cid'
 import * as Like from '../../../lexicon/types/app/bsky/feed/like'
 import * as lex from '../../../lexicon/lexicons'
 import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
 import RecordProcessor from '../processor'
-import { toSimplifiedISOSafe } from '../util'
 import { countAll, excluded } from '../../../db/util'
 import { PrimaryDatabase } from '../../../db'
 import { BackgroundQueue } from '../../../background'

diff --git a/packages/bsky/src/services/indexing/plugins/list-block.ts b/packages/bsky/src/services/indexing/plugins/list-block.ts
@@ -1,5 +1,6 @@
 import { Selectable } from 'kysely'
 import { AtUri } from '@atproto/syntax'
+import { toSimplifiedISOSafe } from '@atproto/common'
 import { CID } from 'multiformats/cid'
 import * as ListBlock from '../../../lexicon/types/app/bsky/graph/listblock'
 import * as lex from '../../../lexicon/lexicons'
@@ -8,7 +9,6 @@ import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
 import RecordProcessor from '../processor'
 import { BackgroundQueue } from '../../../background'
 import { NotificationServer } from '../../../notifications'
-import { toSimplifiedISOSafe } from '../util'
 
 const lexId = lex.ids.AppBskyGraphListblock
 type IndexedListBlock = Selectable<DatabaseSchemaType['list_block']>

diff --git a/packages/bsky/src/services/indexing/plugins/list-item.ts b/packages/bsky/src/services/indexing/plugins/list-item.ts
@@ -1,11 +1,11 @@
 import { Selectable } from 'kysely'
 import { AtUri } from '@atproto/syntax'
+import { toSimplifiedISOSafe } from '@atproto/common'
 import { CID } from 'multiformats/cid'
 import * as ListItem from '../../../lexicon/types/app/bsky/graph/listitem'
 import * as lex from '../../../lexicon/lexicons'
 import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
 import RecordProcessor from '../processor'
-import { toSimplifiedISOSafe } from '../util'
 import { InvalidRequestError } from '@atproto/xrpc-server'
 import { PrimaryDatabase } from '../../../db'
 import { BackgroundQueue } from '../../../background'

diff --git a/packages/bsky/src/services/indexing/plugins/list.ts b/packages/bsky/src/services/indexing/plugins/list.ts
@@ -1,11 +1,11 @@
 import { Selectable } from 'kysely'
 import { AtUri } from '@atproto/syntax'
+import { toSimplifiedISOSafe } from '@atproto/common'
 import { CID } from 'multiformats/cid'
 import * as List from '../../../lexicon/types/app/bsky/graph/list'
 import * as lex from '../../../lexicon/lexicons'
 import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
 import RecordProcessor from '../processor'
-import { toSimplifiedISOSafe } from '../util'
 import { PrimaryDatabase } from '../../../db'
 import { BackgroundQueue } from '../../../background'
 import { NotificationServer } from '../../../notifications'

diff --git a/packages/bsky/src/services/indexing/plugins/post.ts b/packages/bsky/src/services/indexing/plugins/post.ts
@@ -1,6 +1,7 @@
 import { Insertable, Selectable, sql } from 'kysely'
 import { CID } from 'multiformats/cid'
 import { AtUri } from '@atproto/syntax'
+import { toSimplifiedISOSafe } from '@atproto/common'
 import { jsonStringToLex } from '@atproto/lexicon'
 import {
   Record as PostRecord,
@@ -19,7 +20,6 @@ import * as lex from '../../../lexicon/lexicons'
 import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
 import RecordProcessor from '../processor'
 import { Notification } from '../../../db/tables/notification'
-import { toSimplifiedISOSafe } from '../util'
 import { PrimaryDatabase } from '../../../db'
 import { countAll, excluded } from '../../../db/util'
 import { BackgroundQueue } from '../../../background'