Support accounts on entryway PDS without a local repo

packages/pds/src/api/com/atproto/server/createAccount.ts

@@ -157,7 +157,7 @@ export const ensureCodeIsAvailable = async (
     .selectAll()
     .whereNotExists((qb) =>
       qb
-        .selectFrom('repo_root')
+        .selectFrom('user_account')
         .selectAll()
         .where('takedownId', 'is not', null)
         .whereRef('did', '=', ref('invite_code.forUser')),

packages/pds/src/api/com/atproto/sync/getBlob.ts

@@ -11,10 +11,20 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.optionalAccessOrRoleVerifier,
     handler: async ({ params, res, auth }) => {
       const { ref } = ctx.db.db.dynamic
+      const { did } = params
+
+      if (!isUserOrAdmin(auth, did)) {
+        const available = await ctx.services
+          .account(ctx.db)
+          .isRepoAvailable(did)
+        if (!available) {
+          throw new InvalidRequestError(`Could not find repo for DID: ${did}`)
+        }
+      }
+
       const found = await ctx.db.db
         .selectFrom('blob')
         .selectAll()
-        .innerJoin('repo_root', 'repo_root.did', 'blob.creator')
         .innerJoin('repo_blob', (join) =>
           join
             .onRef('repo_blob.cid', '=', 'blob.cid')
@@ -23,10 +33,6 @@ export default function (server: Server, ctx: AppContext) {
         .where('blob.cid', '=', params.cid)
         .where('blob.creator', '=', params.did)
         .where(notSoftDeletedClause(ref('repo_blob')))
-        .if(!isUserOrAdmin(auth, params.did), (qb) =>
-          // takedown check for anyone other than an admin or the user
-          qb.where(notSoftDeletedClause(ref('repo_root'))),
-        )
         .executeTakeFirst()
       if (!found) {
         throw new InvalidRequestError('Blob not found')

packages/pds/src/api/com/atproto/sync/listRepos.ts

@@ -8,10 +8,11 @@ export default function (server: Server, ctx: AppContext) {
   server.com.atproto.sync.listRepos(async ({ params }) => {
     const { limit, cursor } = params
     const { ref } = ctx.db.db.dynamic
+    // @NOTE this join will become sparse as accounts move off, pagination will become relatively expensive.
     let builder = ctx.db.db
       .selectFrom('user_account')
       .innerJoin('repo_root', 'repo_root.did', 'user_account.did')
-      .where(notSoftDeletedClause(ref('repo_root')))
+      .where(notSoftDeletedClause(ref('user_account')))
       .select([
         'user_account.did as did',
         'repo_root.root as head',

packages/pds/src/db/migrations/20231004T040354739Z-user-account-pds.ts

@@ -18,8 +18,52 @@ export async function up(db: Kysely<unknown>, dialect: Dialect): Promise<void> {
     .alterTable('user_account')
     .addColumn('pdsId', 'integer', (col) => col.references('pds.id'))
     .execute()
+  await db.schema
+    .alterTable('user_account')
+    .addColumn('takedownId', 'integer')
+    .execute()
+  const migrationDb = db as Kysely<MigrationSchema>
+  const { ref } = migrationDb.dynamic
+  await migrationDb
+    .updateTable('user_account')
+    .where(
+      'did',
+      'in',
+      migrationDb
+        .selectFrom('repo_root')
+        .select('repo_root.did')
+        .where('takedownId', 'is not', null),
+    )
+    .set({
+      takedownId: migrationDb
+        .selectFrom('repo_root')
+        .select('repo_root.takedownId')
+        .whereRef('did', '=', ref('user_account.did')),
+    })
+    .execute()
+  // when running manually, ensure to drop column only after it's completely out of use in read path
+  await db.schema.alterTable('repo_root').dropColumn('takedownId').execute()
 }
+
 export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .alterTable('repo_root')
+    .addColumn('takedownId', 'integer')
+    .execute()
+  // @NOTE no data migration for takedownId here
+  await db.schema.alterTable('user_account').dropColumn('takedownId').execute()
   await db.schema.alterTable('user_account').dropColumn('pdsId').execute()
   await db.schema.dropTable('pds').execute()
 }
+
+type MigrationSchema = { repo_root: RepoRoot; user_account: UserAccount }
+
+interface RepoRoot {
+  did: string
+  takedownId: number | null
+}
+
+interface UserAccount {
+  did: string
+  takedownId: number | null
+}

packages/pds/src/db/tables/repo-root.ts

@@ -4,7 +4,6 @@ export interface RepoRoot {
   root: string
   rev: string | null
   indexedAt: string
-  takedownId: number | null
 }
 
 export const tableName = 'repo_root'

packages/pds/src/db/tables/user-account.ts

@@ -9,6 +9,7 @@ export interface UserAccount {
   invitesDisabled: Generated<0 | 1>
   inviteNote: string | null
   pdsId: number | null
+  takedownId: number | null
 }
 
 export type UserAccountEntry = Selectable<UserAccount>

packages/pds/src/db/types.ts

@@ -1,3 +1,5 @@
 import { DynamicReferenceBuilder } from 'kysely/dist/cjs/dynamic/dynamic-reference-builder'
 
 export type Ref = DynamicReferenceBuilder<any>
+
+export type OptionalJoin<T> = { [key in keyof T]: T[key] | null }

packages/pds/src/services/account/index.ts

@@ -14,6 +14,7 @@ import * as sequencer from '../../sequencer'
 import { AppPassword } from '../../lexicon/types/com/atproto/server/createAppPassword'
 import { EmailTokenPurpose } from '../../db/tables/email-token'
 import { getRandomToken } from '../../api/com/atproto/server/util'
+import { OptionalJoin } from '../../db/types'
 
 export class AccountService {
   constructor(public db: Database) {}
@@ -31,10 +32,10 @@ export class AccountService {
     const result = await this.db.db
       .selectFrom('user_account')
       .innerJoin('did_handle', 'did_handle.did', 'user_account.did')
-      .innerJoin('repo_root', 'repo_root.did', 'did_handle.did')
+      .leftJoin('repo_root', 'repo_root.did', 'did_handle.did')
       .leftJoin('pds', 'pds.id', 'user_account.pdsId')
       .if(!includeSoftDeleted, (qb) =>
-        qb.where(notSoftDeletedClause(ref('repo_root'))),
+        qb.where(notSoftDeletedClause(ref('user_account'))),
       )
       .where((qb) => {
         if (handleOrDid.startsWith('did:')) {
@@ -60,10 +61,11 @@ export class AccountService {
   // Repo exists and is not taken-down
   async isRepoAvailable(did: string) {
     const found = await this.db.db
-      .selectFrom('repo_root')
-      .where('did', '=', did)
-      .where('takedownId', 'is', null)
-      .select('did')
+      .selectFrom('user_account')
+      .innerJoin('repo_root', 'repo_root.did', 'user_account.did')
+      .where('user_account.did', '=', did)
+      .where('user_account.takedownId', 'is', null)
+      .select('user_account.did')
       .executeTakeFirst()
     return found !== undefined
   }
@@ -76,10 +78,10 @@ export class AccountService {
     const found = await this.db.db
       .selectFrom('user_account')
       .innerJoin('did_handle', 'did_handle.did', 'user_account.did')
-      .innerJoin('repo_root', 'repo_root.did', 'did_handle.did')
+      .leftJoin('repo_root', 'repo_root.did', 'did_handle.did')
       .leftJoin('pds', 'pds.id', 'user_account.pdsId')
       .if(!includeSoftDeleted, (qb) =>
-        qb.where(notSoftDeletedClause(ref('repo_root'))),
+        qb.where(notSoftDeletedClause(ref('user_account'))),
       )
       .where('email', '=', email.toLowerCase())
       .select(['pds.did as pdsDid'])
@@ -104,9 +106,9 @@ export class AccountService {
     const { ref } = this.db.db.dynamic
     const found = await this.db.db
       .selectFrom('did_handle')
-      .innerJoin('repo_root', 'repo_root.did', 'did_handle.did')
+      .innerJoin('user_account', 'user_account.did', 'did_handle.did')
       .if(!includeSoftDeleted, (qb) =>
-        qb.where(notSoftDeletedClause(ref('repo_root'))),
+        qb.where(notSoftDeletedClause(ref('user_account'))),
       )
       .where('handle', '=', handleOrDid)
       .select('did_handle.did')
@@ -283,6 +285,7 @@ export class AccountService {
       .execute()
   }
 
+  // @NOTE only searches active repos, not all accounts.
   async search(opts: {
     query: string
     limit: number
@@ -297,7 +300,7 @@ export class AccountService {
       .innerJoin('repo_root', 'repo_root.did', 'did_handle.did')
       .innerJoin('user_account', 'user_account.did', 'did_handle.did')
       .if(!includeSoftDeleted, (qb) =>
-        qb.where(notSoftDeletedClause(ref('repo_root'))),
+        qb.where(notSoftDeletedClause(ref('user_account'))),
       )
       .where((qb) => {
         // sqlite doesn't support "ilike", but performs "like" case-insensitively
@@ -324,6 +327,7 @@ export class AccountService {
     }).execute()
   }
 
+  // @NOTE only searches active repos, not all accounts.
   async list(opts: {
     limit: number
     cursor?: string
@@ -336,8 +340,9 @@ export class AccountService {
     let builder = this.db.db
       .selectFrom('repo_root')
       .innerJoin('did_handle', 'did_handle.did', 'repo_root.did')
+      .innerJoin('user_account', 'user_account.did', 'repo_root.did')
       .if(!includeSoftDeleted, (qb) =>
-        qb.where(notSoftDeletedClause(ref('repo_root'))),
+        qb.where(notSoftDeletedClause(ref('user_account'))),
       )
       .selectAll('did_handle')
       .selectAll('repo_root')
@@ -642,4 +647,4 @@ export type HandleSequenceToken = { did: string; handle: string }
 
 type AccountInfo = UserAccountEntry &
   DidHandle &
-  RepoRoot & { pdsDid: string | null }
+  OptionalJoin<RepoRoot> & { pdsDid: string | null }

packages/pds/src/services/moderation/index.ts

@@ -436,7 +436,7 @@ export class ModerationService {
 
   async takedownRepo(info: { takedownId: number; did: string }) {
     await this.db.db
-      .updateTable('repo_root')
+      .updateTable('user_account')
       .set({ takedownId: info.takedownId })
       .where('did', '=', info.did)
       .where('takedownId', 'is', null)
@@ -445,7 +445,7 @@ export class ModerationService {
 
   async reverseTakedownRepo(info: { did: string }) {
     await this.db.db
-      .updateTable('repo_root')
+      .updateTable('user_account')
       .set({ takedownId: null })
       .where('did', '=', info.did)
       .execute()

packages/pds/src/services/moderation/views.ts

@@ -21,6 +21,7 @@ import { AccountService } from '../account'
 import { RecordService } from '../record'
 import { ModerationReportRowWithHandle } from '.'
 import { ids } from '../../lexicon/lexicons'
+import { OptionalJoin } from '../../db/types'
 
 export class ModerationViews {
   constructor(private db: Database) {}
@@ -41,29 +42,29 @@ export class ModerationViews {
 
     const [info, actionResults, invitedBy] = await Promise.all([
       await this.db.db
-        .selectFrom('did_handle')
-        .leftJoin('user_account', 'user_account.did', 'did_handle.did')
+        .selectFrom('user_account')
         .leftJoin('record as profile_record', (join) =>
           join
-            .onRef('profile_record.did', '=', 'did_handle.did')
+            .onRef('profile_record.did', '=', 'user_account.did')
             .on('profile_record.collection', '=', ids.AppBskyActorProfile)
             .on('profile_record.rkey', '=', 'self'),
         )
         .leftJoin('ipld_block as profile_block', (join) =>
           join
             .onRef('profile_block.cid', '=', 'profile_record.cid')
-            .onRef('profile_block.creator', '=', 'did_handle.did'),
+            .onRef('profile_block.creator', '=', 'user_account.did'),
         )
         .where(
-          'did_handle.did',
+          'user_account.did',
           'in',
           results.map((r) => r.did),
         )
         .select([
-          'did_handle.did as did',
+          'user_account.did as did',
           'user_account.email as email',
           'user_account.invitesDisabled as invitesDisabled',
           'user_account.inviteNote as inviteNote',
+          'user_account.createdAt',
           'profile_block.content as profileBytes',
         ])
         .execute(),
@@ -93,7 +94,7 @@ export class ModerationViews {
     )
 
     const views = results.map((r) => {
-      const { email, invitesDisabled, profileBytes, inviteNote } =
+      const { email, invitesDisabled, profileBytes, inviteNote, createdAt } =
         infoByDid[r.did] ?? {}
       const action = actionByDid[r.did]
       const relatedRecords: object[] = []
@@ -105,7 +106,7 @@ export class ModerationViews {
         handle: r.handle,
         email: opts.includeEmails && email ? email : undefined,
         relatedRecords,
-        indexedAt: r.indexedAt,
+        indexedAt: r.indexedAt ?? createdAt,
         moderation: {
           currentAction: action
             ? {
@@ -605,7 +606,7 @@ export class ModerationViews {
   }
 }
 
-type RepoResult = DidHandle & RepoRoot
+type RepoResult = DidHandle & OptionalJoin<RepoRoot>
 
 type ActionResult = Selectable<ModerationAction>
 

packages/pds/tests/db.test.ts

@@ -52,7 +52,6 @@ describe('db', () => {
         root: 'x',
         rev: 'x',
         indexedAt: 'bad-date',
-        takedownId: null,
       })
     })