bluesky-social · dholms · Oct 2, 2023 · Jun 12, 2023 · Jun 12, 2023 · Jun 12, 2023
diff --git a/.github/workflows/build-and-push-bsky-aws.yaml b/.github/workflows/build-and-push-bsky-aws.yaml
@@ -3,6 +3,7 @@ on:
   push:
     branches:
       - main
+      - simplify-pds
 env:
   REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }}
   USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }}

diff --git a/.github/workflows/build-and-push-bsky-ghcr.yaml b/.github/workflows/build-and-push-bsky-ghcr.yaml
@@ -3,6 +3,7 @@ on:
   push:
     branches:
       - main
+      - simplify-pds
 env:
   REGISTRY: ghcr.io
   USERNAME: ${{ github.actor }}

diff --git a/package.json b/package.json
@@ -41,6 +41,7 @@
     "eslint": "^8.24.0",
     "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-prettier": "^4.2.1",
+    "get-port": "^6.1.2",
     "jest": "^28.1.2",
     "node-gyp": "^9.3.1",
     "pino-pretty": "^9.1.0",

diff --git a/packages/bsky/tests/auto-moderator/takedowns.test.ts b/packages/bsky/tests/auto-moderator/takedowns.test.ts
@@ -99,7 +99,7 @@ describe('takedowner', () => {
       .where('uri', '=', post.ref.uriStr)
       .select('takedownId')
       .executeTakeFirst()
-    expect(recordPds?.takedownId).toEqual(modAction.id)
+    expect(recordPds?.takedownId).toEqual(modAction.id.toString())
 
     expect(testInvalidator.invalidated.length).toBe(1)
     expect(testInvalidator.invalidated[0].subject).toBe(
@@ -141,7 +141,7 @@ describe('takedowner', () => {
       .where('uri', '=', res.data.uri)
       .select('takedownId')
       .executeTakeFirst()
-    expect(recordPds?.takedownId).toEqual(modAction.id)
+    expect(recordPds?.takedownId).toEqual(modAction.id.toString())
 
     expect(testInvalidator.invalidated.length).toBe(2)
     expect(testInvalidator.invalidated[1].subject).toBe(

diff --git a/packages/common/src/logger.ts b/packages/common/src/logger.ts
@@ -7,7 +7,7 @@ const enabledSystems = (process.env.LOG_SYSTEMS || '')
 
 const enabledEnv = process.env.LOG_ENABLED
 const enabled =
-  enabledEnv === 'true' || enabledEnv === 't' || enabledEnv === '1'
+  enabledEnv !== 'false' && enabledEnv !== 'f' && enabledEnv !== '0'
 
 const level = process.env.LOG_LEVEL || 'info'
 

diff --git a/packages/dev-env/src/bin.ts b/packages/dev-env/src/bin.ts
@@ -15,7 +15,7 @@ const run = async () => {
   const network = await TestNetwork.create({
     pds: {
       port: 2583,
-      publicUrl: 'http://localhost:2583',
+      hostname: 'localhost',
       dbPostgresSchema: 'pds',
     },
     bsky: {

diff --git a/packages/dev-env/src/network-no-appview.ts b/packages/dev-env/src/network-no-appview.ts
@@ -15,12 +15,16 @@ export class TestNetworkNoAppView {
     const dbPostgresUrl = params.dbPostgresUrl || process.env.DB_POSTGRES_URL
     const dbPostgresSchema =
       params.dbPostgresSchema || process.env.DB_POSTGRES_SCHEMA
+    const dbSqliteLocation =
+      dbPostgresUrl === undefined ? ':memory:' : undefined
 
     const plc = await TestPlc.create(params.plc ?? {})
     const pds = await TestPds.create({
       dbPostgresUrl,
       dbPostgresSchema,
-      plcUrl: plc.url,
+      dbPostgresPoolSize: 5,
+      dbSqliteLocation,
+      didPlcUrl: plc.url,
       ...params.pds,
     })
 

diff --git a/packages/dev-env/src/network.ts b/packages/dev-env/src/network.ts
@@ -46,8 +46,9 @@ export class TestNetwork extends TestNetworkNoAppView {
       port: pdsPort,
       dbPostgresUrl,
       dbPostgresSchema,
-      plcUrl: plc.url,
-      bskyAppViewEndpoint: bsky.url,
+      dbPostgresPoolSize: 5,
+      didPlcUrl: plc.url,
+      bskyAppViewUrl: bsky.url,
       bskyAppViewDid: bsky.ctx.cfg.serverDid,
       bskyAppViewModeration: true,
       ...params.pds,

diff --git a/packages/dev-env/src/pds.ts b/packages/dev-env/src/pds.ts
@@ -1,88 +1,70 @@
+import path from 'node:path'
+import os from 'node:os'
 import getPort from 'get-port'
 import * as ui8 from 'uint8arrays'
 import * as pds from '@atproto/pds'
 import { Secp256k1Keypair, randomStr } from '@atproto/crypto'
 import { AtpAgent } from '@atproto/api'
-import { Client as PlcClient } from '@did-plc/lib'
-import { DAY, HOUR } from '@atproto/common-web'
 import { PdsConfig } from './types'
 import { uniqueLockId } from './util'
 
+const ADMIN_PASSWORD = 'admin-pass'
+const MOD_PASSWORD = 'mod-pass'
+const TRIAGE_PASSWORD = 'triage-pass'
+
 export class TestPds {
   constructor(
     public url: string,
     public port: number,
     public server: pds.PDS,
   ) {}
 
-  static async create(cfg: PdsConfig): Promise<TestPds> {
-    const repoSigningKey = await Secp256k1Keypair.create()
-    const plcRotationKey = await Secp256k1Keypair.create()
-    const recoveryKey = await Secp256k1Keypair.create()
+  static async create(config: PdsConfig): Promise<TestPds> {
+    const repoSigningKey = await Secp256k1Keypair.create({ exportable: true })
+    const repoSigningPriv = ui8.toString(await repoSigningKey.export(), 'hex')
+    const plcRotationKey = await Secp256k1Keypair.create({ exportable: true })
+    const plcRotationPriv = ui8.toString(await plcRotationKey.export(), 'hex')
+    const recoveryKey = (await Secp256k1Keypair.create()).did()
 
-    const port = cfg.port || (await getPort())
+    const port = config.port || (await getPort())
     const url = `http://localhost:${port}`
-    const plcClient = new PlcClient(cfg.plcUrl)
-
-    const serverDid = await plcClient.createDid({
-      signingKey: repoSigningKey.did(),
-      rotationKeys: [recoveryKey.did(), plcRotationKey.did()],
-      handle: 'pds.test',
-      pds: `http://localhost:${port}`,
-      signer: plcRotationKey,
-    })
-
-    const config = new pds.ServerConfig({
-      debugMode: true,
-      version: '0.0.0',
-      scheme: 'http',
+
+    const blobstoreLoc = path.join(os.tmpdir(), randomStr(8, 'base32'))
+
+    const env: pds.ServerEnvironment = {
       port,
-      hostname: 'localhost',
-      serverDid,
-      recoveryKey: recoveryKey.did(),
-      adminPassword: 'admin-pass',
-      moderatorPassword: 'moderator-pass',
-      triagePassword: 'triage-pass',
-      inviteRequired: false,
-      userInviteInterval: null,
-      userInviteEpoch: 0,
-      didPlcUrl: cfg.plcUrl,
-      didCacheMaxTTL: DAY,
-      didCacheStaleTTL: HOUR,
+      blobstoreDiskLocation: blobstoreLoc,
+      recoveryDidKey: recoveryKey,
+      adminPassword: ADMIN_PASSWORD,
+      moderatorPassword: MOD_PASSWORD,
+      triagePassword: TRIAGE_PASSWORD,
       jwtSecret: 'jwt-secret',
-      availableUserDomains: ['.test'],
-      rateLimitsEnabled: false,
-      appUrlPasswordReset: 'app://forgot-password',
-      emailNoReplyAddress: '[email protected]',
-      publicUrl: 'https://pds.public.url',
-      dbPostgresUrl: cfg.dbPostgresUrl,
-      maxSubscriptionBuffer: 200,
-      repoBackfillLimitMs: 1000 * 60 * 60, // 1hr
+      serviceHandleDomains: ['.test'],
       sequencerLeaderLockId: uniqueLockId(),
-      dbTxLockNonce: await randomStr(32, 'base32'),
-      bskyAppViewEndpoint: cfg.bskyAppViewEndpoint ?? 'http://fake_address',
-      bskyAppViewDid: cfg.bskyAppViewDid ?? 'did:example:fake',
       bskyAppViewCdnUrlPattern: 'http://cdn.appview.com/%s/%s/%s',
-      ...cfg,
-    })
-
-    const blobstore = new pds.MemoryBlobStore()
-    const db = config.dbPostgresUrl
-      ? pds.Database.postgres({
-          url: config.dbPostgresUrl,
-          schema: config.dbPostgresSchema,
-          txLockNonce: config.dbTxLockNonce,
-        })
-      : pds.Database.memory()
-    await db.migrateToLatestOrThrow()
-
-    const server = pds.PDS.create({
-      db,
-      blobstore,
-      repoSigningKey,
-      plcRotationKey,
-      config,
-    })
+      repoSigningKeyK256PrivateKeyHex: repoSigningPriv,
+      plcRotationKeyK256PrivateKeyHex: plcRotationPriv,
+      inviteRequired: false,
+      ...config,
+    }
+    const cfg = pds.envToCfg(env)
+    const secrets = pds.envToSecrets(env)
+
+    const server = await pds.PDS.create(cfg, secrets)
+
+    // Separate migration db on postgres in case migration changes some
+    // connection state that we need in the tests, e.g. "alter database ... set ..."
+    const migrationDb =
+      cfg.db.dialect === 'pg'
+        ? pds.Database.postgres({
+            url: cfg.db.url,
+            schema: cfg.db.schema,
+          })
+        : server.ctx.db
+    await migrationDb.migrateToLatestOrThrow()
+    if (migrationDb !== server.ctx.db) {
+      await migrationDb.close()
+    }
 
     await server.start()
 
@@ -100,10 +82,10 @@ export class TestPds {
   adminAuth(role: 'admin' | 'moderator' | 'triage' = 'admin'): string {
     const password =
       role === 'triage'
-        ? this.ctx.cfg.triagePassword
+        ? TRIAGE_PASSWORD
         : role === 'moderator'
-        ? this.ctx.cfg.moderatorPassword
-        : this.ctx.cfg.adminPassword
+        ? MOD_PASSWORD
+        : ADMIN_PASSWORD
     return (
       'Basic ' +
       ui8.toString(ui8.fromString(`admin:${password}`, 'utf8'), 'base64pad')

diff --git a/packages/dev-env/src/types.ts b/packages/dev-env/src/types.ts
@@ -7,8 +7,8 @@ export type PlcConfig = {
   version?: string
 }
 
-export type PdsConfig = Partial<pds.ServerConfig> & {
-  plcUrl: string
+export type PdsConfig = Partial<pds.ServerEnvironment> & {
+  didPlcUrl: string
   migration?: string
 }
 
@@ -29,6 +29,6 @@ export type TestServerParams = {
   dbPostgresUrl: string
   dbPostgresSchema: string
   pds: Partial<PdsConfig>
-  plc: Partial<pds.ServerConfig>
+  plc: Partial<PlcConfig>
   bsky: Partial<BskyConfig>
 }
diff --git a/packages/dev-env/src/util.ts b/packages/dev-env/src/util.ts
@@ -21,16 +21,16 @@ export const mockResolvers = (idResolver: IdResolver, pds: TestPds) => {
     const service = result?.service?.find((svc) => svc.id === '#atproto_pds')
     if (typeof service?.serviceEndpoint === 'string') {
       service.serviceEndpoint = service.serviceEndpoint.replace(
-        pds.ctx.cfg.publicUrl,
+        pds.ctx.cfg.service.publicUrl,
         `http://localhost:${pds.port}`,
       )
     }
     return result
   }
 
   idResolver.handle.resolve = async (handle: string) => {
-    const isPdsHandle = pds.ctx.cfg.availableUserDomains.some((domain) =>
-      handle.endsWith(domain),
+    const isPdsHandle = pds.ctx.cfg.identity.serviceHandleDomains.some(
+      (domain) => handle.endsWith(domain),
     )
     if (!isPdsHandle) return undefined
 

diff --git a/packages/pds/.env.example b/packages/pds/.env.example
@@ -0,0 +1,24 @@
+# See more env options in src/config/env.ts
+# Hostname - the public domain that you intend to deploy your service at
+PDS_HOSTNAME="example.com"
+
+# Database config - use one or the other
+PDS_DB_SQLITE_LOCATION="db.test"
+# PDS_DB_POSTGRES_URL="postgresql://pg:password@localhost:5433/postgres"
+
+# Blobstore - filesystem location to store uploaded blobs
+PDS_BLOBSTORE_DISK_LOCATION="blobs"
+
+# Private keys - these are each expected to be 64 char hex strings (256 bit)
+PDS_REPO_SIGNING_KEY_K256_PRIVATE_KEY_HEX="3ee68..."
+PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX="e049f..."
+
+# Secrets - update to secure high-entropy strings
+PDS_JWT_SECRET="jwt-secret"
+PDS_ADMIN_PASSWORD="admin-pass"
+
+# Environment - example is for sandbox
+PDS_DID_PLC_URL="https://plc.bsky-sandbox.dev"
+PDS_BSKY_APP_VIEW_ENDPOINT="https://api.bsky-sandbox.dev"
+PDS_BSKY_APP_VIEW_DID="did:web:api.bsky-sandbox.dev"
+PDS_CRAWLERS="https://bgs.bsky-sandbox.dev"
diff --git a/packages/pds/build.js b/packages/pds/build.js
@@ -6,7 +6,7 @@ const buildShallow =
 
 require('esbuild').build({
   logLevel: 'info',
-  entryPoints: ['src/index.ts', 'src/bin.ts', 'src/db/index.ts'],
+  entryPoints: ['src/index.ts', 'src/db/index.ts'],
   bundle: true,
   sourcemap: true,
   outdir: 'dist',

diff --git a/packages/pds/package.json b/packages/pds/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@atproto/pds",
-  "version": "0.1.19",
+  "version": "0.3.0-beta.3",
   "license": "MIT",
   "description": "Reference implementation of atproto Personal Data Server (PDS)",
   "keywords": [
@@ -23,8 +23,6 @@
     "codegen": "lex gen-server ./src/lexicon ../../lexicons/com/atproto/*/* ../../lexicons/app/bsky/*/*",
     "build": "node ./build.js",
     "postbuild": "tsc --build tsconfig.build.json",
-    "update-main-to-dist": "node ../../update-main-to-dist.js packages/pds",
-    "start": "node dist/bin.js",
     "test": "../dev-infra/with-test-redis-and-db.sh jest",
     "bench": "../dev-infra/with-test-redis-and-db.sh jest --config jest.bench.config.js",
     "test:sqlite": "jest --testPathIgnorePatterns /tests/proxied/*",
@@ -34,6 +32,7 @@
   },
   "dependencies": {
     "@atproto/api": "workspace:^",
+    "@atproto/aws": "workspace:^",
     "@atproto/common": "workspace:^",
     "@atproto/crypto": "workspace:^",
     "@atproto/identity": "workspace:^",

diff --git a/packages/pds/src/api/app/bsky/actor/getProfile.ts b/packages/pds/src/api/app/bsky/actor/getProfile.ts
@@ -11,7 +11,7 @@ export default function (server: Server, ctx: AppContext) {
     handler: async ({ req, auth, params }) => {
       const requester =
         auth.credentials.type === 'access' ? auth.credentials.did : null
-      const res = await ctx.appviewAgent.api.app.bsky.actor.getProfile(
+      const res = await ctx.appViewAgent.api.app.bsky.actor.getProfile(
         params,
         requester ? await ctx.serviceAuthHeaders(requester) : authPassthru(req),
       )

diff --git a/packages/pds/src/api/app/bsky/actor/getProfiles.ts b/packages/pds/src/api/app/bsky/actor/getProfiles.ts
@@ -9,7 +9,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ auth, params }) => {
       const requester = auth.credentials.did
-      const res = await ctx.appviewAgent.api.app.bsky.actor.getProfiles(
+      const res = await ctx.appViewAgent.api.app.bsky.actor.getProfiles(
         params,
         await ctx.serviceAuthHeaders(requester),
       )

diff --git a/packages/pds/src/api/app/bsky/actor/getSuggestions.ts b/packages/pds/src/api/app/bsky/actor/getSuggestions.ts
@@ -1,12 +1,12 @@
-import AppContext from '../../../../context'
 import { Server } from '../../../../lexicon'
+import AppContext from '../../../../context'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.actor.getSuggestions({
     auth: ctx.accessVerifier,
     handler: async ({ params, auth }) => {
       const requester = auth.credentials.did
-      const res = await ctx.appviewAgent.api.app.bsky.actor.getSuggestions(
+      const res = await ctx.appViewAgent.api.app.bsky.actor.getSuggestions(
         params,
         await ctx.serviceAuthHeaders(requester),
       )

diff --git a/packages/pds/src/api/app/bsky/actor/index.ts b/packages/pds/src/api/app/bsky/actor/index.ts
@@ -1,5 +1,6 @@
 import { Server } from '../../../../lexicon'
 import AppContext from '../../../../context'
+
 import getPreferences from './getPreferences'
 import getProfile from './getProfile'
 import getProfiles from './getProfiles'

diff --git a/packages/pds/src/api/app/bsky/actor/searchActors.ts b/packages/pds/src/api/app/bsky/actor/searchActors.ts
@@ -1,12 +1,12 @@
-import AppContext from '../../../../context'
 import { Server } from '../../../../lexicon'
+import AppContext from '../../../../context'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.actor.searchActors({
     auth: ctx.accessVerifier,
-    handler: async ({ auth, params }) => {
+    handler: async ({ params, auth }) => {
       const requester = auth.credentials.did
-      const res = await ctx.appviewAgent.api.app.bsky.actor.searchActors(
+      const res = await ctx.appViewAgent.api.app.bsky.actor.searchActors(
         params,
         await ctx.serviceAuthHeaders(requester),
       )