Add client-set proxy headers to PDS (#2251)

* tidy bsky auth * hook up new auth verifier * update auth throughout ozone * handle mod signing keys * add client proxy heads to pds * hook up rest of routes * simplify pipethrough & add some SSRF protection * tests * fix bad var * fix key parsing in pds * fix admin auth test * rename test * add pipethrough to write routes * update did doc id values * null creds string -> `none` * fix fetchLabels auth check * ✨ Add a couple more proxied requests that we use in ozone ui * Add runit to the services/bsky Dockerfile (#2254) add runit to the services/bsky Dockerfile * Improve tag detection (#2260) * Allow tags to lead with and contain only numbers * Break tags on other whitespace characters * Export regexes from rich text detection * Add test * Add test * Disallow number-only tags * Avoid combining enclosing screen chars * Allow full-width number sign * Clarify tests * Fix punctuation edge case * Reorder * Simplify, add another test * Another test, comment * Version packages (#2261) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> * 🐛 Increment attempt count after each attempt to push ozone event (#2239) * Ozone delegates email sending to actor's pds (#2272) * ozone delegates email sending to user's pds * lexicon: add content field to mod email event * test email sending via mod event * fix auth verifier method * build branch * fix url check * better error handling for get account infos * fix labeler service id * fix iss on auth headers * fix dev-env ozone did * fix tests & another jwt issuer * ozone: fix ip check * fix aud check on pds mod service auth * tidy * fix pipethrough of headers --------- Co-authored-by: Foysal Ahamed <[email protected]> Co-authored-by: Jake Gold <[email protected]> Co-authored-by: Eric Bailey <[email protected]> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: devin ivy <[email protected]>
bluesky-social · Mar 7, 2024 · 2267f1e · 2267f1e
1 parent 3988543
commit 2267f1e
Show file tree

Hide file tree

Showing 63 changed files with 472 additions and 562 deletions.
diff --git a/.github/workflows/build-and-push-bsky-ghcr.yaml b/.github/workflows/build-and-push-bsky-ghcr.yaml
@@ -3,7 +3,7 @@ on:
   push:
     branches:
       - main
-      - appview-v2
+      - pds-proxy-headers
 env:
   REGISTRY: ghcr.io
   USERNAME: ${{ github.actor }}

diff --git a/.github/workflows/build-and-push-ozone-aws.yaml b/.github/workflows/build-and-push-ozone-aws.yaml
@@ -3,7 +3,7 @@ on:
   push:
     branches:
       - main
-      - ozone-cdn-invalidation
+      - pds-proxy-headers
 env:
   REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }}
   USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }}

diff --git a/.github/workflows/build-and-push-pds-ghcr.yaml b/.github/workflows/build-and-push-pds-ghcr.yaml
@@ -3,6 +3,7 @@ on:
   push:
     branches:
       - main
+      - pds-proxy-headers
 env:
   REGISTRY: ghcr.io
   USERNAME: ${{ github.actor }}

diff --git a/packages/common-web/src/did-doc.ts b/packages/common-web/src/did-doc.ts
@@ -81,7 +81,7 @@ export const getNotifEndpoint = (doc: DidDocument): string | undefined => {
 
 export const getServiceEndpoint = (
   doc: DidDocument,
-  opts: { id: string; type: string },
+  opts: { id: string; type?: string },
 ) => {
   const did = getDid(doc)
   let services = doc.service
@@ -94,7 +94,7 @@ export const getServiceEndpoint = (
     (service) => service.id === opts.id || service.id === `${did}${opts.id}`,
   )
   if (!found) return undefined
-  if (found.type !== opts.type) {
+  if (opts.type && found.type !== opts.type) {
     return undefined
   }
   if (typeof found.serviceEndpoint !== 'string') {

diff --git a/packages/dev-env/src/pds.ts b/packages/dev-env/src/pds.ts
@@ -35,6 +35,7 @@ export class TestPds {
     await fs.mkdir(dataDirectory, { recursive: true })
 
     const env: pds.ServerEnvironment = {
+      devMode: true,
       port,
       dataDirectory: dataDirectory,
       blobstoreDiskLocation: blobstoreLoc,

diff --git a/packages/pds/src/api/app/bsky/actor/getProfile.ts b/packages/pds/src/api/app/bsky/actor/getProfile.ts
@@ -1,6 +1,5 @@
 import { Server } from '../../../../lexicon'
 import AppContext from '../../../../context'
-import { authPassthru } from '../../../proxy'
 import { OutputSchema } from '../../../../lexicon/types/app/bsky/actor/getProfile'
 import {
   LocalViewer,
@@ -15,18 +14,10 @@ export default function (server: Server, ctx: AppContext) {
   const { bskyAppView } = ctx.cfg
   if (!bskyAppView) return
   server.app.bsky.actor.getProfile({
-    auth: ctx.authVerifier.accessOrRole,
-    handler: async ({ req, auth, params }) => {
-      const requester =
-        auth.credentials.type === 'access' ? auth.credentials.did : null
-      const res = await pipethrough(
-        bskyAppView.url,
-        METHOD_NSID,
-        params,
-        requester
-          ? await ctx.appviewAuthHeaders(requester, req)
-          : authPassthru(req),
-      )
+    auth: ctx.authVerifier.access,
+    handler: async ({ req, auth }) => {
+      const requester = auth.credentials.did
+      const res = await pipethrough(ctx, req, requester)
       if (!requester) {
         return res
       }

diff --git a/packages/pds/src/api/app/bsky/actor/getProfiles.ts b/packages/pds/src/api/app/bsky/actor/getProfiles.ts
@@ -15,15 +15,10 @@ export default function (server: Server, ctx: AppContext) {
   if (!bskyAppView) return
   server.app.bsky.actor.getProfiles({
     auth: ctx.authVerifier.access,
-    handler: async ({ auth, params, req }) => {
+    handler: async ({ req, auth }) => {
       const requester = auth.credentials.did
 
-      const res = await pipethrough(
-        bskyAppView.url,
-        METHOD_NSID,
-        params,
-        await ctx.appviewAuthHeaders(requester, req),
-      )
+      const res = await pipethrough(ctx, req, requester)
       return handleReadAfterWrite(
         ctx,
         METHOD_NSID,

diff --git a/packages/pds/src/api/app/bsky/actor/getSuggestions.ts b/packages/pds/src/api/app/bsky/actor/getSuggestions.ts
@@ -7,14 +7,9 @@ export default function (server: Server, ctx: AppContext) {
   if (!bskyAppView) return
   server.app.bsky.actor.getSuggestions({
     auth: ctx.authVerifier.access,
-    handler: async ({ params, auth, req }) => {
+    handler: async ({ req, auth }) => {
       const requester = auth.credentials.did
-      return pipethrough(
-        bskyAppView.url,
-        'app.bsky.actor.getSuggestions',
-        params,
-        await ctx.appviewAuthHeaders(requester, req),
-      )
+      return pipethrough(ctx, req, requester)
     },
   })
 }
diff --git a/packages/pds/src/api/app/bsky/actor/searchActors.ts b/packages/pds/src/api/app/bsky/actor/searchActors.ts
@@ -7,14 +7,9 @@ export default function (server: Server, ctx: AppContext) {
   if (!bskyAppView) return
   server.app.bsky.actor.searchActors({
     auth: ctx.authVerifier.access,
-    handler: async ({ params, auth, req }) => {
+    handler: async ({ req, auth }) => {
       const requester = auth.credentials.did
-      return pipethrough(
-        bskyAppView.url,
-        'app.bsky.actor.searchActors',
-        params,
-        await ctx.appviewAuthHeaders(requester, req),
-      )
+      return pipethrough(ctx, req, requester)
     },
   })
 }
diff --git a/packages/pds/src/api/app/bsky/actor/searchActorsTypeahead.ts b/packages/pds/src/api/app/bsky/actor/searchActorsTypeahead.ts
@@ -7,14 +7,9 @@ export default function (server: Server, ctx: AppContext) {
   if (!bskyAppView) return
   server.app.bsky.actor.searchActorsTypeahead({
     auth: ctx.authVerifier.access,
-    handler: async ({ params, auth, req }) => {
+    handler: async ({ req, auth }) => {
       const requester = auth.credentials.did
-      return pipethrough(
-        bskyAppView.url,
-        'app.bsky.actor.searchActorsTypeahead',
-        params,
-        await ctx.appviewAuthHeaders(requester, req),
-      )
+      return pipethrough(ctx, req, requester)
     },
   })
 }
diff --git a/packages/pds/src/api/app/bsky/feed/getActorFeeds.ts b/packages/pds/src/api/app/bsky/feed/getActorFeeds.ts
@@ -7,14 +7,9 @@ export default function (server: Server, ctx: AppContext) {
   if (!bskyAppView) return
   server.app.bsky.feed.getActorFeeds({
     auth: ctx.authVerifier.access,
-    handler: async ({ auth, params, req }) => {
+    handler: async ({ req, auth }) => {
       const requester = auth.credentials.did
-      return pipethrough(
-        bskyAppView.url,
-        'app.bsky.feed.getActorFeeds',
-        params,
-        await ctx.appviewAuthHeaders(requester, req),
-      )
+      return pipethrough(ctx, req, requester)
     },
   })
 }
diff --git a/packages/pds/src/api/app/bsky/feed/getActorLikes.ts b/packages/pds/src/api/app/bsky/feed/getActorLikes.ts
@@ -1,6 +1,5 @@
 import { Server } from '../../../../lexicon'
 import AppContext from '../../../../context'
-import { authPassthru } from '../../../proxy'
 import { OutputSchema } from '../../../../lexicon/types/app/bsky/feed/getAuthorFeed'
 import {
   LocalViewer,
@@ -15,18 +14,10 @@ export default function (server: Server, ctx: AppContext) {
   const { bskyAppView } = ctx.cfg
   if (!bskyAppView) return
   server.app.bsky.feed.getActorLikes({
-    auth: ctx.authVerifier.accessOrRole,
-    handler: async ({ req, params, auth }) => {
-      const requester =
-        auth.credentials.type === 'access' ? auth.credentials.did : null
-      const res = await pipethrough(
-        bskyAppView.url,
-        METHOD_NSID,
-        params,
-        requester
-          ? await ctx.appviewAuthHeaders(requester, req)
-          : authPassthru(req),
-      )
+    auth: ctx.authVerifier.access,
+    handler: async ({ req, auth }) => {
+      const requester = auth.credentials.did
+      const res = await pipethrough(ctx, req, requester)
 
       if (!requester) {
         return res

diff --git a/packages/pds/src/api/app/bsky/feed/getAuthorFeed.ts b/packages/pds/src/api/app/bsky/feed/getAuthorFeed.ts
@@ -1,6 +1,5 @@
 import { Server } from '../../../../lexicon'
 import AppContext from '../../../../context'
-import { authPassthru } from '../../../proxy'
 import { OutputSchema } from '../../../../lexicon/types/app/bsky/feed/getAuthorFeed'
 import { isReasonRepost } from '../../../../lexicon/types/app/bsky/feed/defs'
 import {
@@ -16,18 +15,10 @@ export default function (server: Server, ctx: AppContext) {
   const { bskyAppView } = ctx.cfg
   if (!bskyAppView) return
   server.app.bsky.feed.getAuthorFeed({
-    auth: ctx.authVerifier.accessOrRole,
-    handler: async ({ req, params, auth }) => {
-      const requester =
-        auth.credentials.type === 'access' ? auth.credentials.did : null
-      const res = await pipethrough(
-        bskyAppView.url,
-        METHOD_NSID,
-        params,
-        requester
-          ? await ctx.appviewAuthHeaders(requester, req)
-          : authPassthru(req),
-      )
+    auth: ctx.authVerifier.access,
+    handler: async ({ req, auth }) => {
+      const requester = auth.credentials.did
+      const res = await pipethrough(ctx, req, requester)
       if (!requester) {
         return res
       }

diff --git a/packages/pds/src/api/app/bsky/feed/getFeed.ts b/packages/pds/src/api/app/bsky/feed/getFeed.ts
@@ -14,22 +14,9 @@ export default function (server: Server, ctx: AppContext) {
       const { data: feed } =
         await appViewAgent.api.app.bsky.feed.getFeedGenerator(
           { feed: params.feed },
-          await ctx.appviewAuthHeaders(requester, req),
+          await ctx.appviewAuthHeaders(requester),
         )
-      const serviceAuthHeaders = await ctx.serviceAuthHeaders(
-        requester,
-        feed.view.did,
-        req,
-      )
-      // forward accept-language header to upstream services
-      serviceAuthHeaders.headers['accept-language'] =
-        req.headers['accept-language']
-      return pipethrough(
-        bskyAppView.url,
-        'app.bsky.feed.getFeed',
-        params,
-        serviceAuthHeaders,
-      )
+      return pipethrough(ctx, req, requester, feed.view.did)
     },
   })
 }
diff --git a/packages/pds/src/api/app/bsky/feed/getFeedGenerator.ts b/packages/pds/src/api/app/bsky/feed/getFeedGenerator.ts
@@ -7,14 +7,9 @@ export default function (server: Server, ctx: AppContext) {
   if (!bskyAppView) return
   server.app.bsky.feed.getFeedGenerator({
     auth: ctx.authVerifier.access,
-    handler: async ({ params, auth, req }) => {
+    handler: async ({ req, auth }) => {
       const requester = auth.credentials.did
-      return pipethrough(
-        bskyAppView.url,
-        'app.bsky.feed.getFeedGenerator',
-        params,
-        await ctx.appviewAuthHeaders(requester, req),
-      )
+      return pipethrough(ctx, req, requester)
     },
   })
 }
diff --git a/packages/pds/src/api/app/bsky/feed/getFeedGenerators.ts b/packages/pds/src/api/app/bsky/feed/getFeedGenerators.ts
@@ -7,14 +7,9 @@ export default function (server: Server, ctx: AppContext) {
   if (!bskyAppView) return
   server.app.bsky.feed.getFeedGenerators({
     auth: ctx.authVerifier.access,
-    handler: async ({ params, auth, req }) => {
+    handler: async ({ req, auth }) => {
       const requester = auth.credentials.did
-      return pipethrough(
-        bskyAppView.url,
-        'app.bsky.feed.getFeedGenerators',
-        params,
-        await ctx.appviewAuthHeaders(requester, req),
-      )
+      return pipethrough(ctx, req, requester)
     },
   })
 }
diff --git a/packages/pds/src/api/app/bsky/feed/getLikes.ts b/packages/pds/src/api/app/bsky/feed/getLikes.ts
@@ -7,14 +7,9 @@ export default function (server: Server, ctx: AppContext) {
   if (!bskyAppView) return
   server.app.bsky.feed.getLikes({
     auth: ctx.authVerifier.access,
-    handler: async ({ params, auth, req }) => {
+    handler: async ({ req, auth }) => {
       const requester = auth.credentials.did
-      return pipethrough(
-        bskyAppView.url,
-        'app.bsky.feed.getLikes',
-        params,
-        await ctx.appviewAuthHeaders(requester, req),
-      )
+      return pipethrough(ctx, req, requester)
     },
   })
 }
diff --git a/packages/pds/src/api/app/bsky/feed/getListFeed.ts b/packages/pds/src/api/app/bsky/feed/getListFeed.ts
@@ -7,14 +7,9 @@ export default function (server: Server, ctx: AppContext) {
   if (!bskyAppView) return
   server.app.bsky.feed.getListFeed({
     auth: ctx.authVerifier.access,
-    handler: async ({ auth, params, req }) => {
+    handler: async ({ req, auth }) => {
       const requester = auth.credentials.did
-      return pipethrough(
-        bskyAppView.url,
-        'app.bsky.feed.getListFeed',
-        params,
-        await ctx.appviewAuthHeaders(requester, req),
-      )
+      return pipethrough(ctx, req, requester)
     },
   })
 }
diff --git a/packages/pds/src/api/app/bsky/feed/getPostThread.ts b/packages/pds/src/api/app/bsky/feed/getPostThread.ts
@@ -3,7 +3,6 @@ import { AtUri } from '@atproto/syntax'
 import { Headers, XRPCError } from '@atproto/xrpc'
 import { Server } from '../../../../lexicon'
 import AppContext from '../../../../context'
-import { authPassthru } from '../../../proxy'
 import {
   ThreadViewPost,
   isThreadViewPost,
@@ -30,27 +29,12 @@ export default function (server: Server, ctx: AppContext) {
   const { bskyAppView } = ctx.cfg
   if (!bskyAppView) return
   server.app.bsky.feed.getPostThread({
-    auth: ctx.authVerifier.accessOrRole,
-    handler: async ({ req, params, auth }) => {
-      const requester =
-        auth.credentials.type === 'access' ? auth.credentials.did : null
-
-      if (!requester) {
-        return pipethrough(
-          bskyAppView.url,
-          METHOD_NSID,
-          params,
-          authPassthru(req),
-        )
-      }
+    auth: ctx.authVerifier.access,
+    handler: async ({ req, auth, params }) => {
+      const requester = auth.credentials.did
 
       try {
-        const res = await pipethrough(
-          bskyAppView.url,
-          METHOD_NSID,
-          params,
-          await ctx.appviewAuthHeaders(requester, req),
-        )
+        const res = await pipethrough(ctx, req, requester)
 
         return await handleReadAfterWrite(
           ctx,
@@ -206,7 +190,7 @@ const readAfterWriteNotFound = async (
       assert(ctx.appViewAgent)
       const parentsRes = await ctx.appViewAgent.api.app.bsky.feed.getPostThread(
         { uri: highestParent, parentHeight: params.parentHeight, depth: 0 },
-        await ctx.appviewAuthHeaders(requester, null),
+        await ctx.appviewAuthHeaders(requester),
       )
       thread.parent = parentsRes.data.thread
     } catch (err) {

diff --git a/packages/pds/src/api/app/bsky/feed/getPosts.ts b/packages/pds/src/api/app/bsky/feed/getPosts.ts
@@ -7,14 +7,9 @@ export default function (server: Server, ctx: AppContext) {
   if (!bskyAppView) return
   server.app.bsky.feed.getPosts({
     auth: ctx.authVerifier.access,
-    handler: async ({ params, auth, req }) => {
+    handler: async ({ req, auth }) => {
       const requester = auth.credentials.did
-      return pipethrough(
-        bskyAppView.url,
-        'app.bsky.feed.getPosts',
-        params,
-        await ctx.appviewAuthHeaders(requester, req),
-      )
+      return pipethrough(ctx, req, requester)
     },
   })
 }