From 7b146605c9ad051ac7cdec3b2235dec28717501e Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Tue, 12 Mar 2024 16:04:12 -0500 Subject: [PATCH] Updated semantics for atproto labelers header (#2292) * update labelers header semantics * add response header * pds: pipe through res headers * fix up tests * revamp parsing --- packages/bsky/package.json | 1 + .../bsky/src/api/app/bsky/actor/getProfile.ts | 9 ++-- .../src/api/app/bsky/actor/getProfiles.ts | 9 ++-- .../src/api/app/bsky/actor/getSuggestions.ts | 2 + .../src/api/app/bsky/actor/searchActors.ts | 2 + .../app/bsky/actor/searchActorsTypeahead.ts | 2 + .../src/api/app/bsky/feed/getActorFeeds.ts | 3 +- .../src/api/app/bsky/feed/getActorLikes.ts | 9 ++-- .../src/api/app/bsky/feed/getAuthorFeed.ts | 9 ++-- .../bsky/src/api/app/bsky/feed/getFeed.ts | 14 +++--- .../src/api/app/bsky/feed/getFeedGenerator.ts | 2 + .../api/app/bsky/feed/getFeedGenerators.ts | 2 + .../bsky/src/api/app/bsky/feed/getLikes.ts | 3 +- .../bsky/src/api/app/bsky/feed/getListFeed.ts | 6 +-- .../src/api/app/bsky/feed/getPostThread.ts | 11 +++-- .../bsky/src/api/app/bsky/feed/getPosts.ts | 2 + .../src/api/app/bsky/feed/getRepostedBy.ts | 3 +- .../api/app/bsky/feed/getSuggestedFeeds.ts | 2 + .../bsky/src/api/app/bsky/feed/getTimeline.ts | 6 +-- .../bsky/src/api/app/bsky/feed/searchPosts.ts | 2 + .../bsky/src/api/app/bsky/graph/getBlocks.ts | 3 +- .../src/api/app/bsky/graph/getFollowers.ts | 3 +- .../bsky/src/api/app/bsky/graph/getFollows.ts | 3 +- .../bsky/src/api/app/bsky/graph/getList.ts | 3 +- .../src/api/app/bsky/graph/getListBlocks.ts | 3 +- .../src/api/app/bsky/graph/getListMutes.ts | 3 +- .../bsky/src/api/app/bsky/graph/getLists.ts | 3 +- .../bsky/src/api/app/bsky/graph/getMutes.ts | 3 +- .../bsky/graph/getSuggestedFollowsByActor.ts | 2 + .../src/api/app/bsky/labeler/getServices.ts | 2 + .../bsky/notification/listNotifications.ts | 3 +- .../unspecced/getPopularFeedGenerators.ts | 3 +- packages/bsky/src/api/util.ts | 23 ++++++++-- packages/bsky/src/context.ts | 27 +++++++----- packages/bsky/src/hydration/hydrator.ts | 12 +++-- packages/bsky/src/util.ts | 44 +++++++++++++++++++ packages/bsky/tests/label-hydration.test.ts | 17 ++++++- packages/pds/src/pipethrough.ts | 27 ++++++++---- pnpm-lock.yaml | 8 ++++ 39 files changed, 223 insertions(+), 68 deletions(-) create mode 100644 packages/bsky/src/util.ts diff --git a/packages/bsky/package.json b/packages/bsky/package.json index ff1725ca980..9138c16d0be 100644 --- a/packages/bsky/package.json +++ b/packages/bsky/package.json @@ -63,6 +63,7 @@ "pino": "^8.15.0", "pino-http": "^8.2.1", "sharp": "^0.32.6", + "structured-headers": "^1.0.1", "typed-emitter": "^2.1.0", "uint8arrays": "3.0.0" }, diff --git a/packages/bsky/src/api/app/bsky/actor/getProfile.ts b/packages/bsky/src/api/app/bsky/actor/getProfile.ts index b7860791507..f58ecc7dbee 100644 --- a/packages/bsky/src/api/app/bsky/actor/getProfile.ts +++ b/packages/bsky/src/api/app/bsky/actor/getProfile.ts @@ -2,7 +2,7 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import { QueryParams } from '../../../../lexicon/types/app/bsky/actor/getProfile' import AppContext from '../../../../context' -import { setRepoRev } from '../../../util' +import { resHeaders } from '../../../util' import { createPipeline, noRules } from '../../../../pipeline' import { HydrateCtx, @@ -15,7 +15,7 @@ export default function (server: Server, ctx: AppContext) { const getProfile = createPipeline(skeleton, hydration, noRules, presentation) server.app.bsky.actor.getProfile({ auth: ctx.authVerifier.optionalStandardOrRole, - handler: async ({ auth, params, req, res }) => { + handler: async ({ auth, params, req }) => { const { viewer, canViewTakedowns } = ctx.authVerifier.parseCreds(auth) const labelers = ctx.reqLabelers(req) const hydrateCtx = { labelers, viewer } @@ -26,11 +26,14 @@ export default function (server: Server, ctx: AppContext) { ) const repoRev = await ctx.hydrator.actor.getRepoRevSafe(viewer) - setRepoRev(res, repoRev) return { encoding: 'application/json', body: result, + headers: resHeaders({ + repoRev, + labelers, + }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/actor/getProfiles.ts b/packages/bsky/src/api/app/bsky/actor/getProfiles.ts index 862ac239c7f..014f9339fe8 100644 --- a/packages/bsky/src/api/app/bsky/actor/getProfiles.ts +++ b/packages/bsky/src/api/app/bsky/actor/getProfiles.ts @@ -2,7 +2,7 @@ import { mapDefined } from '@atproto/common' import { Server } from '../../../../lexicon' import { QueryParams } from '../../../../lexicon/types/app/bsky/actor/getProfiles' import AppContext from '../../../../context' -import { setRepoRev } from '../../../util' +import { resHeaders } from '../../../util' import { createPipeline, noRules } from '../../../../pipeline' import { HydrateCtx, @@ -15,7 +15,7 @@ export default function (server: Server, ctx: AppContext) { const getProfile = createPipeline(skeleton, hydration, noRules, presentation) server.app.bsky.actor.getProfiles({ auth: ctx.authVerifier.standardOptional, - handler: async ({ auth, params, req, res }) => { + handler: async ({ auth, params, req }) => { const viewer = auth.credentials.iss const labelers = ctx.reqLabelers(req) const hydrateCtx = { viewer, labelers } @@ -23,11 +23,14 @@ export default function (server: Server, ctx: AppContext) { const result = await getProfile({ ...params, hydrateCtx }, ctx) const repoRev = await ctx.hydrator.actor.getRepoRevSafe(viewer) - setRepoRev(res, repoRev) return { encoding: 'application/json', body: result, + headers: resHeaders({ + repoRev, + labelers, + }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/actor/getSuggestions.ts b/packages/bsky/src/api/app/bsky/actor/getSuggestions.ts index db312373ea0..451fcedcb5e 100644 --- a/packages/bsky/src/api/app/bsky/actor/getSuggestions.ts +++ b/packages/bsky/src/api/app/bsky/actor/getSuggestions.ts @@ -11,6 +11,7 @@ import { import { Views } from '../../../../views' import { DataPlaneClient } from '../../../../data-plane' import { parseString } from '../../../../hydration/util' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getSuggestions = createPipeline( @@ -30,6 +31,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/actor/searchActors.ts b/packages/bsky/src/api/app/bsky/actor/searchActors.ts index b7bf881ab3a..adb3b6704ca 100644 --- a/packages/bsky/src/api/app/bsky/actor/searchActors.ts +++ b/packages/bsky/src/api/app/bsky/actor/searchActors.ts @@ -14,6 +14,7 @@ import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator' import { Views } from '../../../../views' import { DataPlaneClient } from '../../../../data-plane' import { parseString } from '../../../../hydration/util' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const searchActors = createPipeline( @@ -32,6 +33,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: results, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/actor/searchActorsTypeahead.ts b/packages/bsky/src/api/app/bsky/actor/searchActorsTypeahead.ts index 0ac1506f86d..7517c58ab9d 100644 --- a/packages/bsky/src/api/app/bsky/actor/searchActorsTypeahead.ts +++ b/packages/bsky/src/api/app/bsky/actor/searchActorsTypeahead.ts @@ -14,6 +14,7 @@ import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator' import { Views } from '../../../../views' import { DataPlaneClient } from '../../../../data-plane' import { parseString } from '../../../../hydration/util' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const searchActorsTypeahead = createPipeline( @@ -35,6 +36,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: results, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getActorFeeds.ts b/packages/bsky/src/api/app/bsky/feed/getActorFeeds.ts index b94486940fa..0ba5c15409a 100644 --- a/packages/bsky/src/api/app/bsky/feed/getActorFeeds.ts +++ b/packages/bsky/src/api/app/bsky/feed/getActorFeeds.ts @@ -12,7 +12,7 @@ import { import { Views } from '../../../../views' import { DataPlaneClient } from '../../../../data-plane' import { parseString } from '../../../../hydration/util' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getActorFeeds = createPipeline( @@ -31,6 +31,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getActorLikes.ts b/packages/bsky/src/api/app/bsky/feed/getActorLikes.ts index a8b85a3c155..4411f7295ea 100644 --- a/packages/bsky/src/api/app/bsky/feed/getActorLikes.ts +++ b/packages/bsky/src/api/app/bsky/feed/getActorLikes.ts @@ -3,7 +3,7 @@ import { mapDefined } from '@atproto/common' import { Server } from '../../../../lexicon' import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getActorLikes' import AppContext from '../../../../context' -import { clearlyBadCursor, setRepoRev } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' import { createPipeline } from '../../../../pipeline' import { HydrateCtx, @@ -25,7 +25,7 @@ export default function (server: Server, ctx: AppContext) { ) server.app.bsky.feed.getActorLikes({ auth: ctx.authVerifier.standardOptional, - handler: async ({ params, auth, req, res }) => { + handler: async ({ params, auth, req }) => { const viewer = auth.credentials.iss const labelers = ctx.reqLabelers(req) const hydrateCtx = { labelers, viewer } @@ -33,11 +33,14 @@ export default function (server: Server, ctx: AppContext) { const result = await getActorLikes({ ...params, hydrateCtx }, ctx) const repoRev = await ctx.hydrator.actor.getRepoRevSafe(viewer) - setRepoRev(res, repoRev) return { encoding: 'application/json', body: result, + headers: resHeaders({ + repoRev, + labelers, + }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getAuthorFeed.ts b/packages/bsky/src/api/app/bsky/feed/getAuthorFeed.ts index a8174b164fc..eb686b80910 100644 --- a/packages/bsky/src/api/app/bsky/feed/getAuthorFeed.ts +++ b/packages/bsky/src/api/app/bsky/feed/getAuthorFeed.ts @@ -3,7 +3,7 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getAuthorFeed' import AppContext from '../../../../context' -import { clearlyBadCursor, setRepoRev } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' import { createPipeline } from '../../../../pipeline' import { HydrateCtx, @@ -27,7 +27,7 @@ export default function (server: Server, ctx: AppContext) { ) server.app.bsky.feed.getAuthorFeed({ auth: ctx.authVerifier.optionalStandardOrRole, - handler: async ({ params, auth, req, res }) => { + handler: async ({ params, auth, req }) => { const { viewer, canViewTakedowns } = ctx.authVerifier.parseCreds(auth) const labelers = ctx.reqLabelers(req) const hydrateCtx = { labelers, viewer } @@ -38,11 +38,14 @@ export default function (server: Server, ctx: AppContext) { ) const repoRev = await ctx.hydrator.actor.getRepoRevSafe(viewer) - setRepoRev(res, repoRev) return { encoding: 'application/json', body: result, + headers: resHeaders({ + repoRev, + labelers, + }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getFeed.ts b/packages/bsky/src/api/app/bsky/feed/getFeed.ts index 465348f0522..26e17ea3e2d 100644 --- a/packages/bsky/src/api/app/bsky/feed/getFeed.ts +++ b/packages/bsky/src/api/app/bsky/feed/getFeed.ts @@ -28,6 +28,7 @@ import { isDataplaneError, unpackIdentityServices, } from '../../../../data-plane' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getFeed = createPipeline( @@ -47,16 +48,19 @@ export default function (server: Server, ctx: AppContext) { 'accept-language': req.headers['accept-language'], }) // @NOTE feed cursors should not be affected by appview swap - const { timerSkele, timerHydr, resHeaders, ...result } = await getFeed( - { ...params, hydrateCtx, headers }, - ctx, - ) + const { + timerSkele, + timerHydr, + resHeaders: feedResHeaders, + ...result + } = await getFeed({ ...params, hydrateCtx, headers }, ctx) return { encoding: 'application/json', body: result, headers: { - ...(resHeaders ?? {}), + ...(feedResHeaders ?? {}), + ...resHeaders({ labelers }), 'server-timing': serverTimingHeader([timerSkele, timerHydr]), }, } diff --git a/packages/bsky/src/api/app/bsky/feed/getFeedGenerator.ts b/packages/bsky/src/api/app/bsky/feed/getFeedGenerator.ts index 171c497e2ae..9c94fb2b213 100644 --- a/packages/bsky/src/api/app/bsky/feed/getFeedGenerator.ts +++ b/packages/bsky/src/api/app/bsky/feed/getFeedGenerator.ts @@ -8,6 +8,7 @@ import { isDataplaneError, unpackIdentityServices, } from '../../../../data-plane' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { server.app.bsky.feed.getFeedGenerator({ @@ -63,6 +64,7 @@ export default function (server: Server, ctx: AppContext) { isOnline: true, isValid: true, }, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getFeedGenerators.ts b/packages/bsky/src/api/app/bsky/feed/getFeedGenerators.ts index d8225384c97..65d5a5dd316 100644 --- a/packages/bsky/src/api/app/bsky/feed/getFeedGenerators.ts +++ b/packages/bsky/src/api/app/bsky/feed/getFeedGenerators.ts @@ -9,6 +9,7 @@ import { Hydrator, } from '../../../../hydration/hydrator' import { Views } from '../../../../views' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getFeedGenerators = createPipeline( @@ -27,6 +28,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: view, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getLikes.ts b/packages/bsky/src/api/app/bsky/feed/getLikes.ts index 5e9ebf7d70c..402a6c44a8a 100644 --- a/packages/bsky/src/api/app/bsky/feed/getLikes.ts +++ b/packages/bsky/src/api/app/bsky/feed/getLikes.ts @@ -12,7 +12,7 @@ import { import { Views } from '../../../../views' import { parseString } from '../../../../hydration/util' import { creatorFromUri } from '../../../../views/util' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getLikes = createPipeline(skeleton, hydration, noBlocks, presentation) @@ -27,6 +27,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getListFeed.ts b/packages/bsky/src/api/app/bsky/feed/getListFeed.ts index daed2a99025..75ae9edf815 100644 --- a/packages/bsky/src/api/app/bsky/feed/getListFeed.ts +++ b/packages/bsky/src/api/app/bsky/feed/getListFeed.ts @@ -1,7 +1,7 @@ import { Server } from '../../../../lexicon' import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getListFeed' import AppContext from '../../../../context' -import { clearlyBadCursor, setRepoRev } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' import { createPipeline } from '../../../../pipeline' import { HydrateCtx, @@ -23,7 +23,7 @@ export default function (server: Server, ctx: AppContext) { ) server.app.bsky.feed.getListFeed({ auth: ctx.authVerifier.standardOptional, - handler: async ({ params, auth, req, res }) => { + handler: async ({ params, auth, req }) => { const viewer = auth.credentials.iss const labelers = ctx.reqLabelers(req) const hydrateCtx = { labelers, viewer } @@ -31,11 +31,11 @@ export default function (server: Server, ctx: AppContext) { const result = await getListFeed({ ...params, hydrateCtx }, ctx) const repoRev = await ctx.hydrator.actor.getRepoRevSafe(viewer) - setRepoRev(res, repoRev) return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers, repoRev }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getPostThread.ts b/packages/bsky/src/api/app/bsky/feed/getPostThread.ts index d7c4009bdaa..3cfd69f47f8 100644 --- a/packages/bsky/src/api/app/bsky/feed/getPostThread.ts +++ b/packages/bsky/src/api/app/bsky/feed/getPostThread.ts @@ -6,7 +6,7 @@ import { OutputSchema, } from '../../../../lexicon/types/app/bsky/feed/getPostThread' import AppContext from '../../../../context' -import { setRepoRev } from '../../../util' +import { ATPROTO_REPO_REV, resHeaders } from '../../../util' import { HydrationFnInput, PresentationFnInput, @@ -37,16 +37,21 @@ export default function (server: Server, ctx: AppContext) { result = await getPostThread({ ...params, hydrateCtx }, ctx) } catch (err) { const repoRev = await ctx.hydrator.actor.getRepoRevSafe(viewer) - setRepoRev(res, repoRev) + if (repoRev) { + res.setHeader(ATPROTO_REPO_REV, repoRev) + } throw err } const repoRev = await ctx.hydrator.actor.getRepoRevSafe(viewer) - setRepoRev(res, repoRev) return { encoding: 'application/json', body: result, + headers: resHeaders({ + repoRev, + labelers, + }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getPosts.ts b/packages/bsky/src/api/app/bsky/feed/getPosts.ts index fd5b89f96e0..322197c7e3f 100644 --- a/packages/bsky/src/api/app/bsky/feed/getPosts.ts +++ b/packages/bsky/src/api/app/bsky/feed/getPosts.ts @@ -10,6 +10,7 @@ import { } from '../../../../hydration/hydrator' import { Views } from '../../../../views' import { creatorFromUri } from '../../../../views/util' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getPosts = createPipeline(skeleton, hydration, noBlocks, presentation) @@ -25,6 +26,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: results, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getRepostedBy.ts b/packages/bsky/src/api/app/bsky/feed/getRepostedBy.ts index 887200476b7..eee518961b2 100644 --- a/packages/bsky/src/api/app/bsky/feed/getRepostedBy.ts +++ b/packages/bsky/src/api/app/bsky/feed/getRepostedBy.ts @@ -11,7 +11,7 @@ import { import { Views } from '../../../../views' import { parseString } from '../../../../hydration/util' import { creatorFromUri } from '../../../../views/util' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getRepostedBy = createPipeline( @@ -31,6 +31,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getSuggestedFeeds.ts b/packages/bsky/src/api/app/bsky/feed/getSuggestedFeeds.ts index ee42ea74dd3..e848eddd634 100644 --- a/packages/bsky/src/api/app/bsky/feed/getSuggestedFeeds.ts +++ b/packages/bsky/src/api/app/bsky/feed/getSuggestedFeeds.ts @@ -2,6 +2,7 @@ import { mapDefined } from '@atproto/common' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { parseString } from '../../../../hydration/util' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { server.app.bsky.feed.getSuggestedFeeds({ @@ -31,6 +32,7 @@ export default function (server: Server, ctx: AppContext) { feeds: feedViews, cursor: parseString(suggestedRes.cursor), }, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/getTimeline.ts b/packages/bsky/src/api/app/bsky/feed/getTimeline.ts index e9e31bcaceb..c58199d195b 100644 --- a/packages/bsky/src/api/app/bsky/feed/getTimeline.ts +++ b/packages/bsky/src/api/app/bsky/feed/getTimeline.ts @@ -1,7 +1,7 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getTimeline' -import { clearlyBadCursor, setRepoRev } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' import { createPipeline } from '../../../../pipeline' import { HydrateCtx, @@ -23,7 +23,7 @@ export default function (server: Server, ctx: AppContext) { ) server.app.bsky.feed.getTimeline({ auth: ctx.authVerifier.standard, - handler: async ({ params, auth, req, res }) => { + handler: async ({ params, auth, req }) => { const viewer = auth.credentials.iss const labelers = ctx.reqLabelers(req) const hydrateCtx = { labelers, viewer } @@ -31,11 +31,11 @@ export default function (server: Server, ctx: AppContext) { const result = await getTimeline({ ...params, hydrateCtx }, ctx) const repoRev = await ctx.hydrator.actor.getRepoRevSafe(viewer) - setRepoRev(res, repoRev) return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers, repoRev }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/feed/searchPosts.ts b/packages/bsky/src/api/app/bsky/feed/searchPosts.ts index c511883fbde..0ab4fb8ba3a 100644 --- a/packages/bsky/src/api/app/bsky/feed/searchPosts.ts +++ b/packages/bsky/src/api/app/bsky/feed/searchPosts.ts @@ -15,6 +15,7 @@ import { Views } from '../../../../views' import { DataPlaneClient } from '../../../../data-plane' import { parseString } from '../../../../hydration/util' import { creatorFromUri } from '../../../../views/util' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const searchPosts = createPipeline( @@ -33,6 +34,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: results, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/graph/getBlocks.ts b/packages/bsky/src/api/app/bsky/graph/getBlocks.ts index e7a9df2030e..b6723cb3c45 100644 --- a/packages/bsky/src/api/app/bsky/graph/getBlocks.ts +++ b/packages/bsky/src/api/app/bsky/graph/getBlocks.ts @@ -11,7 +11,7 @@ import { } from '../../../../pipeline' import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator' import { Views } from '../../../../views' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getBlocks = createPipeline(skeleton, hydration, noRules, presentation) @@ -25,6 +25,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/graph/getFollowers.ts b/packages/bsky/src/api/app/bsky/graph/getFollowers.ts index b2d3288b4bd..42e31f7288f 100644 --- a/packages/bsky/src/api/app/bsky/graph/getFollowers.ts +++ b/packages/bsky/src/api/app/bsky/graph/getFollowers.ts @@ -17,7 +17,7 @@ import { mergeStates, } from '../../../../hydration/hydrator' import { Views } from '../../../../views' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getFollowers = createPipeline( @@ -41,6 +41,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/graph/getFollows.ts b/packages/bsky/src/api/app/bsky/graph/getFollows.ts index d1840f9a19a..cfe8e220cc7 100644 --- a/packages/bsky/src/api/app/bsky/graph/getFollows.ts +++ b/packages/bsky/src/api/app/bsky/graph/getFollows.ts @@ -16,7 +16,7 @@ import { mergeStates, } from '../../../../hydration/hydrator' import { Views } from '../../../../views' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getFollows = createPipeline(skeleton, hydration, noBlocks, presentation) @@ -36,6 +36,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/graph/getList.ts b/packages/bsky/src/api/app/bsky/graph/getList.ts index 42c298ecbe2..92ea332d0ed 100644 --- a/packages/bsky/src/api/app/bsky/graph/getList.ts +++ b/packages/bsky/src/api/app/bsky/graph/getList.ts @@ -16,7 +16,7 @@ import { mergeStates, } from '../../../../hydration/hydrator' import { Views } from '../../../../views' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' import { ListItemInfo } from '../../../../proto/bsky_pb' export default function (server: Server, ctx: AppContext) { @@ -31,6 +31,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/graph/getListBlocks.ts b/packages/bsky/src/api/app/bsky/graph/getListBlocks.ts index 9b7e74f69b6..2360781a6ef 100644 --- a/packages/bsky/src/api/app/bsky/graph/getListBlocks.ts +++ b/packages/bsky/src/api/app/bsky/graph/getListBlocks.ts @@ -11,7 +11,7 @@ import { } from '../../../../pipeline' import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator' import { Views } from '../../../../views' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getListBlocks = createPipeline( @@ -30,6 +30,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/graph/getListMutes.ts b/packages/bsky/src/api/app/bsky/graph/getListMutes.ts index 7004e43249f..cc4f5a64942 100644 --- a/packages/bsky/src/api/app/bsky/graph/getListMutes.ts +++ b/packages/bsky/src/api/app/bsky/graph/getListMutes.ts @@ -11,7 +11,7 @@ import { } from '../../../../pipeline' import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator' import { Views } from '../../../../views' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getListMutes = createPipeline( @@ -30,6 +30,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/graph/getLists.ts b/packages/bsky/src/api/app/bsky/graph/getLists.ts index c53b7e14482..3ebc7fbb628 100644 --- a/packages/bsky/src/api/app/bsky/graph/getLists.ts +++ b/packages/bsky/src/api/app/bsky/graph/getLists.ts @@ -11,7 +11,7 @@ import { } from '../../../../pipeline' import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator' import { Views } from '../../../../views' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getLists = createPipeline(skeleton, hydration, noRules, presentation) @@ -26,6 +26,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/graph/getMutes.ts b/packages/bsky/src/api/app/bsky/graph/getMutes.ts index db81c46ab1b..6890087afa9 100644 --- a/packages/bsky/src/api/app/bsky/graph/getMutes.ts +++ b/packages/bsky/src/api/app/bsky/graph/getMutes.ts @@ -11,7 +11,7 @@ import { createPipeline, noRules, } from '../../../../pipeline' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getMutes = createPipeline(skeleton, hydration, noRules, presentation) @@ -25,6 +25,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/graph/getSuggestedFollowsByActor.ts b/packages/bsky/src/api/app/bsky/graph/getSuggestedFollowsByActor.ts index 579a9ab329b..356b33aa7fe 100644 --- a/packages/bsky/src/api/app/bsky/graph/getSuggestedFollowsByActor.ts +++ b/packages/bsky/src/api/app/bsky/graph/getSuggestedFollowsByActor.ts @@ -12,6 +12,7 @@ import { } from '../../../../pipeline' import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator' import { Views } from '../../../../views' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const getSuggestedFollowsByActor = createPipeline( @@ -33,6 +34,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/labeler/getServices.ts b/packages/bsky/src/api/app/bsky/labeler/getServices.ts index 9f151757361..aeca1a10834 100644 --- a/packages/bsky/src/api/app/bsky/labeler/getServices.ts +++ b/packages/bsky/src/api/app/bsky/labeler/getServices.ts @@ -1,6 +1,7 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { mapDefined } from '@atproto/common' +import { resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { server.app.bsky.labeler.getServices({ @@ -38,6 +39,7 @@ export default function (server: Server, ctx: AppContext) { body: { views, }, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/notification/listNotifications.ts b/packages/bsky/src/api/app/bsky/notification/listNotifications.ts index 6ee8951fc66..8eb3996b2a2 100644 --- a/packages/bsky/src/api/app/bsky/notification/listNotifications.ts +++ b/packages/bsky/src/api/app/bsky/notification/listNotifications.ts @@ -14,7 +14,7 @@ import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator' import { Views } from '../../../../views' import { Notification } from '../../../../proto/bsky_pb' import { didFromUri } from '../../../../hydration/util' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' export default function (server: Server, ctx: AppContext) { const listNotifications = createPipeline( @@ -33,6 +33,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', body: result, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts b/packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts index c676e115908..883a451c8ac 100644 --- a/packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts +++ b/packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts @@ -2,7 +2,7 @@ import { mapDefined } from '@atproto/common' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { parseString } from '../../../../hydration/util' -import { clearlyBadCursor } from '../../../util' +import { clearlyBadCursor, resHeaders } from '../../../util' // THIS IS A TEMPORARY UNSPECCED ROUTE // @TODO currently mirrors getSuggestedFeeds and ignores the "query" param. @@ -53,6 +53,7 @@ export default function (server: Server, ctx: AppContext) { feeds: feedViews, cursor, }, + headers: resHeaders({ labelers }), } }, }) diff --git a/packages/bsky/src/api/util.ts b/packages/bsky/src/api/util.ts index 2fe54a8a7be..3ee0ea2c59b 100644 --- a/packages/bsky/src/api/util.ts +++ b/packages/bsky/src/api/util.ts @@ -1,9 +1,24 @@ -import express from 'express' +import { ParsedLabelers, formatLabelerHeader } from '../util' -export const setRepoRev = (res: express.Response, rev: string | null) => { - if (rev !== null) { - res.setHeader('Atproto-Repo-Rev', rev) +export const ATPROTO_CONTENT_LABELERS = 'Atproto-Content-Labelers' +export const ATPROTO_REPO_REV = 'Atproto-Repo-Rev' + +type ResHeaderOpts = { + labelers: ParsedLabelers + repoRev: string | null +} + +export const resHeaders = ( + opts: Partial, +): Record => { + const headers = {} + if (opts.labelers) { + headers[ATPROTO_CONTENT_LABELERS] = formatLabelerHeader(opts.labelers) + } + if (opts.repoRev) { + headers[ATPROTO_REPO_REV] = opts.repoRev } + return headers } export const clearlyBadCursor = (cursor?: string) => { diff --git a/packages/bsky/src/context.ts b/packages/bsky/src/context.ts index b166368cdf8..e8a15f5197a 100644 --- a/packages/bsky/src/context.ts +++ b/packages/bsky/src/context.ts @@ -9,9 +9,14 @@ import { DataPlaneClient } from './data-plane/client' import { Hydrator } from './hydration/hydrator' import { Views } from './views' import { AuthVerifier } from './auth-verifier' -import { dedupeStrs } from '@atproto/common' import { BsyncClient } from './bsync' import { CourierClient } from './courier' +import { + ParsedLabelers, + defaultLabelerHeader, + parseLabelerHeader, +} from './util' +import { httpLogger as log } from './logger' export class AppContext { constructor( @@ -82,15 +87,17 @@ export class AppContext { }) } - reqLabelers(req: express.Request): string[] { - const val = req.header('atproto-labelers') - if (!val) return this.cfg.labelsFromIssuerDids - return dedupeStrs( - val - .split(',') - .map((did) => did.trim()) - .slice(0, 10), - ) + reqLabelers(req: express.Request): ParsedLabelers { + const val = req.header('atproto-accept-labelers') + let parsed: ParsedLabelers | null + try { + parsed = parseLabelerHeader(val) + } catch (err) { + parsed = null + log.info({ err, val }, 'failed to parse labeler header') + } + if (!parsed) return defaultLabelerHeader(this.cfg.labelsFromIssuerDids) + return parsed } } diff --git a/packages/bsky/src/hydration/hydrator.ts b/packages/bsky/src/hydration/hydrator.ts index 9181032b7da..de64a333ace 100644 --- a/packages/bsky/src/hydration/hydrator.ts +++ b/packages/bsky/src/hydration/hydrator.ts @@ -45,9 +45,10 @@ import { FeedItem, ItemRef, } from './feed' +import { ParsedLabelers } from '../util' export type HydrateCtx = { - labelers: string[] + labelers: ParsedLabelers viewer: string | null } @@ -136,7 +137,10 @@ export class Hydrator { ): Promise { const [actors, labels, profileViewersState] = await Promise.all([ this.actor.getActors(dids, includeTakedowns), - this.label.getLabelsForSubjects(labelSubjectsForDid(dids), ctx.labelers), + this.label.getLabelsForSubjects( + labelSubjectsForDid(dids), + ctx.labelers.dids, + ), this.hydrateProfileViewers(dids, ctx), ]) return mergeStates(profileViewersState ?? {}, { @@ -298,7 +302,7 @@ export class Hydrator { ] = await Promise.all([ this.feed.getPostAggregates(refs), ctx.viewer ? this.feed.getPostViewerStates(refs, ctx.viewer) : undefined, - this.label.getLabelsForSubjects(allPostUris, ctx.labelers), + this.label.getLabelsForSubjects(allPostUris, ctx.labelers.dids), this.hydratePostBlocks(posts), this.hydrateProfiles(allPostUris.map(didFromUri), ctx, includeTakedowns), this.hydrateLists([...nestedListUris, ...gateListUris], ctx), @@ -494,7 +498,7 @@ export class Hydrator { this.feed.getLikes(likeUris), // reason: like this.feed.getReposts(repostUris), // reason: repost this.graph.getFollows(followUris), // reason: follow - this.label.getLabelsForSubjects(uris, ctx.labelers), + this.label.getLabelsForSubjects(uris, ctx.labelers.dids), this.hydrateProfiles(uris.map(didFromUri), ctx), ]) return mergeStates(profileState, { diff --git a/packages/bsky/src/util.ts b/packages/bsky/src/util.ts new file mode 100644 index 00000000000..50c0c1fa565 --- /dev/null +++ b/packages/bsky/src/util.ts @@ -0,0 +1,44 @@ +import { parseList } from 'structured-headers' + +export type ParsedLabelers = { + dids: string[] + redact: Set +} + +export const parseLabelerHeader = ( + header: string | undefined, +): ParsedLabelers | null => { + if (!header) return null + const labelerDids = new Set() + const redactDids = new Set() + const parsed = parseList(header) + for (const item of parsed) { + const did = item[0].toString() + if (!did) { + return null + } + labelerDids.add(did) + const redact = item[1].get('redact')?.valueOf() + if (redact === true) { + redactDids.add(did) + } + } + return { + dids: [...labelerDids], + redact: redactDids, + } +} + +export const defaultLabelerHeader = (dids: string[]): ParsedLabelers => { + return { + dids, + redact: new Set(dids), + } +} + +export const formatLabelerHeader = (parsed: ParsedLabelers): string => { + const parts = parsed.dids.map((did) => + parsed.redact.has(did) ? `${did};redact` : did, + ) + return parts.join(',') +} diff --git a/packages/bsky/tests/label-hydration.test.ts b/packages/bsky/tests/label-hydration.test.ts index a7fe678581c..b6927dd8afe 100644 --- a/packages/bsky/tests/label-hydration.test.ts +++ b/packages/bsky/tests/label-hydration.test.ts @@ -40,11 +40,17 @@ describe('label hydration', () => { it('hydrates labels based on a supplied labeler header', async () => { const res = await pdsAgent.api.app.bsky.actor.getProfile( { actor: carol }, - { headers: { ...sc.getHeaders(bob), 'atproto-labelers': alice } }, + { + headers: { + ...sc.getHeaders(bob), + 'atproto-accept-labelers': `${alice};redact`, + }, + }, ) expect(res.data.labels?.length).toBe(1) expect(res.data.labels?.[0].src).toBe(alice) expect(res.data.labels?.[0].val).toBe('spam') + expect(res.headers['atproto-content-labelers']).toEqual(`${alice};redact`) }) it('hydrates labels based on multiple a supplied labelers', async () => { @@ -53,7 +59,7 @@ describe('label hydration', () => { { headers: { ...sc.getHeaders(bob), - 'atproto-labelers': `${alice},${bob}, ${labelerDid}`, + 'atproto-accept-labelers': `${alice},${bob};redact, ${labelerDid}`, }, }, ) @@ -65,6 +71,10 @@ describe('label hydration', () => { expect(res.data.labels?.find((l) => l.src === labelerDid)?.val).toEqual( 'misleading', ) + const labelerHeaderDids = res.headers['atproto-content-labelers'].split(',') + expect(labelerHeaderDids.sort()).toEqual( + [alice, `${bob};redact`, labelerDid].sort(), + ) }) it('defaults to service labels when no labeler header is provided', async () => { @@ -75,6 +85,9 @@ describe('label hydration', () => { expect(res.data.labels?.length).toBe(1) expect(res.data.labels?.[0].src).toBe(labelerDid) expect(res.data.labels?.[0].val).toBe('misleading') + expect(res.headers['atproto-content-labelers']).toEqual( + `${labelerDid};redact`, + ) }) it('hydrates labels onto list views.', async () => { diff --git a/packages/pds/src/pipethrough.ts b/packages/pds/src/pipethrough.ts index 0d9c00737b5..8f633a7448f 100644 --- a/packages/pds/src/pipethrough.ts +++ b/packages/pds/src/pipethrough.ts @@ -95,10 +95,10 @@ export const parseProxyHeader = async ( return { did, serviceUrl } } -const HEADERS_TO_FORWARD = [ +const REQ_HEADERS_TO_FORWARD = [ 'accept-language', 'content-type', - 'atproto-labelers', + 'atproto-accept-labelers', ] export const createUrlAndHeaders = async ( @@ -122,7 +122,7 @@ export const createUrlAndHeaders = async ( ? (await ctx.serviceAuthHeaders(requester, aud)).headers : {} // forward select headers to upstream services - for (const header of HEADERS_TO_FORWARD) { + for (const header of REQ_HEADERS_TO_FORWARD) { const val = req.headers[header] if (val) { headers[header] = val @@ -152,15 +152,24 @@ export const doProxy = async (url: URL, reqInit: RequestInit) => { ) } const encoding = res.headers.get('content-type') ?? 'application/json' - const repoRevHeader = res.headers.get('atproto-repo-rev') - const contentLanguage = res.headers.get('content-language') - const resHeaders = noUndefinedVals({ - ['atproto-repo-rev']: repoRevHeader ?? undefined, - ['content-language']: contentLanguage ?? undefined, - }) + const resHeaders = makeResHeaders(res) return { encoding, buffer, headers: resHeaders } } +const RES_HEADERS_TO_FORWARD = [ + 'atproto-repo-rev', + 'content-language', + 'atproto-content-labelers', +] + +const makeResHeaders = (res: Response): Record => { + const headers = RES_HEADERS_TO_FORWARD.reduce((acc, cur) => { + acc[cur] = res.headers.get(cur) ?? undefined + return acc + }, {} as Record) + return noUndefinedVals(headers) +} + const isSafeUrl = (url: URL) => { if (url.protocol !== 'https:') return false if (!url.hostname || url.hostname === 'localhost') return false diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1f673eeaa28..e8c64766285 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -254,6 +254,9 @@ importers: sharp: specifier: ^0.32.6 version: 0.32.6 + structured-headers: + specifier: ^1.0.1 + version: 1.0.1 typed-emitter: specifier: ^2.1.0 version: 2.1.0 @@ -11317,6 +11320,11 @@ packages: '@tokenizer/token': 0.3.0 peek-readable: 4.1.0 + /structured-headers@1.0.1: + resolution: {integrity: sha512-QYBxdBtA4Tl5rFPuqmbmdrS9kbtren74RTJTcs0VSQNVV5iRhJD4QlYTLD0+81SBwUQctjEQzjTRI3WG4DzICA==} + engines: {node: '>= 14', npm: '>=6'} + dev: false + /supports-color@5.5.0: resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} engines: {node: '>=4'}