From 8f613b5b88d4424ef86056d61ea212daee51863e Mon Sep 17 00:00:00 2001 From: Devin Ivy Date: Wed, 31 Jan 2024 14:58:22 -0500 Subject: [PATCH 1/2] add protos for feedgen search --- packages/bsky/proto/bsky.proto | 10 ++ packages/bsky/src/proto/bsky_connect.ts | 11 +++ packages/bsky/src/proto/bsky_pb.ts | 125 ++++++++++++++++++++++++ 3 files changed, 146 insertions(+) diff --git a/packages/bsky/proto/bsky.proto b/packages/bsky/proto/bsky.proto index b9d0c4f055f..ffff8efa043 100644 --- a/packages/bsky/proto/bsky.proto +++ b/packages/bsky/proto/bsky.proto @@ -636,6 +636,15 @@ message GetSuggestedFeedsResponse { string cursor = 2; } +message SearchFeedGeneratorsRequest { + string query = 1; + int32 limit = 2; +} + +message SearchFeedGeneratorsResponse { + repeated string uris = 1; +} + // - Returns feed generator validity and online status with uris A, B, C… // - Not currently being used, but could be worhthwhile. message GetFeedGeneratorStatusRequest { @@ -997,6 +1006,7 @@ service Service { rpc GetActorFeeds(GetActorFeedsRequest) returns (GetActorFeedsResponse); rpc GetSuggestedFeeds(GetSuggestedFeedsRequest) returns (GetSuggestedFeedsResponse); rpc GetFeedGeneratorStatus(GetFeedGeneratorStatusRequest) returns (GetFeedGeneratorStatusResponse); + rpc SearchFeedGenerators(SearchFeedGeneratorsRequest) returns (SearchFeedGeneratorsResponse); // Feeds rpc GetAuthorFeed(GetAuthorFeedRequest) returns (GetAuthorFeedResponse); diff --git a/packages/bsky/src/proto/bsky_connect.ts b/packages/bsky/src/proto/bsky_connect.ts index 1dbea106631..00e2e5b9204 100644 --- a/packages/bsky/src/proto/bsky_connect.ts +++ b/packages/bsky/src/proto/bsky_connect.ts @@ -140,6 +140,8 @@ import { PingResponse, SearchActorsRequest, SearchActorsResponse, + SearchFeedGeneratorsRequest, + SearchFeedGeneratorsResponse, SearchPostsRequest, SearchPostsResponse, TakedownActorRequest, @@ -620,6 +622,15 @@ export const Service = { O: GetFeedGeneratorStatusResponse, kind: MethodKind.Unary, }, + /** + * @generated from rpc bsky.Service.SearchFeedGenerators + */ + searchFeedGenerators: { + name: 'SearchFeedGenerators', + I: SearchFeedGeneratorsRequest, + O: SearchFeedGeneratorsResponse, + kind: MethodKind.Unary, + }, /** * Feeds * diff --git a/packages/bsky/src/proto/bsky_pb.ts b/packages/bsky/src/proto/bsky_pb.ts index 02c1c890ee8..7c5ddcf1865 100644 --- a/packages/bsky/src/proto/bsky_pb.ts +++ b/packages/bsky/src/proto/bsky_pb.ts @@ -6546,6 +6546,131 @@ export class GetSuggestedFeedsResponse extends Message { + /** + * @generated from field: string query = 1; + */ + query = '' + + /** + * @generated from field: int32 limit = 2; + */ + limit = 0 + + constructor(data?: PartialMessage) { + super() + proto3.util.initPartial(data, this) + } + + static readonly runtime: typeof proto3 = proto3 + static readonly typeName = 'bsky.SearchFeedGeneratorsRequest' + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: 'query', kind: 'scalar', T: 9 /* ScalarType.STRING */ }, + { no: 2, name: 'limit', kind: 'scalar', T: 5 /* ScalarType.INT32 */ }, + ]) + + static fromBinary( + bytes: Uint8Array, + options?: Partial, + ): SearchFeedGeneratorsRequest { + return new SearchFeedGeneratorsRequest().fromBinary(bytes, options) + } + + static fromJson( + jsonValue: JsonValue, + options?: Partial, + ): SearchFeedGeneratorsRequest { + return new SearchFeedGeneratorsRequest().fromJson(jsonValue, options) + } + + static fromJsonString( + jsonString: string, + options?: Partial, + ): SearchFeedGeneratorsRequest { + return new SearchFeedGeneratorsRequest().fromJsonString(jsonString, options) + } + + static equals( + a: + | SearchFeedGeneratorsRequest + | PlainMessage + | undefined, + b: + | SearchFeedGeneratorsRequest + | PlainMessage + | undefined, + ): boolean { + return proto3.util.equals(SearchFeedGeneratorsRequest, a, b) + } +} + +/** + * @generated from message bsky.SearchFeedGeneratorsResponse + */ +export class SearchFeedGeneratorsResponse extends Message { + /** + * @generated from field: repeated string uris = 1; + */ + uris: string[] = [] + + constructor(data?: PartialMessage) { + super() + proto3.util.initPartial(data, this) + } + + static readonly runtime: typeof proto3 = proto3 + static readonly typeName = 'bsky.SearchFeedGeneratorsResponse' + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { + no: 1, + name: 'uris', + kind: 'scalar', + T: 9 /* ScalarType.STRING */, + repeated: true, + }, + ]) + + static fromBinary( + bytes: Uint8Array, + options?: Partial, + ): SearchFeedGeneratorsResponse { + return new SearchFeedGeneratorsResponse().fromBinary(bytes, options) + } + + static fromJson( + jsonValue: JsonValue, + options?: Partial, + ): SearchFeedGeneratorsResponse { + return new SearchFeedGeneratorsResponse().fromJson(jsonValue, options) + } + + static fromJsonString( + jsonString: string, + options?: Partial, + ): SearchFeedGeneratorsResponse { + return new SearchFeedGeneratorsResponse().fromJsonString( + jsonString, + options, + ) + } + + static equals( + a: + | SearchFeedGeneratorsResponse + | PlainMessage + | undefined, + b: + | SearchFeedGeneratorsResponse + | PlainMessage + | undefined, + ): boolean { + return proto3.util.equals(SearchFeedGeneratorsResponse, a, b) + } +} + /** * - Returns feed generator validity and online status with uris A, B, C… * - Not currently being used, but could be worhthwhile. From c03fcecd0547ca0d35e5aa435113bf79909f5233 Mon Sep 17 00:00:00 2001 From: Devin Ivy Date: Wed, 31 Jan 2024 16:05:53 -0500 Subject: [PATCH 2/2] support feed search on getPopularFeedGenerators --- .../unspecced/getPopularFeedGenerators.ts | 28 +++++++++++++----- .../src/data-plane/server/routes/feed-gens.ts | 25 +++++++++++++++- .../feed-generation.test.ts.snap | 1 + packages/bsky/tests/feed-generation.test.ts | 29 ++++++++++++------- 4 files changed, 64 insertions(+), 19 deletions(-) diff --git a/packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts b/packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts index 55263e8483a..a5a3d8a8cef 100644 --- a/packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts +++ b/packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts @@ -20,12 +20,26 @@ export default function (server: Server, ctx: AppContext) { } } - const suggestedRes = await ctx.dataplane.getSuggestedFeeds({ - actorDid: viewer ?? undefined, - limit: params.limit, - cursor: params.cursor, - }) - const uris = suggestedRes.uris + let uris: string[] + let cursor: string | undefined + + const query = params.query?.trim() ?? '' + if (query) { + const res = await ctx.dataplane.searchFeedGenerators({ + query, + limit: params.limit, + }) + uris = res.uris + } else { + const res = await ctx.dataplane.getSuggestedFeeds({ + actorDid: viewer ?? undefined, + limit: params.limit, + cursor: params.cursor, + }) + uris = res.uris + cursor = parseString(res.cursor) + } + const hydration = await ctx.hydrator.hydrateFeedGens(uris, viewer) const feedViews = mapDefined(uris, (uri) => ctx.views.feedGenerator(uri, hydration), @@ -35,7 +49,7 @@ export default function (server: Server, ctx: AppContext) { encoding: 'application/json', body: { feeds: feedViews, - cursor: parseString(suggestedRes.cursor), + cursor, }, } }, diff --git a/packages/bsky/src/data-plane/server/routes/feed-gens.ts b/packages/bsky/src/data-plane/server/routes/feed-gens.ts index adaff47d3e3..129229f923f 100644 --- a/packages/bsky/src/data-plane/server/routes/feed-gens.ts +++ b/packages/bsky/src/data-plane/server/routes/feed-gens.ts @@ -30,14 +30,37 @@ export default (db: Database): Partial> => ({ } }, - async getSuggestedFeeds() { + async getSuggestedFeeds(req) { const feeds = await db.db .selectFrom('suggested_feed') .orderBy('suggested_feed.order', 'asc') + .if(!!req.cursor, (q) => q.where('order', '>', parseInt(req.cursor, 10))) + .limit(req.limit || 50) .selectAll() .execute() return { uris: feeds.map((f) => f.uri), + cursor: feeds.at(-1)?.order.toString(), + } + }, + + async searchFeedGenerators(req) { + const { ref } = db.db.dynamic + const limit = req.limit + const query = req.query.trim() + let builder = db.db + .selectFrom('feed_generator') + .if(!!query, (q) => q.where('displayName', 'ilike', `%${query}%`)) + .selectAll() + const keyset = new TimeCidKeyset( + ref('feed_generator.createdAt'), + ref('feed_generator.cid'), + ) + builder = paginate(builder, { limit, keyset }) + const feeds = await builder.execute() + return { + uris: feeds.map((f) => f.uri), + cursor: keyset.packFromResult(feeds), } }, diff --git a/packages/bsky/tests/__snapshots__/feed-generation.test.ts.snap b/packages/bsky/tests/__snapshots__/feed-generation.test.ts.snap index 2de2fd20ffb..8aea15ddfc8 100644 --- a/packages/bsky/tests/__snapshots__/feed-generation.test.ts.snap +++ b/packages/bsky/tests/__snapshots__/feed-generation.test.ts.snap @@ -1597,6 +1597,7 @@ Object { exports[`feed generation getSuggestedFeeds returns list of suggested feed generators 1`] = ` Object { + "cursor": "4", "feeds": Array [ Object { "cid": "cids(0)", diff --git a/packages/bsky/tests/feed-generation.test.ts b/packages/bsky/tests/feed-generation.test.ts index 23cd5f1e759..00426075fe3 100644 --- a/packages/bsky/tests/feed-generation.test.ts +++ b/packages/bsky/tests/feed-generation.test.ts @@ -348,18 +348,26 @@ describe('feed generation', () => { }) }) - // @TODO support from dataplane - describe.skip('getPopularFeedGenerators', () => { + describe('getPopularFeedGenerators', () => { it('gets popular feed generators', async () => { - const resEven = - await agent.api.app.bsky.unspecced.getPopularFeedGenerators( - {}, - { headers: await network.serviceHeaders(sc.dids.bob) }, - ) - expect(resEven.data.feeds.map((f) => f.likeCount)).toEqual([ - 2, 0, 0, 0, 0, + const res = await agent.api.app.bsky.unspecced.getPopularFeedGenerators( + {}, + { headers: await network.serviceHeaders(sc.dids.bob) }, + ) + expect(res.data.feeds.map((f) => f.uri)).not.toContain(feedUriPrime) // taken-down + expect(res.data.feeds.map((f) => f.uri)).toEqual([ + feedUriAll, + feedUriEven, + feedUriBadPagination, ]) - expect(resEven.data.feeds.map((f) => f.uri)).not.toContain(feedUriPrime) // taken-down + }) + + it('searches feed generators', async () => { + const res = await agent.api.app.bsky.unspecced.getPopularFeedGenerators( + { query: 'all' }, + { headers: await network.serviceHeaders(sc.dids.bob) }, + ) + expect(res.data.feeds.map((f) => f.uri)).toEqual([feedUriAll]) }) it('paginates', async () => { @@ -368,7 +376,6 @@ describe('feed generation', () => { {}, { headers: await network.serviceHeaders(sc.dids.bob) }, ) - const resOne = await agent.api.app.bsky.unspecced.getPopularFeedGenerators( { limit: 2 },