fold in cid, auth, tracing, node version changes

.github/workflows/build-and-push-bsky-ghcr.yaml

@@ -3,8 +3,8 @@ on:
   push:
     branches:
       - main
-      - bav-v2-drop-pg
-      - bav-v2-dataplane-clients
+      - appview-v2-testing
+      - appview-v2-testing-grpc
 env:
   REGISTRY: ghcr.io
   USERNAME: ${{ github.actor }}

packages/bsky/src/api/com/atproto/admin/getSubjectStatus.ts

@@ -38,7 +38,7 @@ export default function (server: Server, ctx: AppContext) {
             subject: {
               $type: 'com.atproto.repo.strongRef',
               uri,
-              cid: res.cid.toString(),
+              cid: res.cid,
             },
             takedown: {
               applied: !!res.takedownRef,

packages/bsky/src/api/com/atproto/repo/getRecord.ts

@@ -14,15 +14,15 @@ export default function (server: Server, ctx: AppContext) {
     const uri = AtUri.make(did, collection, rkey).toString()
     const result = await ctx.hydrator.getRecord(uri, true)
 
-    if (!result || (cid && result.cid.toString() !== cid)) {
+    if (!result || (cid && result.cid !== cid)) {
       throw new InvalidRequestError(`Could not locate record: ${uri}`)
     }
 
     return {
       encoding: 'application/json' as const,
       body: {
         uri: uri,
-        cid: result.cid.toString(),
+        cid: result.cid,
         value: result.record,
       },
     }

packages/bsky/src/auth-verifier.ts

@@ -73,6 +73,18 @@ export class AuthVerifier {
   // verifiers (arrow fns to preserve scope)
 
   standard = async (ctx: ReqCtx): Promise<StandardOutput> => {
+    // @TODO remove! basic auth + did supported just for testing.
+    if (isBasicToken(ctx.req)) {
+      const aud = this.ownDid
+      const iss = ctx.req.headers['appview-as-did']
+      if (typeof iss !== 'string' || !iss.startsWith('did:')) {
+        throw new AuthRequiredError('bad issuer')
+      }
+      if (!this.parseRoleCreds(ctx.req).admin) {
+        throw new AuthRequiredError('bad credentials')
+      }
+      return { credentials: { type: 'standard', iss, aud } }
+    }
     const { iss, aud } = await this.verifyServiceJwt(ctx, {
       aud: this.ownDid,
       iss: null,
@@ -83,7 +95,7 @@ export class AuthVerifier {
   standardOptional = async (
     ctx: ReqCtx,
   ): Promise<StandardOutput | NullOutput> => {
-    if (isBearerToken(ctx.req)) {
+    if (isBearerToken(ctx.req) || isBasicToken(ctx.req)) {
       return this.standard(ctx)
     }
     return this.nullCreds()
@@ -252,6 +264,10 @@ const isBearerToken = (req: express.Request): boolean => {
   return req.headers.authorization?.startsWith(BEARER) ?? false
 }
 
+const isBasicToken = (req: express.Request): boolean => {
+  return req.headers.authorization?.startsWith(BASIC) ?? false
+}
+
 const bearerTokenFromReq = (req: express.Request) => {
   const header = req.headers.authorization || ''
   if (!header.startsWith(BEARER)) return null

packages/bsky/src/data-plane/client.ts

@@ -8,7 +8,7 @@ import {
   createPromiseClient,
   makeAnyClient,
 } from '@connectrpc/connect'
-import { createConnectTransport } from '@connectrpc/connect-node'
+import { createGrpcTransport } from '@connectrpc/connect-node'
 import { getDidKeyFromMultibase } from '@atproto/identity'
 import { Service } from '../proto/bsky_connect'
 
@@ -66,9 +66,10 @@ const createBaseClient = (
   opts: { httpVersion?: HttpVersion; rejectUnauthorized?: boolean },
 ): BaseClient => {
   const { httpVersion = '2', rejectUnauthorized = true } = opts
-  const transport = createConnectTransport({
+  const transport = createGrpcTransport({
     baseUrl,
     httpVersion,
+    acceptCompression: [],
     nodeOptions: { rejectUnauthorized },
   })
   return {

packages/bsky/src/hydration/actor.ts

@@ -1,9 +1,7 @@
-import { CID } from 'multiformats/cid'
 import { DataPlaneClient } from '../data-plane/client'
 import { Record as ProfileRecord } from '../lexicon/types/app/bsky/actor/profile'
 import {
   HydrationMap,
-  parseCid,
   parseRecordBytes,
   parseString,
   safeTakedownRef,
@@ -13,7 +11,7 @@ export type Actor = {
   did: string
   handle?: string
   profile?: ProfileRecord
-  profileCid?: CID
+  profileCid?: string
   profileTakedownRef?: string
   sortedAt?: Date
   takedownRef?: string
@@ -98,7 +96,7 @@ export class ActorHydrator {
         did,
         handle: parseString(actor.handle),
         profile: parseRecordBytes<ProfileRecord>(profile?.record),
-        profileCid: parseCid(profile?.cid),
+        profileCid: profile?.cid,
         profileTakedownRef: safeTakedownRef(profile),
         sortedAt: profile?.sortedAt?.toDate(),
         takedownRef: safeTakedownRef(actor),

packages/bsky/src/hydration/hydrator.ts

@@ -371,7 +371,7 @@ export class Hydrator {
     const postAndReplyRefs: ItemRef[] = []
     posts.forEach((post, uri) => {
       if (!post) return
-      postAndReplyRefs.push({ uri, cid: post.cid.toString() })
+      postAndReplyRefs.push({ uri, cid: post.cid })
       if (post.record.reply) {
         postAndReplyRefs.push(post.record.reply.root, post.record.reply.parent)
       }

packages/bsky/src/hydration/util.ts

@@ -2,6 +2,7 @@ import { AtUri } from '@atproto/syntax'
 import { jsonToLex } from '@atproto/lexicon'
 import { CID } from 'multiformats/cid'
 import * as ui8 from 'uint8arrays'
+import { lexicons } from '../lexicon/lexicons'
 import { Record } from '../proto/bsky_pb'
 
 export class HydrationMap<T> extends Map<string, T | null> {
@@ -15,7 +16,7 @@ export class HydrationMap<T> extends Map<string, T | null> {
 
 export type RecordInfo<T> = {
   record: T
-  cid: CID
+  cid: string
   sortedAt: Date
   takedownRef: string | undefined
 }
@@ -28,9 +29,12 @@ export const parseRecord = <T>(
     return undefined
   }
   const record = parseRecordBytes<T>(entry.record)
-  const cid = parseCid(entry.cid)
+  const cid = entry.cid
   const sortedAt = entry.sortedAt?.toDate() ?? new Date(0)
   if (!record || !cid) return
+  if (!isValidRecord(record)) {
+    return
+  }
   return {
     record,
     cid,
@@ -39,11 +43,24 @@ export const parseRecord = <T>(
   }
 }
 
+const isValidRecord = (json: unknown) => {
+  const lexRecord = jsonToLex(json)
+  if (typeof lexRecord?.['$type'] !== 'string') {
+    return false
+  }
+  try {
+    lexicons.assertValidRecord(lexRecord['$type'], lexRecord)
+    return true
+  } catch {
+    return false
+  }
+}
+
+// @NOTE not parsed into lex format, so will not match lexicon record types on CID and blob values.
 export const parseRecordBytes = <T>(
   bytes: Uint8Array | undefined,
 ): T | undefined => {
-  const parsed = parseJsonBytes(bytes)
-  return jsonToLex(parsed) as T
+  return parseJsonBytes(bytes) as T
 }
 
 export const parseJsonBytes = (

packages/bsky/src/image/uri.ts

@@ -1,4 +1,3 @@
-import { CID } from 'multiformats/cid'
 import { Options } from './util'
 
 // @NOTE if there are any additions here, ensure to include them on ImageUriBuilder.presets
@@ -20,7 +19,7 @@ export class ImageUriBuilder {
     'feed_fullsize',
   ]
 
-  getPresetUri(id: ImagePreset, did: string, cid: string | CID): string {
+  getPresetUri(id: ImagePreset, did: string, cid: string): string {
     const options = presets[id]
     if (!options) {
       throw new Error(`Unrecognized requested common uri type: ${id}`)
@@ -30,14 +29,14 @@ export class ImageUriBuilder {
       ImageUriBuilder.getPath({
         preset: id,
         did,
-        cid: typeof cid === 'string' ? CID.parse(cid) : cid,
+        cid,
       })
     )
   }
 
   static getPath(opts: { preset: ImagePreset } & BlobLocation) {
     const { format } = presets[opts.preset]
-    return `/${opts.preset}/plain/${opts.did}/${opts.cid.toString()}@${format}`
+    return `/${opts.preset}/plain/${opts.did}/${opts.cid}@${format}`
   }
 
   static getOptions(
@@ -59,14 +58,14 @@ export class ImageUriBuilder {
     return {
       ...presets[preset],
       did,
-      cid: CID.parse(cid),
+      cid,
       preset,
       format,
     }
   }
 }
 
-type BlobLocation = { cid: CID; did: string }
+type BlobLocation = { cid: string; did: string }
 
 export class BadPathError extends Error {}
 

packages/bsky/src/views/index.ts

@@ -1,4 +1,5 @@
 import { AtUri, INVALID_HANDLE, normalizeDatetimeAlways } from '@atproto/syntax'
+import { mapDefined } from '@atproto/common'
 import { ImageUriBuilder } from '../image/uri'
 import { HydrationState } from '../hydration/hydrator'
 import { ids } from '../lexicon/lexicons'
@@ -19,8 +20,7 @@ import {
   ThreadgateView,
 } from '../lexicon/types/app/bsky/feed/defs'
 import { ListView, ListViewBasic } from '../lexicon/types/app/bsky/graph/defs'
-import { creatorFromUri, parseThreadGate } from './util'
-import { mapDefined } from '@atproto/common'
+import { creatorFromUri, parseThreadGate, cidFromBlobJson } from './util'
 import { isListRule } from '../lexicon/types/app/bsky/feed/threadgate'
 import { isSelfLabels } from '../lexicon/types/com/atproto/label/defs'
 import {
@@ -92,7 +92,7 @@ export class Views {
         ? this.imgUriBuilder.getPresetUri(
             'banner',
             did,
-            actor.profile.banner.ref,
+            cidFromBlobJson(actor.profile.banner),
           )
         : undefined,
       followersCount: profileAggs?.followers,
@@ -141,7 +141,7 @@ export class Views {
         ? this.imgUriBuilder.getPresetUri(
             'avatar',
             did,
-            actor.profile.avatar.ref,
+            cidFromBlobJson(actor.profile.avatar),
           )
         : undefined,
       viewer: this.profileViewer(did, state),
@@ -217,14 +217,14 @@ export class Views {
     const creator = new AtUri(uri).hostname
     return {
       uri,
-      cid: list.cid.toString(),
+      cid: list.cid,
       name: list.record.name,
       purpose: list.record.purpose,
       avatar: list.record.avatar
         ? this.imgUriBuilder.getPresetUri(
             'avatar',
             creator,
-            list.record.avatar.ref,
+            cidFromBlobJson(list.record.avatar),
           )
         : undefined,
       indexedAt: list.sortedAt.toISOString(),
@@ -293,7 +293,7 @@ export class Views {
 
     return {
       uri,
-      cid: feedgen.cid.toString(),
+      cid: feedgen.cid,
       did: feedgen.record.did,
       creator,
       displayName: feedgen.record.displayName,
@@ -303,7 +303,7 @@ export class Views {
         ? this.imgUriBuilder.getPresetUri(
             'avatar',
             creatorDid,
-            feedgen.record.avatar.ref,
+            cidFromBlobJson(feedgen.record.avatar),
           )
         : undefined,
       likeCount: aggs?.likes,
@@ -321,7 +321,7 @@ export class Views {
     if (!gate) return
     return {
       uri,
-      cid: gate.cid.toString(),
+      cid: gate.cid,
       record: gate.record,
       lists: mapDefined(gate.record.allow ?? [], (rule) => {
         if (!isListRule(rule)) return
@@ -348,13 +348,13 @@ export class Views {
       ...(state.labels?.get(uri) ?? []),
       ...this.selfLabels({
         uri,
-        cid: post.cid.toString(),
+        cid: post.cid,
         record: post.record,
       }),
     ]
     return {
       uri,
-      cid: post.cid.toString(),
+      cid: post.cid,
       author,
       record: post.record,
       embed:
@@ -614,12 +614,12 @@ export class Views {
       thumb: this.imgUriBuilder.getPresetUri(
         'feed_thumbnail',
         did,
-        img.image.ref,
+        cidFromBlobJson(img.image),
       ),
       fullsize: this.imgUriBuilder.getPresetUri(
         'feed_fullsize',
         did,
-        img.image.ref,
+        cidFromBlobJson(img.image),
       ),
       alt: img.alt,
       aspectRatio: img.aspectRatio,
@@ -639,7 +639,11 @@ export class Views {
         title,
         description,
         thumb: thumb
-          ? this.imgUriBuilder.getPresetUri('feed_thumbnail', did, thumb.ref)
+          ? this.imgUriBuilder.getPresetUri(
+              'feed_thumbnail',
+              did,
+              cidFromBlobJson(thumb),
+            )
           : undefined,
       },
     }
@@ -816,13 +820,13 @@ export class Views {
     const labels = state.labels?.get(notif.uri) ?? []
     const selfLabels = this.selfLabels({
       uri: notif.uri,
-      cid: recordInfo.cid.toString(),
+      cid: recordInfo.cid,
       record: recordInfo.record,
     })
     const indexedAt = notif.timestamp.toDate().toISOString()
     return {
       uri: notif.uri,
-      cid: recordInfo.cid.toString(),
+      cid: recordInfo.cid,
       author,
       reason: notif.reason,
       reasonSubject: notif.reasonSubject || undefined,

packages/bsky/src/views/util.ts

@@ -1,4 +1,5 @@
 import { AtUri } from '@atproto/syntax'
+import { BlobRef } from '@atproto/lexicon'
 import { Record as PostRecord } from '../lexicon/types/app/bsky/feed/post'
 import {
   Record as GateRecord,
@@ -50,3 +51,14 @@ type ParsedThreadGate = {
   allowFollowing?: boolean
   allowListUris?: string[]
 }
+
+export const cidFromBlobJson = (json: BlobRef) => {
+  if (json instanceof BlobRef) {
+    return json.ref.toString()
+  }
+  // @NOTE below handles the fact that parseRecordBytes() produces raw json rather than lexicon values
+  if (json['$type'] === 'blob') {
+    return (json['ref']?.['$link'] ?? '') as string
+  }
+  return (json['cid'] ?? '') as string
+}