feat: egress client - ucanto integration

.gitignore

@@ -2,5 +2,5 @@ node_modules
 dist
 .mf
 .env
-.dev.vars
+.dev.vars*
 .wrangler

README.md

@@ -101,7 +101,7 @@ npm run test:unit
 
 **Integration Tests**
 ```sh
-TBD
+npm run test:integration
 ```
 
 ## Deployment

package.json

@@ -2,43 +2,47 @@
   "name": "freeway",
   "version": "2.21.0",
   "description": "An IPFS gateway for accessing UnixFS data via CAR CIDs",
-  "main": "src/index.js",
-  "types": "dist/src/index.d.ts",
+  "keywords": [
+    "IPFS",
+    "gateway",
+    "CAR",
+    "CID",
+    "IPLD",
+    "UnixFS"
+  ],
+  "license": "Apache-2.0 OR MIT",
+  "author": "Alan Shaw",
   "type": "module",
   "exports": {
     ".": {
       "import": "./src/index.js",
       "types": "./dist/src/index.d.ts"
     }
   },
+  "main": "src/index.js",
+  "types": "dist/src/index.d.ts",
   "scripts": {
-    "prepare": "npm run build",
-    "start": "npm run dev",
-    "dev": "npm run build:debug && miniflare dist/worker.mjs --watch --debug -m --r2-persist --global-async-io --global-timers",
     "build": "esbuild --bundle src/index.js --format=esm --external:node:buffer --external:node:events --external:node:async_hooks --sourcemap --minify --outfile=dist/worker.mjs && npm run build:tsc",
     "build:debug": "esbuild --bundle src/index.js --format=esm --external:node:buffer --external:node:events --external:node:async_hooks --outfile=dist/worker.mjs",
     "build:tsc": "tsc --build",
-    "test:miniflare": "npm run build:debug && mocha --experimental-vm-modules --recursive test/miniflare/**/*.spec.js",
-    "test:unit": "npm run build:debug && mocha --experimental-vm-modules --recursive test/unit/**/*.spec.js",
-    "test:integration": "npm run build:debug && mocha --experimental-vm-modules --recursive test/integration/**/*.spec.js --require test/fixtures/worker-fixture.js",
+    "dev": "npm run build:debug && miniflare dist/worker.mjs --watch --debug -m --r2-persist --global-async-io --global-timers",
     "lint": "standard",
-    "lint:fix": "standard --fix"
+    "lint:fix": "standard --fix",
+    "prepare": "npm run build",
+    "start": "npm run dev",
+    "test:integration": "npm run build:debug && mocha --experimental-vm-modules --recursive test/integration/**/*.spec.js --require test/fixtures/worker-fixture.js",
+    "test:miniflare": "npm run build:debug && mocha --experimental-vm-modules --recursive test/miniflare/**/*.spec.js",
+    "test:unit": "npm run build:debug && mocha --experimental-vm-modules --recursive test/unit/**/*.spec.js"
   },
-  "keywords": [
-    "IPFS",
-    "gateway",
-    "CAR",
-    "CID",
-    "IPLD",
-    "UnixFS"
-  ],
-  "author": "Alan Shaw",
-  "license": "Apache-2.0 OR MIT",
   "dependencies": {
     "@microlabs/otel-cf-workers": "^1.0.0-rc.48",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/sdk-trace-base": "^1.27.0",
+    "@ucanto/client": "^9.0.1",
+    "@ucanto/principal": "^9.0.1",
+    "@ucanto/transport": "^9.1.1",
     "@web3-storage/blob-fetcher": "^2.3.1",
+    "@web3-storage/capabilities": "^17.4.1",
     "@web3-storage/gateway-lib": "^5.1.2",
     "dagula": "^8.0.0",
     "http-range-parse": "^1.0.0",
@@ -51,7 +55,6 @@
     "@types/mocha": "^10.0.9",
     "@types/node-fetch": "^2.6.11",
     "@types/sinon": "^17.0.3",
-    "@ucanto/principal": "^9.0.1",
     "@web3-storage/content-claims": "^5.0.0",
     "@web3-storage/public-bucket": "^1.1.0",
     "@web3-storage/upload-client": "^16.1.1",
@@ -67,7 +70,7 @@
     "standard": "^17.1.0",
     "tree-kill": "^1.2.2",
     "typescript": "^5.6.3",
-    "wrangler": "^3.84.1"
+    "wrangler": "^3.86.1"
   },
   "standard": {
     "ignore": [

scripts/delegate-serve.js

@@ -1,38 +1,73 @@
 import sade from 'sade'
 import { getClient } from '@web3-storage/w3cli/lib.js'
-import * as ed25519 from '@ucanto/principal/ed25519'
-import * as serve from '../src/capabilities/serve.js'
+import { Space } from '@web3-storage/capabilities'
 
-const cli = sade('delegate-serve.js <space> [token]')
+const cli = sade('delegate-serve.js [space] [token] [accountDID] [gatewayDID]')
 
 cli
+  .option('--space', 'The space DID to delegate. If not provided, a new space will be created.')
+  .option('--token', 'The auth token to use. If not provided, the delegation will not be authenticated.')
+  .option('--accountDID', 'The account DID to use when creating a new space.')
+  .option('--gatewayDID', 'The gateway DID to use when delegating the space/content/serve capability. Defaults to did:web:staging.w3s.link.')
   .describe(
-    `Delegates ${serve.star.can} to the Gateway for <space>, with an optional token. Outputs a base64url string suitable for the stub_delegation query parameter. Pipe the output to pbcopy or similar for the quickest workflow.`
+    `Delegates ${Space.contentServe.can} to the Gateway for a test space generated by the script, with an optional auth token. Outputs a base64url string suitable for the stub_delegation query parameter. Pipe the output to pbcopy or similar for the quickest workflow.`
   )
-  .action(async (space, token) => {
+  .action(async (space, token, accountDID, gatewayDID, options) => {
+    const { space: spaceOption, token: tokenOption, accountDID: accountDIDOption, gatewayDID: gatewayDIDOption } = options
+    space = spaceOption || undefined
+    token = tokenOption || undefined
+    accountDID = accountDIDOption || undefined
+    gatewayDID = gatewayDIDOption || 'did:web:staging.w3s.link'
     const client = await getClient()
 
-    const gatewayIdentity = (await ed25519.Signer.generate()).withDID(
-      'did:web:w3s.link'
-    )
-
-    const delegation = await serve.star.delegate({
-      issuer: client.agent.issuer,
-      audience: gatewayIdentity,
-      with: space,
-      nb: { token: token ?? null },
-      expiration: Infinity,
-      proofs: client.proofs([
+    let spaceDID
+    let proofs = []
+    if (!space) {
+      const provider = /** @type {`did:web:${string}`} */ (client.defaultProvider())
+      const account = client.accounts()[accountDID]
+      const newSpace = await client.agent.createSpace('test')
+      const provision = await account.provision(newSpace.did(), { provider })
+      if (provision.error) throw provision.error
+      await newSpace.save()
+      const authProof = await newSpace.createAuthorization(client.agent)
+      proofs = [authProof]
+      spaceDID = newSpace.did()
+    } else {
+      client.addSpace(space)
+      spaceDID = space
+      proofs = client.proofs([
         {
-          can: serve.star.can,
-          with: space
+          can: Space.contentServe.can,
+          with: spaceDID
         }
       ])
+    }
+
+    /** @type {import('@ucanto/client').Principal<`did:${string}:${string}`>} */
+    const gatewayIdentity = {
+      did: () => gatewayDID
+    }
+
+    // @ts-expect-error - The client still needs to be updated to support the capability type
+    const delegation = await client.createDelegation(gatewayIdentity, [Space.contentServe.can], {
+      expiration: Infinity,
+      proofs
+    })
+
+    await client.capability.access.delegate({
+      delegations: [delegation]
     })
 
     const carResult = await delegation.archive()
     if (carResult.error) throw carResult.error
-    process.stdout.write(Buffer.from(carResult.ok).toString('base64url'))
+    const base64Url = Buffer.from(carResult.ok).toString('base64url')
+    process.stdout.write(`Agent Proofs: ${proofs.flatMap(p => p.capabilities).map(c => `${c.can} with ${c.with}`).join('\n')}\n`)
+    process.stdout.write(`Issuer: ${client.agent.issuer.did()}\n`)
+    process.stdout.write(`Audience: ${gatewayIdentity.did()}\n`)
+    process.stdout.write(`Space: ${spaceDID}\n`)
+    process.stdout.write(`Token: ${token ?? 'none'}\n`)
+    process.stdout.write(`Delegation: ${delegation.capabilities.map(c => `${c.can} with ${c.with}`).join('\n')}\n`)
+    process.stdout.write(`Stubs: stub_space=${spaceDID}&stub_delegation=${base64Url}&authToken=${token ?? ''}\n`)
   })
 
 cli.parse(process.argv)

src/bindings.d.ts

@@ -4,22 +4,15 @@ import { Environment as CarBlockEnvironment } from './middleware/withCarBlockHan
 import { Environment as ContentClaimsDagulaEnvironment } from './middleware/withCarBlockHandler.types.ts'
 import { Environment as EgressTrackerEnvironment } from './middleware/withEgressTracker.types.ts'
 import { UnknownLink } from 'multiformats'
+import { DIDKey } from '@ucanto/principal/ed25519'
+
 export interface Environment
   extends CarBlockEnvironment,
     RateLimiterEnvironment,
     ContentClaimsDagulaEnvironment,
     EgressTrackerEnvironment {
   VERSION: string
   CONTENT_CLAIMS_SERVICE_URL?: string
-  ACCOUNTING_SERVICE_URL: string
   HONEYCOMB_API_KEY: string
-}
-
-export interface AccountingService {
-  record: (resource: UnknownLink, bytes: number, servedAt: string) => Promise<void>
-  getTokenMetadata: (token: string) => Promise<TokenMetadata | null>
-}
-
-export interface Accounting {
-  create: ({ serviceURL }: { serviceURL: string }) => AccountingService
+  FF_TELEMETRY_ENABLED: string
 }

src/index.js

@@ -20,8 +20,10 @@ import {
   withVersionHeader,
   withAuthToken,
   withCarBlockHandler,
+  withGatewayIdentity,
   withRateLimit,
   withEgressTracker,
+  withEgressClient,
   withAuthorizedSpace,
   withLocator,
   withDelegationStubs
@@ -42,45 +44,49 @@ import { NoopSpanProcessor } from '@opentelemetry/sdk-trace-base'
  * @import { Environment } from './bindings.js'
  */
 
-const handler = {
-  /** @type {Handler<Context, Environment>} */
-  fetch (request, env, ctx) {
-    console.log(request.method, request.url)
-    const middleware = composeMiddleware(
-      // Prepare the Context
-      withCdnCache,
-      withContext,
-      withCorsHeaders,
-      withVersionHeader,
-      withErrorHandler,
-      withParsedIpfsUrl,
-      createWithHttpMethod('GET', 'HEAD'),
-      withAuthToken,
-      withLocator,
-      withDelegationStubs,
+/**
+ * The middleware stack
+ */
+const middleware = composeMiddleware(
+  // Prepare the Context
+  withCdnCache,
+  withContext,
+  withCorsHeaders,
+  withVersionHeader,
+  withErrorHandler,
+  withParsedIpfsUrl,
+  createWithHttpMethod('GET', 'HEAD'),
+  withAuthToken,
+  withLocator,
+  withGatewayIdentity,
+  // TODO: replace this with a handler to fetch the real delegations
+  withDelegationStubs,
+
+  // Rate-limit requests
+  withRateLimit,
 
-      // Rate-limit requests
-      withRateLimit,
+  // Fetch CAR data - Double-check why this can't be placed after the authorized space middleware
+  withCarBlockHandler,
 
-      // Track egress bytes
-      withEgressTracker,
+  // Authorize requests
+  withAuthorizedSpace,
 
-      // Fetch data
-      withCarBlockHandler,
-      withAuthorizedSpace,
-      withContentClaimsDagula,
-      withFormatRawHandler,
-      withFormatCarHandler,
+  // Track Egress
+  withEgressClient,
+  withEgressTracker,
 
-      // Prepare the Response
-      withContentDispositionHeader,
-      withFixedLengthStream
-    )
-    return middleware(handleUnixfs)(request, env, ctx)
-  }
-}
+  // Fetch data
+  withContentClaimsDagula,
+  withFormatRawHandler,
+  withFormatCarHandler,
+
+  // Prepare the Response
+  withContentDispositionHeader,
+  withFixedLengthStream
+)
 
 /**
+ * Configure the OpenTelemetry exporter based on the environment
  *
  * @param {Environment} env
  * @param {*} _trigger
@@ -101,7 +107,41 @@ function config (env, _trigger) {
   }
 }
 
-export default instrument(handler, config)
+/**
+ * The promise to the pre-configured handler
+ *
+ * @type {Promise<Handler<Context, Environment>> | null}
+ */
+let handlerPromise = null
+
+/**
+ * Pre-configure the handler based on the environment.
+ *
+ * @param {Environment} env
+ * @returns {Promise<Handler<Context, Environment>>}
+ */
+async function initializeHandler (env) {
+  const baseHandler = middleware(handleUnixfs)
+  const finalHandler = env.FF_TELEMETRY_ENABLED === 'true'
+    ? instrument(baseHandler, config)
+    : baseHandler
+  return finalHandler
+}
+
+const handler = {
+  /** @type {Handler<Context, Environment>} */
+  async fetch (request, env, ctx) {
+    console.log(request.method, request.url)
+    // Initialize the handler only once and reuse the promise
+    if (!handlerPromise) {
+      handlerPromise = initializeHandler(env)
+    }
+    const handler = await handlerPromise
+    return handler(request, env, ctx)
+  }
+}
+
+export default handler
 
 /**
  * @type {Middleware<BlockContext & UnixfsContext & IpfsUrlContext, BlockContext & UnixfsContext & IpfsUrlContext, Environment>}

src/middleware/index.js

@@ -6,4 +6,6 @@ export { withVersionHeader } from './withVersionHeader.js'
 export { withAuthorizedSpace } from './withAuthorizedSpace.js'
 export { withLocator } from './withLocator.js'
 export { withEgressTracker } from './withEgressTracker.js'
+export { withEgressClient } from './withEgressClient.js'
 export { withDelegationStubs } from './withDelegationStubs.js'
+export { withGatewayIdentity } from './withGatewayIdentity.js'