From c2d0578b5a9517d4c5804cff9dcc6c4ea5c4e7c0 Mon Sep 17 00:00:00 2001 From: devin ivy Date: Tue, 9 Jan 2024 15:40:35 -0500 Subject: [PATCH] Bsky appview sync service (#2031) * init bsky-sync * add bsync models and config * rename bsky-sync to bsync * protos and gen for bsync service * start roughing-out bsync routes * adjust bsync model, validation * bsync auth, context, notify * implement bsync scan mute ops, listen for mute op event * setup basic bsync tests, misc fixes * rename some files * reorg bsync server routes * reorg bsync server routes * tests * test input validation to addMuteOperation * add db stats to bsync * add bsync service * redact bsync auth header from logs * upgrade typescript to v5.3 * prettier on codegened bsync files --- package.json | 6 +- packages/bsync/README.md | 15 + packages/bsync/babel.config.js | 3 + packages/bsync/bin/migration-create.ts | 38 ++ packages/bsync/buf.gen.yaml | 13 + packages/bsync/build.js | 18 + packages/bsync/jest.config.js | 6 + packages/bsync/package.json | 52 ++ packages/bsync/proto/bsync.proto | 54 ++ packages/bsync/src/client.ts | 25 + packages/bsync/src/config.ts | 86 ++++ packages/bsync/src/context.ts | 42 ++ packages/bsync/src/db/index.ts | 194 ++++++++ .../db/migrations/20240108T220751294Z-init.ts | 26 + packages/bsync/src/db/migrations/index.ts | 5 + packages/bsync/src/db/migrations/provider.ts | 8 + packages/bsync/src/db/schema/index.ts | 9 + packages/bsync/src/db/schema/mute_item.ts | 13 + packages/bsync/src/db/schema/mute_op.ts | 18 + packages/bsync/src/db/types.ts | 15 + packages/bsync/src/gen/bsync_connect.ts | 54 ++ packages/bsync/src/gen/bsync_pb.ts | 453 +++++++++++++++++ packages/bsync/src/index.ts | 91 ++++ packages/bsync/src/logger.ts | 22 + .../bsync/src/routes/add-mute-operation.ts | 158 ++++++ packages/bsync/src/routes/auth.ts | 15 + packages/bsync/src/routes/index.ts | 18 + .../bsync/src/routes/scan-mute-operations.ts | 69 +++ packages/bsync/tests/mutes.test.ts | 350 +++++++++++++ packages/bsync/tsconfig.build.json | 4 + packages/bsync/tsconfig.json | 14 + pnpm-lock.yaml | 463 ++++++++++++++---- services/bsync/Dockerfile | 49 ++ services/bsync/index.js | 27 + services/bsync/package.json | 8 + tsconfig.json | 1 + 36 files changed, 2333 insertions(+), 109 deletions(-) create mode 100644 packages/bsync/README.md create mode 100644 packages/bsync/babel.config.js create mode 100644 packages/bsync/bin/migration-create.ts create mode 100644 packages/bsync/buf.gen.yaml create mode 100644 packages/bsync/build.js create mode 100644 packages/bsync/jest.config.js create mode 100644 packages/bsync/package.json create mode 100644 packages/bsync/proto/bsync.proto create mode 100644 packages/bsync/src/client.ts create mode 100644 packages/bsync/src/config.ts create mode 100644 packages/bsync/src/context.ts create mode 100644 packages/bsync/src/db/index.ts create mode 100644 packages/bsync/src/db/migrations/20240108T220751294Z-init.ts create mode 100644 packages/bsync/src/db/migrations/index.ts create mode 100644 packages/bsync/src/db/migrations/provider.ts create mode 100644 packages/bsync/src/db/schema/index.ts create mode 100644 packages/bsync/src/db/schema/mute_item.ts create mode 100644 packages/bsync/src/db/schema/mute_op.ts create mode 100644 packages/bsync/src/db/types.ts create mode 100644 packages/bsync/src/gen/bsync_connect.ts create mode 100644 packages/bsync/src/gen/bsync_pb.ts create mode 100644 packages/bsync/src/index.ts create mode 100644 packages/bsync/src/logger.ts create mode 100644 packages/bsync/src/routes/add-mute-operation.ts create mode 100644 packages/bsync/src/routes/auth.ts create mode 100644 packages/bsync/src/routes/index.ts create mode 100644 packages/bsync/src/routes/scan-mute-operations.ts create mode 100644 packages/bsync/tests/mutes.test.ts create mode 100644 packages/bsync/tsconfig.build.json create mode 100644 packages/bsync/tsconfig.json create mode 100644 services/bsync/Dockerfile create mode 100644 services/bsync/index.js create mode 100644 services/bsync/package.json diff --git a/package.json b/package.json index c7c74e4a615..7fe531a530f 100644 --- a/package.json +++ b/package.json @@ -31,8 +31,8 @@ "@swc/jest": "^0.2.24", "@types/jest": "^28.1.4", "@types/node": "^18.0.0", - "@typescript-eslint/eslint-plugin": "^5.38.1", - "@typescript-eslint/parser": "^5.38.1", + "@typescript-eslint/eslint-plugin": "^6.14.0", + "@typescript-eslint/parser": "^6.14.0", "babel-eslint": "^10.1.0", "dotenv": "^16.0.3", "esbuild": "^0.14.48", @@ -48,7 +48,7 @@ "prettier": "^2.7.1", "prettier-config-standard": "^5.0.0", "ts-node": "^10.8.2", - "typescript": "^4.8.4" + "typescript": "^5.3.3" }, "workspaces": { "packages": [ diff --git a/packages/bsync/README.md b/packages/bsync/README.md new file mode 100644 index 00000000000..413bcf81eff --- /dev/null +++ b/packages/bsync/README.md @@ -0,0 +1,15 @@ +# @atproto/bsync: Synchronizing Service for the Bluesky AppView + +This is an optional service that may be used to synchronize certain state between otherwise independent AppViews. + +[![NPM](https://img.shields.io/npm/v/@atproto/bsync)](https://www.npmjs.com/package/@atproto/bsync) +[![Github CI Status](https://github.com/bluesky-social/atproto/actions/workflows/repo.yaml/badge.svg)](https://github.com/bluesky-social/atproto/actions/workflows/repo.yaml) + +## License + +This project is dual-licensed under MIT and Apache 2.0 terms: + +- MIT license ([LICENSE-MIT.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-MIT.txt) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0, ([LICENSE-APACHE.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-APACHE.txt) or http://www.apache.org/licenses/LICENSE-2.0) + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. diff --git a/packages/bsync/babel.config.js b/packages/bsync/babel.config.js new file mode 100644 index 00000000000..ee58f35df11 --- /dev/null +++ b/packages/bsync/babel.config.js @@ -0,0 +1,3 @@ +module.exports = { + presets: [['@babel/preset-env']], +} diff --git a/packages/bsync/bin/migration-create.ts b/packages/bsync/bin/migration-create.ts new file mode 100644 index 00000000000..b51c536c4f2 --- /dev/null +++ b/packages/bsync/bin/migration-create.ts @@ -0,0 +1,38 @@ +#!/usr/bin/env ts-node + +import * as fs from 'fs/promises' +import * as path from 'path' + +export async function main() { + const now = new Date() + const prefix = now.toISOString().replace(/[^a-z0-9]/gi, '') // Order of migrations matches alphabetical order of their names + const name = process.argv[2] + if (!name || !name.match(/^[a-z0-9-]+$/)) { + process.exitCode = 1 + return console.error( + 'Must pass a migration name consisting of lowercase digits, numbers, and dashes.', + ) + } + const filename = `${prefix}-${name}` + const dir = path.join(__dirname, '..', 'src', 'db', 'migrations') + + await fs.writeFile(path.join(dir, `${filename}.ts`), template, { flag: 'wx' }) + await fs.writeFile( + path.join(dir, 'index.ts'), + `export * as _${prefix} from './${filename}'\n`, + { flag: 'a' }, + ) +} + +const template = `import { Kysely } from 'kysely' + +export async function up(db: Kysely): Promise { + // Migration code +} + +export async function down(db: Kysely): Promise { + // Migration code +} +` + +main() diff --git a/packages/bsync/buf.gen.yaml b/packages/bsync/buf.gen.yaml new file mode 100644 index 00000000000..9cccf918873 --- /dev/null +++ b/packages/bsync/buf.gen.yaml @@ -0,0 +1,13 @@ +version: v1 +plugins: + - plugin: es + opt: + - target=ts + - import_extension=.ts + + out: src/gen + - plugin: connect-es + opt: + - target=ts + - import_extension=.ts + out: src/gen diff --git a/packages/bsync/build.js b/packages/bsync/build.js new file mode 100644 index 00000000000..25452318508 --- /dev/null +++ b/packages/bsync/build.js @@ -0,0 +1,18 @@ +const { nodeExternalsPlugin } = require('esbuild-node-externals') + +const buildShallow = + process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true' + +require('esbuild').build({ + logLevel: 'info', + entryPoints: ['src/index.ts'], + bundle: true, + sourcemap: true, + outdir: 'dist', + platform: 'node', + external: [ + // Referenced in pg driver, but optional and we don't use it + 'pg-native', + ], + plugins: buildShallow ? [nodeExternalsPlugin()] : [], +}) diff --git a/packages/bsync/jest.config.js b/packages/bsync/jest.config.js new file mode 100644 index 00000000000..9abb11c632d --- /dev/null +++ b/packages/bsync/jest.config.js @@ -0,0 +1,6 @@ +const base = require('../../jest.config.base.js') + +module.exports = { + ...base, + displayName: 'Bsync', +} diff --git a/packages/bsync/package.json b/packages/bsync/package.json new file mode 100644 index 00000000000..609c991f826 --- /dev/null +++ b/packages/bsync/package.json @@ -0,0 +1,52 @@ +{ + "name": "@atproto/bsync", + "version": "0.0.0", + "license": "MIT", + "description": "Sychronizing service for app.bsky App View (Bluesky API)", + "keywords": [ + "atproto", + "bluesky" + ], + "homepage": "https://atproto.com", + "repository": { + "type": "git", + "url": "https://github.com/bluesky-social/atproto", + "directory": "packages/bsync" + }, + "main": "src/index.ts", + "publishConfig": { + "main": "dist/index.js", + "types": "dist/index.d.ts" + }, + "scripts": { + "build": "node ./build.js", + "postbuild": "tsc --build tsconfig.build.json", + "update-main-to-dist": "node ../../update-main-to-dist.js packages/bsync", + "start": "node --enable-source-maps dist/bin.js", + "test": "../dev-infra/with-test-db.sh jest", + "test:log": "tail -50 test.log | pino-pretty", + "test:updateSnapshot": "jest --updateSnapshot", + "migration:create": "ts-node ./bin/migration-create.ts", + "buf:gen": "buf generate proto" + }, + "dependencies": { + "@atproto/common": "workspace:^", + "@atproto/syntax": "workspace:^", + "@bufbuild/protobuf": "^1.5.0", + "@connectrpc/connect": "^1.1.4", + "@connectrpc/connect-express": "^1.1.4", + "@connectrpc/connect-node": "^1.1.4", + "http-terminator": "^3.2.0", + "kysely": "^0.22.0", + "pg": "^8.10.0", + "pino": "^8.15.0", + "pino-http": "^8.2.1", + "typed-emitter": "^2.1.0" + }, + "devDependencies": { + "@bufbuild/buf": "^1.28.1", + "@bufbuild/protoc-gen-es": "^1.5.0", + "@connectrpc/protoc-gen-connect-es": "^1.1.4", + "@types/pg": "^8.6.6" + } +} diff --git a/packages/bsync/proto/bsync.proto b/packages/bsync/proto/bsync.proto new file mode 100644 index 00000000000..d8b49dc56f4 --- /dev/null +++ b/packages/bsync/proto/bsync.proto @@ -0,0 +1,54 @@ +syntax = "proto3"; + +package bsync; +option go_package = "./;bsync"; + +// +// Sync +// + + +message MuteOperation { + enum Type { + ADD = 0; + REMOVE = 1; + CLEAR = 2; + } + string id = 1; + Type type = 2; + string actor_did = 3; + string subject = 4; +} + +message AddMuteOperationRequest { + MuteOperation.Type type = 1; + string actor_did = 2; + string subject = 3; +} + +message AddMuteOperationResponse { + MuteOperation operation = 1; +} + +message ScanMuteOperationsRequest { + string cursor = 1; + int32 limit = 2; +} + +message ScanMuteOperationsResponse { + repeated MuteOperation operations = 1; + string cursor = 2; +} + +// Ping +message PingRequest {} +message PingResponse {} + + +service Service { + // Sync + rpc AddMuteOperation(AddMuteOperationRequest) returns (AddMuteOperationResponse); + rpc ScanMuteOperations(ScanMuteOperationsRequest) returns (ScanMuteOperationsResponse); + // Ping + rpc Ping(PingRequest) returns (PingResponse); +} diff --git a/packages/bsync/src/client.ts b/packages/bsync/src/client.ts new file mode 100644 index 00000000000..ea38f3d01ee --- /dev/null +++ b/packages/bsync/src/client.ts @@ -0,0 +1,25 @@ +import { + Interceptor, + PromiseClient, + createPromiseClient, +} from '@connectrpc/connect' +import { + ConnectTransportOptions, + createConnectTransport, +} from '@connectrpc/connect-node' +import { Service } from './gen/bsync_connect' + +export type BsyncClient = PromiseClient + +export const createClient = (opts: ConnectTransportOptions): BsyncClient => { + const transport = createConnectTransport(opts) + return createPromiseClient(Service, transport) +} + +export const authWithApiKey = + (apiKey: string): Interceptor => + (next) => + (req) => { + req.header.set('authorization', `Bearer ${apiKey}`) + return next(req) + } diff --git a/packages/bsync/src/config.ts b/packages/bsync/src/config.ts new file mode 100644 index 00000000000..ee2385c0a7a --- /dev/null +++ b/packages/bsync/src/config.ts @@ -0,0 +1,86 @@ +import assert from 'node:assert' +import { envInt, envStr, envList } from '@atproto/common' + +export const envToCfg = (env: ServerEnvironment): ServerConfig => { + const serviceCfg: ServerConfig['service'] = { + port: env.port ?? 2585, + version: env.version ?? 'unknown', + longPollTimeoutMs: env.longPollTimeoutMs ?? 10000, + } + + assert(env.dbUrl, 'missing postgres url') + const dbCfg: ServerConfig['db'] = { + url: env.dbUrl, + schema: env.dbSchema, + poolSize: env.dbPoolSize, + poolMaxUses: env.dbPoolMaxUses, + poolIdleTimeoutMs: env.dbPoolIdleTimeoutMs, + } + + assert(env.apiKeys.length > 0, 'missing api keys') + const authCfg: ServerConfig['auth'] = { + apiKeys: new Set(env.apiKeys), + } + + return { + service: serviceCfg, + db: dbCfg, + auth: authCfg, + } +} + +export type ServerConfig = { + service: ServiceConfig + db: DatabaseConfig + auth: AuthConfig +} + +type ServiceConfig = { + port: number + version?: string + longPollTimeoutMs: number +} + +type DatabaseConfig = { + url: string + schema?: string + poolSize?: number + poolMaxUses?: number + poolIdleTimeoutMs?: number +} + +type AuthConfig = { + apiKeys: Set +} + +export const readEnv = (): ServerEnvironment => { + return { + // service + port: envInt('BSYNC_PORT'), + version: envStr('BSYNC_VERSION'), + longPollTimeoutMs: envInt('BSYNC_LONG_POLL_TIMEOUT_MS'), + // database + dbUrl: envStr('BSYNC_DB_POSTGRES_URL'), + dbSchema: envStr('BSYNC_DB_POSTGRES_SCHEMA'), + dbPoolSize: envInt('BSYNC_DB_POOL_SIZE'), + dbPoolMaxUses: envInt('BSYNC_DB_POOL_MAX_USES'), + dbPoolIdleTimeoutMs: envInt('BSYNC_DB_POOL_IDLE_TIMEOUT_MS'), + // secrets + apiKeys: envList('BSYNC_API_KEYS'), + } +} + +export type ServerEnvironment = { + // service + port?: number + version?: string + longPollTimeoutMs?: number + // database + dbUrl?: string + dbSchema?: string + dbPoolSize?: number + dbPoolMaxUses?: number + dbPoolIdleTimeoutMs?: number + // secrets + apiKeys: string[] +} diff --git a/packages/bsync/src/context.ts b/packages/bsync/src/context.ts new file mode 100644 index 00000000000..ff6e25bd08b --- /dev/null +++ b/packages/bsync/src/context.ts @@ -0,0 +1,42 @@ +import TypedEventEmitter from 'typed-emitter' +import { ServerConfig } from './config' +import Database from './db' +import { createMuteOpChannel } from './db/schema/mute_op' +import { EventEmitter } from 'stream' + +export type AppContextOptions = { + db: Database + cfg: ServerConfig +} + +export class AppContext { + db: Database + cfg: ServerConfig + events: TypedEventEmitter + + constructor(opts: AppContextOptions) { + this.db = opts.db + this.cfg = opts.cfg + this.events = new EventEmitter() as TypedEventEmitter + } + + static async fromConfig( + cfg: ServerConfig, + overrides?: Partial, + ): Promise { + const db = new Database({ + url: cfg.db.url, + schema: cfg.db.schema, + poolSize: cfg.db.poolSize, + poolMaxUses: cfg.db.poolMaxUses, + poolIdleTimeoutMs: cfg.db.poolIdleTimeoutMs, + }) + return new AppContext({ db, cfg, ...overrides }) + } +} + +export default AppContext + +export type AppEvents = { + [createMuteOpChannel]: () => void +} diff --git a/packages/bsync/src/db/index.ts b/packages/bsync/src/db/index.ts new file mode 100644 index 00000000000..7922c421bf5 --- /dev/null +++ b/packages/bsync/src/db/index.ts @@ -0,0 +1,194 @@ +import assert from 'assert' +import { + Kysely, + PostgresDialect, + Migrator, + KyselyPlugin, + PluginTransformQueryArgs, + PluginTransformResultArgs, + RootOperationNode, + QueryResult, + UnknownRow, +} from 'kysely' +import TypedEmitter from 'typed-emitter' +import { Pool as PgPool, types as pgTypes } from 'pg' +import DatabaseSchema, { DatabaseSchemaType } from './schema' +import { PgOptions } from './types' +import { dbLogger } from '../logger' +import { EventEmitter } from 'stream' +import * as migrations from './migrations' +import { DbMigrationProvider } from './migrations/provider' + +export class Database { + pool: PgPool + db: DatabaseSchema + migrator: Migrator + txEvt = new EventEmitter() as TxnEmitter + destroyed = false + + constructor( + public opts: PgOptions, + instances?: { db: DatabaseSchema; pool: PgPool }, + ) { + // if instances are provided, use those + if (instances) { + this.db = instances.db + this.pool = instances.pool + return + } + + // else create a pool & connect + const { schema, url } = opts + const pool = + opts.pool ?? + new PgPool({ + connectionString: url, + max: opts.poolSize, + maxUses: opts.poolMaxUses, + idleTimeoutMillis: opts.poolIdleTimeoutMs, + }) + + // Select count(*) and other pg bigints as js integer + pgTypes.setTypeParser(pgTypes.builtins.INT8, (n) => parseInt(n, 10)) + + // Setup schema usage, primarily for test parallelism (each test suite runs in its own pg schema) + if (schema && !/^[a-z_]+$/i.test(schema)) { + throw new Error(`Postgres schema must only contain [A-Za-z_]: ${schema}`) + } + + pool.on('error', onPoolError) + pool.on('connect', (client) => { + client.on('error', onClientError) + if (schema) { + // Shared objects such as extensions will go in the public schema + client.query(`SET search_path TO "${schema}",public;`) + } + }) + + this.pool = pool + this.db = new Kysely({ + dialect: new PostgresDialect({ pool }), + }) + this.migrator = new Migrator({ + db: this.db, + migrationTableSchema: opts.schema, + provider: new DbMigrationProvider(migrations), + }) + } + + get schema(): string | undefined { + return this.opts.schema + } + + get isTransaction() { + return this.db.isTransaction + } + + assertTransaction() { + assert(this.isTransaction, 'Transaction required') + } + + assertNotTransaction() { + assert(!this.isTransaction, 'Cannot be in a transaction') + } + + async transaction(fn: (db: Database) => Promise): Promise { + const leakyTxPlugin = new LeakyTxPlugin() + const { dbTxn, txRes } = await this.db + .withPlugin(leakyTxPlugin) + .transaction() + .execute(async (txn) => { + const dbTxn = new Database(this.opts, { + db: txn, + pool: this.pool, + }) + const txRes = await fn(dbTxn) + .catch(async (err) => { + leakyTxPlugin.endTx() + // ensure that all in-flight queries are flushed & the connection is open + await dbTxn.db.getExecutor().provideConnection(noopAsync) + throw err + }) + .finally(() => leakyTxPlugin.endTx()) + return { dbTxn, txRes } + }) + dbTxn?.txEvt.emit('commit') + return txRes + } + + onCommit(fn: () => void) { + this.assertTransaction() + this.txEvt.once('commit', fn) + } + + async close(): Promise { + if (this.destroyed) return + await this.db.destroy() + this.destroyed = true + } + + async migrateToOrThrow(migration: string) { + if (this.schema) { + await this.db.schema.createSchema(this.schema).ifNotExists().execute() + } + const { error, results } = await this.migrator.migrateTo(migration) + if (error) { + throw error + } + if (!results) { + throw new Error('An unknown failure occurred while migrating') + } + return results + } + + async migrateToLatestOrThrow() { + if (this.schema) { + await this.db.schema.createSchema(this.schema).ifNotExists().execute() + } + const { error, results } = await this.migrator.migrateToLatest() + if (error) { + throw error + } + if (!results) { + throw new Error('An unknown failure occurred while migrating') + } + return results + } +} + +export default Database + +const onPoolError = (err: Error) => dbLogger.error({ err }, 'db pool error') +const onClientError = (err: Error) => dbLogger.error({ err }, 'db client error') + +// utils +// ------- + +class LeakyTxPlugin implements KyselyPlugin { + private txOver: boolean + + endTx() { + this.txOver = true + } + + transformQuery(args: PluginTransformQueryArgs): RootOperationNode { + if (this.txOver) { + throw new Error('tx already failed') + } + return args.node + } + + async transformResult( + args: PluginTransformResultArgs, + ): Promise> { + return args.result + } +} + +type TxnEmitter = TypedEmitter + +type TxnEvents = { + commit: () => void +} + +const noopAsync = async () => {} diff --git a/packages/bsync/src/db/migrations/20240108T220751294Z-init.ts b/packages/bsync/src/db/migrations/20240108T220751294Z-init.ts new file mode 100644 index 00000000000..0018c4c40b2 --- /dev/null +++ b/packages/bsync/src/db/migrations/20240108T220751294Z-init.ts @@ -0,0 +1,26 @@ +import { Kysely, sql } from 'kysely' + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('mute_op') + .addColumn('id', 'bigserial', (col) => col.primaryKey()) + .addColumn('type', 'int2', (col) => col.notNull()) // integer enum: 0->add, 1->remove, 2->clear + .addColumn('actorDid', 'varchar', (col) => col.notNull()) + .addColumn('subject', 'varchar', (col) => col.notNull()) + .addColumn('createdAt', 'timestamptz', (col) => + col.notNull().defaultTo(sql`CURRENT_TIMESTAMP`), + ) + .execute() + await db.schema + .createTable('mute_item') + .addColumn('actorDid', 'varchar', (col) => col.notNull()) + .addColumn('subject', 'varchar', (col) => col.notNull()) + .addColumn('fromId', 'bigint', (col) => col.notNull()) + .addPrimaryKeyConstraint('mute_item_pkey', ['actorDid', 'subject']) + .execute() +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable('mute_item').execute() + await db.schema.dropTable('mute_op').execute() +} diff --git a/packages/bsync/src/db/migrations/index.ts b/packages/bsync/src/db/migrations/index.ts new file mode 100644 index 00000000000..d5c2967a0df --- /dev/null +++ b/packages/bsync/src/db/migrations/index.ts @@ -0,0 +1,5 @@ +// NOTE this file can be edited by hand, but it is also appended to by the migration:create command. +// It's important that every migration is exported from here with the proper name. We'd simplify +// this with kysely's FileMigrationProvider, but it doesn't play nicely with the build process. + +export * as _20240108T220751294Z from './20240108T220751294Z-init' diff --git a/packages/bsync/src/db/migrations/provider.ts b/packages/bsync/src/db/migrations/provider.ts new file mode 100644 index 00000000000..87cde343766 --- /dev/null +++ b/packages/bsync/src/db/migrations/provider.ts @@ -0,0 +1,8 @@ +import { Migration, MigrationProvider } from 'kysely' + +export class DbMigrationProvider implements MigrationProvider { + constructor(private migrations: Record) {} + async getMigrations(): Promise> { + return this.migrations + } +} diff --git a/packages/bsync/src/db/schema/index.ts b/packages/bsync/src/db/schema/index.ts new file mode 100644 index 00000000000..0ff36ff2ca3 --- /dev/null +++ b/packages/bsync/src/db/schema/index.ts @@ -0,0 +1,9 @@ +import { Kysely } from 'kysely' +import * as muteOp from './mute_op' +import * as muteItem from './mute_item' + +export type DatabaseSchemaType = muteItem.PartialDB & muteOp.PartialDB + +export type DatabaseSchema = Kysely + +export default DatabaseSchema diff --git a/packages/bsync/src/db/schema/mute_item.ts b/packages/bsync/src/db/schema/mute_item.ts new file mode 100644 index 00000000000..ce4f44e515b --- /dev/null +++ b/packages/bsync/src/db/schema/mute_item.ts @@ -0,0 +1,13 @@ +import { Selectable } from 'kysely' + +export interface MuteItem { + actorDid: string + subject: string // did or aturi for list + fromId: number +} + +export type MuteItemEntry = Selectable + +export const tableName = 'mute_item' + +export type PartialDB = { [tableName]: MuteItem } diff --git a/packages/bsync/src/db/schema/mute_op.ts b/packages/bsync/src/db/schema/mute_op.ts new file mode 100644 index 00000000000..0efa56daf7b --- /dev/null +++ b/packages/bsync/src/db/schema/mute_op.ts @@ -0,0 +1,18 @@ +import { GeneratedAlways, Selectable } from 'kysely' +import { MuteOperation_Type } from '../../gen/bsync_pb' + +export interface MuteOp { + id: GeneratedAlways + type: MuteOperation_Type // integer enum: 0->add, 1->remove, 2->clear + actorDid: string + subject: string // did or aturi for list + createdAt: GeneratedAlways +} + +export type MuteOpEntry = Selectable + +export const tableName = 'mute_op' + +export type PartialDB = { [tableName]: MuteOp } + +export const createMuteOpChannel = 'mute_op_create' // used with listen/notify diff --git a/packages/bsync/src/db/types.ts b/packages/bsync/src/db/types.ts new file mode 100644 index 00000000000..c38271ee119 --- /dev/null +++ b/packages/bsync/src/db/types.ts @@ -0,0 +1,15 @@ +import { Pool as PgPool } from 'pg' +import { DynamicModule, RawBuilder, SelectQueryBuilder } from 'kysely' + +export type DbRef = RawBuilder | ReturnType + +export type AnyQb = SelectQueryBuilder + +export type PgOptions = { + url: string + pool?: PgPool + schema?: string + poolSize?: number + poolMaxUses?: number + poolIdleTimeoutMs?: number +} diff --git a/packages/bsync/src/gen/bsync_connect.ts b/packages/bsync/src/gen/bsync_connect.ts new file mode 100644 index 00000000000..94a37266bfd --- /dev/null +++ b/packages/bsync/src/gen/bsync_connect.ts @@ -0,0 +1,54 @@ +// @generated by protoc-gen-connect-es v1.3.0 with parameter "target=ts,import_extension=.ts" +// @generated from file bsync.proto (package bsync, syntax proto3) +/* eslint-disable */ +// @ts-nocheck + +import { + AddMuteOperationRequest, + AddMuteOperationResponse, + PingRequest, + PingResponse, + ScanMuteOperationsRequest, + ScanMuteOperationsResponse, +} from './bsync_pb.ts' +import { MethodKind } from '@bufbuild/protobuf' + +/** + * @generated from service bsync.Service + */ +export const Service = { + typeName: 'bsync.Service', + methods: { + /** + * Sync + * + * @generated from rpc bsync.Service.AddMuteOperation + */ + addMuteOperation: { + name: 'AddMuteOperation', + I: AddMuteOperationRequest, + O: AddMuteOperationResponse, + kind: MethodKind.Unary, + }, + /** + * @generated from rpc bsync.Service.ScanMuteOperations + */ + scanMuteOperations: { + name: 'ScanMuteOperations', + I: ScanMuteOperationsRequest, + O: ScanMuteOperationsResponse, + kind: MethodKind.Unary, + }, + /** + * Ping + * + * @generated from rpc bsync.Service.Ping + */ + ping: { + name: 'Ping', + I: PingRequest, + O: PingResponse, + kind: MethodKind.Unary, + }, + }, +} as const diff --git a/packages/bsync/src/gen/bsync_pb.ts b/packages/bsync/src/gen/bsync_pb.ts new file mode 100644 index 00000000000..932bb5337e0 --- /dev/null +++ b/packages/bsync/src/gen/bsync_pb.ts @@ -0,0 +1,453 @@ +// @generated by protoc-gen-es v1.6.0 with parameter "target=ts,import_extension=.ts" +// @generated from file bsync.proto (package bsync, syntax proto3) +/* eslint-disable */ +// @ts-nocheck + +import type { + BinaryReadOptions, + FieldList, + JsonReadOptions, + JsonValue, + PartialMessage, + PlainMessage, +} from '@bufbuild/protobuf' +import { Message, proto3 } from '@bufbuild/protobuf' + +/** + * @generated from message bsync.MuteOperation + */ +export class MuteOperation extends Message { + /** + * @generated from field: string id = 1; + */ + id = '' + + /** + * @generated from field: bsync.MuteOperation.Type type = 2; + */ + type = MuteOperation_Type.ADD + + /** + * @generated from field: string actor_did = 3; + */ + actorDid = '' + + /** + * @generated from field: string subject = 4; + */ + subject = '' + + constructor(data?: PartialMessage) { + super() + proto3.util.initPartial(data, this) + } + + static readonly runtime: typeof proto3 = proto3 + static readonly typeName = 'bsync.MuteOperation' + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: 'id', kind: 'scalar', T: 9 /* ScalarType.STRING */ }, + { + no: 2, + name: 'type', + kind: 'enum', + T: proto3.getEnumType(MuteOperation_Type), + }, + { no: 3, name: 'actor_did', kind: 'scalar', T: 9 /* ScalarType.STRING */ }, + { no: 4, name: 'subject', kind: 'scalar', T: 9 /* ScalarType.STRING */ }, + ]) + + static fromBinary( + bytes: Uint8Array, + options?: Partial, + ): MuteOperation { + return new MuteOperation().fromBinary(bytes, options) + } + + static fromJson( + jsonValue: JsonValue, + options?: Partial, + ): MuteOperation { + return new MuteOperation().fromJson(jsonValue, options) + } + + static fromJsonString( + jsonString: string, + options?: Partial, + ): MuteOperation { + return new MuteOperation().fromJsonString(jsonString, options) + } + + static equals( + a: MuteOperation | PlainMessage | undefined, + b: MuteOperation | PlainMessage | undefined, + ): boolean { + return proto3.util.equals(MuteOperation, a, b) + } +} + +/** + * @generated from enum bsync.MuteOperation.Type + */ +export enum MuteOperation_Type { + /** + * @generated from enum value: ADD = 0; + */ + ADD = 0, + + /** + * @generated from enum value: REMOVE = 1; + */ + REMOVE = 1, + + /** + * @generated from enum value: CLEAR = 2; + */ + CLEAR = 2, +} +// Retrieve enum metadata with: proto3.getEnumType(MuteOperation_Type) +proto3.util.setEnumType(MuteOperation_Type, 'bsync.MuteOperation.Type', [ + { no: 0, name: 'ADD' }, + { no: 1, name: 'REMOVE' }, + { no: 2, name: 'CLEAR' }, +]) + +/** + * @generated from message bsync.AddMuteOperationRequest + */ +export class AddMuteOperationRequest extends Message { + /** + * @generated from field: bsync.MuteOperation.Type type = 1; + */ + type = MuteOperation_Type.ADD + + /** + * @generated from field: string actor_did = 2; + */ + actorDid = '' + + /** + * @generated from field: string subject = 3; + */ + subject = '' + + constructor(data?: PartialMessage) { + super() + proto3.util.initPartial(data, this) + } + + static readonly runtime: typeof proto3 = proto3 + static readonly typeName = 'bsync.AddMuteOperationRequest' + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { + no: 1, + name: 'type', + kind: 'enum', + T: proto3.getEnumType(MuteOperation_Type), + }, + { no: 2, name: 'actor_did', kind: 'scalar', T: 9 /* ScalarType.STRING */ }, + { no: 3, name: 'subject', kind: 'scalar', T: 9 /* ScalarType.STRING */ }, + ]) + + static fromBinary( + bytes: Uint8Array, + options?: Partial, + ): AddMuteOperationRequest { + return new AddMuteOperationRequest().fromBinary(bytes, options) + } + + static fromJson( + jsonValue: JsonValue, + options?: Partial, + ): AddMuteOperationRequest { + return new AddMuteOperationRequest().fromJson(jsonValue, options) + } + + static fromJsonString( + jsonString: string, + options?: Partial, + ): AddMuteOperationRequest { + return new AddMuteOperationRequest().fromJsonString(jsonString, options) + } + + static equals( + a: + | AddMuteOperationRequest + | PlainMessage + | undefined, + b: + | AddMuteOperationRequest + | PlainMessage + | undefined, + ): boolean { + return proto3.util.equals(AddMuteOperationRequest, a, b) + } +} + +/** + * @generated from message bsync.AddMuteOperationResponse + */ +export class AddMuteOperationResponse extends Message { + /** + * @generated from field: bsync.MuteOperation operation = 1; + */ + operation?: MuteOperation + + constructor(data?: PartialMessage) { + super() + proto3.util.initPartial(data, this) + } + + static readonly runtime: typeof proto3 = proto3 + static readonly typeName = 'bsync.AddMuteOperationResponse' + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: 'operation', kind: 'message', T: MuteOperation }, + ]) + + static fromBinary( + bytes: Uint8Array, + options?: Partial, + ): AddMuteOperationResponse { + return new AddMuteOperationResponse().fromBinary(bytes, options) + } + + static fromJson( + jsonValue: JsonValue, + options?: Partial, + ): AddMuteOperationResponse { + return new AddMuteOperationResponse().fromJson(jsonValue, options) + } + + static fromJsonString( + jsonString: string, + options?: Partial, + ): AddMuteOperationResponse { + return new AddMuteOperationResponse().fromJsonString(jsonString, options) + } + + static equals( + a: + | AddMuteOperationResponse + | PlainMessage + | undefined, + b: + | AddMuteOperationResponse + | PlainMessage + | undefined, + ): boolean { + return proto3.util.equals(AddMuteOperationResponse, a, b) + } +} + +/** + * @generated from message bsync.ScanMuteOperationsRequest + */ +export class ScanMuteOperationsRequest extends Message { + /** + * @generated from field: string cursor = 1; + */ + cursor = '' + + /** + * @generated from field: int32 limit = 2; + */ + limit = 0 + + constructor(data?: PartialMessage) { + super() + proto3.util.initPartial(data, this) + } + + static readonly runtime: typeof proto3 = proto3 + static readonly typeName = 'bsync.ScanMuteOperationsRequest' + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: 'cursor', kind: 'scalar', T: 9 /* ScalarType.STRING */ }, + { no: 2, name: 'limit', kind: 'scalar', T: 5 /* ScalarType.INT32 */ }, + ]) + + static fromBinary( + bytes: Uint8Array, + options?: Partial, + ): ScanMuteOperationsRequest { + return new ScanMuteOperationsRequest().fromBinary(bytes, options) + } + + static fromJson( + jsonValue: JsonValue, + options?: Partial, + ): ScanMuteOperationsRequest { + return new ScanMuteOperationsRequest().fromJson(jsonValue, options) + } + + static fromJsonString( + jsonString: string, + options?: Partial, + ): ScanMuteOperationsRequest { + return new ScanMuteOperationsRequest().fromJsonString(jsonString, options) + } + + static equals( + a: + | ScanMuteOperationsRequest + | PlainMessage + | undefined, + b: + | ScanMuteOperationsRequest + | PlainMessage + | undefined, + ): boolean { + return proto3.util.equals(ScanMuteOperationsRequest, a, b) + } +} + +/** + * @generated from message bsync.ScanMuteOperationsResponse + */ +export class ScanMuteOperationsResponse extends Message { + /** + * @generated from field: repeated bsync.MuteOperation operations = 1; + */ + operations: MuteOperation[] = [] + + /** + * @generated from field: string cursor = 2; + */ + cursor = '' + + constructor(data?: PartialMessage) { + super() + proto3.util.initPartial(data, this) + } + + static readonly runtime: typeof proto3 = proto3 + static readonly typeName = 'bsync.ScanMuteOperationsResponse' + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { + no: 1, + name: 'operations', + kind: 'message', + T: MuteOperation, + repeated: true, + }, + { no: 2, name: 'cursor', kind: 'scalar', T: 9 /* ScalarType.STRING */ }, + ]) + + static fromBinary( + bytes: Uint8Array, + options?: Partial, + ): ScanMuteOperationsResponse { + return new ScanMuteOperationsResponse().fromBinary(bytes, options) + } + + static fromJson( + jsonValue: JsonValue, + options?: Partial, + ): ScanMuteOperationsResponse { + return new ScanMuteOperationsResponse().fromJson(jsonValue, options) + } + + static fromJsonString( + jsonString: string, + options?: Partial, + ): ScanMuteOperationsResponse { + return new ScanMuteOperationsResponse().fromJsonString(jsonString, options) + } + + static equals( + a: + | ScanMuteOperationsResponse + | PlainMessage + | undefined, + b: + | ScanMuteOperationsResponse + | PlainMessage + | undefined, + ): boolean { + return proto3.util.equals(ScanMuteOperationsResponse, a, b) + } +} + +/** + * Ping + * + * @generated from message bsync.PingRequest + */ +export class PingRequest extends Message { + constructor(data?: PartialMessage) { + super() + proto3.util.initPartial(data, this) + } + + static readonly runtime: typeof proto3 = proto3 + static readonly typeName = 'bsync.PingRequest' + static readonly fields: FieldList = proto3.util.newFieldList(() => []) + + static fromBinary( + bytes: Uint8Array, + options?: Partial, + ): PingRequest { + return new PingRequest().fromBinary(bytes, options) + } + + static fromJson( + jsonValue: JsonValue, + options?: Partial, + ): PingRequest { + return new PingRequest().fromJson(jsonValue, options) + } + + static fromJsonString( + jsonString: string, + options?: Partial, + ): PingRequest { + return new PingRequest().fromJsonString(jsonString, options) + } + + static equals( + a: PingRequest | PlainMessage | undefined, + b: PingRequest | PlainMessage | undefined, + ): boolean { + return proto3.util.equals(PingRequest, a, b) + } +} + +/** + * @generated from message bsync.PingResponse + */ +export class PingResponse extends Message { + constructor(data?: PartialMessage) { + super() + proto3.util.initPartial(data, this) + } + + static readonly runtime: typeof proto3 = proto3 + static readonly typeName = 'bsync.PingResponse' + static readonly fields: FieldList = proto3.util.newFieldList(() => []) + + static fromBinary( + bytes: Uint8Array, + options?: Partial, + ): PingResponse { + return new PingResponse().fromBinary(bytes, options) + } + + static fromJson( + jsonValue: JsonValue, + options?: Partial, + ): PingResponse { + return new PingResponse().fromJson(jsonValue, options) + } + + static fromJsonString( + jsonString: string, + options?: Partial, + ): PingResponse { + return new PingResponse().fromJsonString(jsonString, options) + } + + static equals( + a: PingResponse | PlainMessage | undefined, + b: PingResponse | PlainMessage | undefined, + ): boolean { + return proto3.util.equals(PingResponse, a, b) + } +} diff --git a/packages/bsync/src/index.ts b/packages/bsync/src/index.ts new file mode 100644 index 00000000000..29583f8afcd --- /dev/null +++ b/packages/bsync/src/index.ts @@ -0,0 +1,91 @@ +import http from 'node:http' +import events from 'node:events' +import { createHttpTerminator, HttpTerminator } from 'http-terminator' +import { connectNodeAdapter } from '@connectrpc/connect-node' +import { dbLogger, loggerMiddleware } from './logger' +import AppContext, { AppContextOptions } from './context' +import { ServerConfig } from './config' +import routes from './routes' +import { createMuteOpChannel } from './db/schema/mute_op' + +export * from './config' +export * from './client' +export { Database } from './db' +export { AppContext } from './context' +export { httpLogger } from './logger' + +export class BsyncService { + public ctx: AppContext + public server: http.Server + private ac: AbortController + private terminator: HttpTerminator + private dbStatsInterval: NodeJS.Timer + + constructor(opts: { + ctx: AppContext + server: http.Server + ac: AbortController + }) { + this.ctx = opts.ctx + this.server = opts.server + this.ac = opts.ac + this.terminator = createHttpTerminator({ server: this.server }) + } + + static async create( + cfg: ServerConfig, + overrides?: Partial, + ): Promise { + const ctx = await AppContext.fromConfig(cfg, overrides) + const ac = new AbortController() + const handler = connectNodeAdapter({ + routes: routes(ctx), + shutdownSignal: ac.signal, + }) + const server = http.createServer((req, res) => { + loggerMiddleware(req, res) + handler(req, res) + }) + return new BsyncService({ ctx, server, ac }) + } + + async start(): Promise { + this.dbStatsInterval = setInterval(() => { + dbLogger.info( + { + idleCount: this.ctx.db.pool.idleCount, + totalCount: this.ctx.db.pool.totalCount, + waitingCount: this.ctx.db.pool.waitingCount, + }, + 'db pool stats', + ) + }, 10000) + await this.setupAppEvents() + this.server.listen(this.ctx.cfg.service.port) + this.server.keepAliveTimeout = 90000 + await events.once(this.server, 'listening') + return this.server + } + + async destroy(): Promise { + this.ac.abort() + await this.terminator.terminate() + await this.ctx.db.close() + clearInterval(this.dbStatsInterval) + } + + async setupAppEvents() { + const conn = await this.ctx.db.pool.connect() + this.ac.signal.addEventListener('abort', () => conn.release(), { + once: true, + }) + conn.query(`listen ${createMuteOpChannel}`) // if this errors, unhandled rejection should cause process to exit + conn.on('notification', (notif) => { + if (notif.channel === createMuteOpChannel) { + this.ctx.events.emit('mute_op_create') + } + }) + } +} + +export default BsyncService diff --git a/packages/bsync/src/logger.ts b/packages/bsync/src/logger.ts new file mode 100644 index 00000000000..740e8c64ace --- /dev/null +++ b/packages/bsync/src/logger.ts @@ -0,0 +1,22 @@ +import pinoHttp from 'pino-http' +import { subsystemLogger } from '@atproto/common' + +export const dbLogger: ReturnType = + subsystemLogger('bsync:db') +export const httpLogger: ReturnType = + subsystemLogger('bsync') + +export const loggerMiddleware = pinoHttp({ + logger: httpLogger, + redact: { + paths: ['req.headers.authorization'], + }, + serializers: { + err: (err) => { + return { + code: err?.code, + message: err?.message, + } + }, + }, +}) diff --git a/packages/bsync/src/routes/add-mute-operation.ts b/packages/bsync/src/routes/add-mute-operation.ts new file mode 100644 index 00000000000..6e433c0aa4e --- /dev/null +++ b/packages/bsync/src/routes/add-mute-operation.ts @@ -0,0 +1,158 @@ +import { sql } from 'kysely' +import { + AtUri, + InvalidDidError, + ensureValidAtUri, + ensureValidDid, +} from '@atproto/syntax' +import { Code, ConnectError, ServiceImpl } from '@connectrpc/connect' +import { Service } from '../gen/bsync_connect' +import { AddMuteOperationResponse, MuteOperation_Type } from '../gen/bsync_pb' +import AppContext from '../context' +import { createMuteOpChannel } from '../db/schema/mute_op' +import { authWithApiKey } from './auth' +import Database from '../db' + +export default (ctx: AppContext): Partial> => ({ + async addMuteOperation(req, handlerCtx) { + authWithApiKey(ctx, handlerCtx) + const { db } = ctx + const op = validMuteOp(req) + const id = await db.transaction(async (txn) => { + // create mute op + const id = await createMuteOp(txn, op) + // update mute state + if (op.type === MuteOperation_Type.ADD) { + await addMuteItem(txn, id, op) + } else if (op.type === MuteOperation_Type.REMOVE) { + await removeMuteItem(txn, op) + } else if (op.type === MuteOperation_Type.CLEAR) { + await clearMuteItems(txn, op) + } else { + const exhaustiveCheck: never = op.type + throw new Error(`unreachable: ${exhaustiveCheck}`) + } + return id + }) + return new AddMuteOperationResponse({ + operation: { + id: String(id), + type: op.type, + actorDid: op.actorDid, + subject: op.subject, + }, + }) + }, +}) + +const createMuteOp = async (db: Database, op: MuteOpInfo) => { + const { ref } = db.db.dynamic + const { id } = await db.db + .insertInto('mute_op') + .values({ + type: op.type, + actorDid: op.actorDid, + subject: op.subject, + }) + .returning('id') + .executeTakeFirstOrThrow() + await sql`notify ${ref(createMuteOpChannel)}`.execute(db.db) // emitted transactionally + return id +} + +const addMuteItem = async (db: Database, fromId: number, op: MuteOpInfo) => { + const { ref } = db.db.dynamic + await db.db + .insertInto('mute_item') + .values({ + actorDid: op.actorDid, + subject: op.subject, + fromId, + }) + .onConflict((oc) => + oc + .constraint('mute_item_pkey') + .doUpdateSet({ fromId: sql`${ref('excluded.fromId')}` }), + ) + .execute() +} + +const removeMuteItem = async (db: Database, op: MuteOpInfo) => { + await db.db + .deleteFrom('mute_item') + .where('actorDid', '=', op.actorDid) + .where('subject', '=', op.subject) + .execute() +} + +const clearMuteItems = async (db: Database, op: MuteOpInfo) => { + await db.db + .deleteFrom('mute_item') + .where('actorDid', '=', op.actorDid) + .execute() +} + +const validMuteOp = (op: MuteOpInfo): MuteOpInfo => { + if (!Object.values(MuteOperation_Type).includes(op.type)) { + throw new ConnectError('bad mute operation type', Code.InvalidArgument) + } + if (!isValidDid(op.actorDid)) { + throw new ConnectError( + 'actor_did must be a valid did', + Code.InvalidArgument, + ) + } + if (op.type === MuteOperation_Type.CLEAR) { + if (op.subject !== '') { + throw new ConnectError( + 'subject must not be set on a clear op', + Code.InvalidArgument, + ) + } + } else { + if (isValidDid(op.subject)) { + // all good + } else if (isValidAtUri(op.subject)) { + const uri = new AtUri(op.subject) + if (uri.collection !== 'app.bsky.graph.list') { + throw new ConnectError( + 'subject aturis must reference a list record', + Code.InvalidArgument, + ) + } + } else { + throw new ConnectError( + 'subject must be a did or aturi on add or remove op', + Code.InvalidArgument, + ) + } + } + return op +} + +const isValidDid = (did: string) => { + try { + ensureValidDid(did) + return true + } catch (err) { + if (err instanceof InvalidDidError) { + return false + } + throw err + } +} + +const isValidAtUri = (uri: string) => { + try { + ensureValidAtUri(uri) + return true + } catch { + return false + } +} + +type MuteOpInfo = { + type: MuteOperation_Type + actorDid: string + subject: string +} diff --git a/packages/bsync/src/routes/auth.ts b/packages/bsync/src/routes/auth.ts new file mode 100644 index 00000000000..003bb03ad9b --- /dev/null +++ b/packages/bsync/src/routes/auth.ts @@ -0,0 +1,15 @@ +import { Code, ConnectError, HandlerContext } from '@connectrpc/connect' +import AppContext from '../context' + +const BEARER = 'Bearer ' + +export const authWithApiKey = (ctx: AppContext, handlerCtx: HandlerContext) => { + const authorization = handlerCtx.requestHeader.get('authorization') + if (!authorization?.startsWith(BEARER)) { + throw new ConnectError('missing auth', Code.Unauthenticated) + } + const key = authorization.slice(BEARER.length) + if (!ctx.cfg.auth.apiKeys.has(key)) { + throw new ConnectError('invalid api key', Code.Unauthenticated) + } +} diff --git a/packages/bsync/src/routes/index.ts b/packages/bsync/src/routes/index.ts new file mode 100644 index 00000000000..726949f4c64 --- /dev/null +++ b/packages/bsync/src/routes/index.ts @@ -0,0 +1,18 @@ +import { sql } from 'kysely' +import { ConnectRouter } from '@connectrpc/connect' +import { Service } from '../gen/bsync_connect' +import AppContext from '../context' +import addMuteOperation from './add-mute-operation' +import scanMuteOperations from './scan-mute-operations' + +export default (ctx: AppContext) => (router: ConnectRouter) => { + return router.service(Service, { + ...addMuteOperation(ctx), + ...scanMuteOperations(ctx), + async ping() { + const { db } = ctx + await sql`select 1`.execute(db.db) + return {} + }, + }) +} diff --git a/packages/bsync/src/routes/scan-mute-operations.ts b/packages/bsync/src/routes/scan-mute-operations.ts new file mode 100644 index 00000000000..c0f4f403ef8 --- /dev/null +++ b/packages/bsync/src/routes/scan-mute-operations.ts @@ -0,0 +1,69 @@ +import { once } from 'node:events' +import { Code, ConnectError, ServiceImpl } from '@connectrpc/connect' +import { Service } from '../gen/bsync_connect' +import { ScanMuteOperationsResponse } from '../gen/bsync_pb' +import AppContext from '../context' +import { createMuteOpChannel } from '../db/schema/mute_op' +import { authWithApiKey } from './auth' + +export default (ctx: AppContext): Partial> => ({ + async scanMuteOperations(req, handlerCtx) { + authWithApiKey(ctx, handlerCtx) + const { db, events } = ctx + const limit = req.limit || 1000 + const cursor = validCursor(req.cursor) + const nextMuteOpPromise = once(events, createMuteOpChannel, { + signal: AbortSignal.timeout(ctx.cfg.service.longPollTimeoutMs), + }) + nextMuteOpPromise.catch(() => null) // ensure timeout is always handled + + const nextMuteOpPageQb = db.db + .selectFrom('mute_op') + .selectAll() + .where('id', '>', cursor ?? -1) + .orderBy('id', 'asc') + .limit(limit) + + let ops = await nextMuteOpPageQb.execute() + + if (!ops.length) { + // if there were no ops on the page, wait for an event then try again. + try { + await nextMuteOpPromise + } catch (err) { + return new ScanMuteOperationsResponse({ + operations: [], + cursor: req.cursor, + }) + } + ops = await nextMuteOpPageQb.execute() + if (!ops.length) { + return new ScanMuteOperationsResponse({ + operations: [], + cursor: req.cursor, + }) + } + } + + const lastOp = ops[ops.length - 1] + + return new ScanMuteOperationsResponse({ + operations: ops.map((op) => ({ + id: op.id.toString(), + type: op.type, + actorDid: op.actorDid, + subject: op.subject, + })), + cursor: lastOp.id.toString(), + }) + }, +}) + +const validCursor = (cursor: string): number | null => { + if (cursor === '') return null + const int = parseInt(cursor, 10) + if (isNaN(int) || int < 0) { + throw new ConnectError('invalid cursor', Code.InvalidArgument) + } + return int +} diff --git a/packages/bsync/tests/mutes.test.ts b/packages/bsync/tests/mutes.test.ts new file mode 100644 index 00000000000..03dee110f85 --- /dev/null +++ b/packages/bsync/tests/mutes.test.ts @@ -0,0 +1,350 @@ +import { wait } from '@atproto/common' +import { Code, ConnectError } from '@connectrpc/connect' +import { + BsyncClient, + BsyncService, + Database, + authWithApiKey, + createClient, + envToCfg, +} from '../src' +import { MuteOperation, MuteOperation_Type } from '../src/gen/bsync_pb' + +describe('mutes', () => { + let bsync: BsyncService + let client: BsyncClient + + beforeAll(async () => { + bsync = await BsyncService.create( + envToCfg({ + dbUrl: process.env.DB_POSTGRES_URL, + dbSchema: 'bsync_mutes', + apiKeys: ['key-1'], + longPollTimeoutMs: 500, + }), + ) + await bsync.ctx.db.migrateToLatestOrThrow() + await bsync.start() + client = createClient({ + httpVersion: '1.1', + baseUrl: `http://localhost:${bsync.ctx.cfg.service.port}`, + interceptors: [authWithApiKey('key-1')], + }) + }) + + afterAll(async () => { + await bsync.destroy() + }) + + beforeEach(async () => { + await clearMutes(bsync.ctx.db) + }) + + describe('addMuteOperation', () => { + it('adds mute operations to add mutes.', async () => { + await client.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: 'did:example:a', + subject: 'did:example:b', + }) + await client.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: 'did:example:a', + subject: 'did:example:c', + }) + // dupe has no effect + await client.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: 'did:example:a', + subject: 'did:example:c', + }) + await client.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: 'did:example:b', + subject: 'did:example:c', + }) + await client.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: 'did:example:c', + subject: 'at://did:example:d/app.bsky.graph.list/rkey1', + }) + expect(await dumpMuteState(bsync.ctx.db)).toEqual({ + 'did:example:a': ['did:example:b', 'did:example:c'], + 'did:example:b': ['did:example:c'], + 'did:example:c': ['at://did:example:d/app.bsky.graph.list/rkey1'], + }) + }) + + it('adds mute operations to remove mutes.', async () => { + await client.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: 'did:example:a', + subject: 'did:example:b', + }) + await client.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: 'did:example:a', + subject: 'did:example:c', + }) + await client.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: 'did:example:b', + subject: 'did:example:c', + }) + await client.addMuteOperation({ + type: MuteOperation_Type.REMOVE, + actorDid: 'did:example:a', + subject: 'did:example:c', + }) + // removes nothing + await client.addMuteOperation({ + type: MuteOperation_Type.REMOVE, + actorDid: 'did:example:b', + subject: 'did:example:d', + }) + expect(await dumpMuteState(bsync.ctx.db)).toEqual({ + 'did:example:a': ['did:example:b'], + 'did:example:b': ['did:example:c'], + }) + }) + + it('adds mute operations to clear mutes.', async () => { + await client.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: 'did:example:a', + subject: 'did:example:b', + }) + await client.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: 'did:example:a', + subject: 'did:example:c', + }) + await client.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: 'did:example:b', + subject: 'did:example:c', + }) + await client.addMuteOperation({ + type: MuteOperation_Type.CLEAR, + actorDid: 'did:example:a', + }) + expect(await dumpMuteState(bsync.ctx.db)).toEqual({ + 'did:example:b': ['did:example:c'], + }) + }) + + it('fails on bad inputs', async () => { + await expect( + client.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: 'did:example:a', + subject: 'invalid', + }), + ).rejects.toEqual( + new ConnectError( + 'subject must be a did or aturi on add or remove op', + Code.InvalidArgument, + ), + ) + await expect( + client.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: 'did:example:a', + }), + ).rejects.toEqual( + new ConnectError( + 'subject must be a did or aturi on add or remove op', + Code.InvalidArgument, + ), + ) + await expect( + client.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: 'did:example:a', + subject: 'at://did:example:b/bad.collection/rkey1', + }), + ).rejects.toEqual( + new ConnectError( + 'subject must be a did or aturi on add or remove op', + Code.InvalidArgument, + ), + ) + await expect( + client.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: 'invalid', + subject: 'did:example:b', + }), + ).rejects.toEqual( + new ConnectError('actor_did must be a valid did', Code.InvalidArgument), + ) + await expect( + client.addMuteOperation({ + type: MuteOperation_Type.REMOVE, + actorDid: 'did:example:a', + subject: 'invalid', + }), + ).rejects.toEqual( + new ConnectError( + 'subject must be a did or aturi on add or remove op', + Code.InvalidArgument, + ), + ) + await expect( + client.addMuteOperation({ + type: MuteOperation_Type.CLEAR, + actorDid: 'did:example:a', + subject: 'did:example:b', + }), + ).rejects.toEqual( + new ConnectError( + 'subject must not be set on a clear op', + Code.InvalidArgument, + ), + ) + await expect( + client.addMuteOperation({ + type: MuteOperation_Type.CLEAR, + actorDid: 'invalid', + }), + ).rejects.toEqual( + new ConnectError('actor_did must be a valid did', Code.InvalidArgument), + ) + await expect( + client.addMuteOperation({ + type: 100 as any, + actorDid: 'did:example:a', + subject: 'did:example:b', + }), + ).rejects.toEqual( + new ConnectError('bad mute operation type', Code.InvalidArgument), + ) + }) + + it('requires auth', async () => { + // unauthed + const unauthedClient = createClient({ + httpVersion: '1.1', + baseUrl: `http://localhost:${bsync.ctx.cfg.service.port}`, + }) + const tryAddMuteOperation1 = unauthedClient.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: 'did:example:a', + subject: 'did:example:b', + }) + await expect(tryAddMuteOperation1).rejects.toEqual( + new ConnectError('missing auth', Code.Unauthenticated), + ) + // bad auth + const badauthedClient = createClient({ + httpVersion: '1.1', + baseUrl: `http://localhost:${bsync.ctx.cfg.service.port}`, + interceptors: [authWithApiKey('key-bad')], + }) + const tryAddMuteOperation2 = badauthedClient.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: 'did:example:a', + subject: 'did:example:b', + }) + await expect(tryAddMuteOperation2).rejects.toEqual( + new ConnectError('invalid api key', Code.Unauthenticated), + ) + }) + }) + + describe('scanMuteOperations', () => { + it('requires auth', async () => { + // unauthed + const unauthedClient = createClient({ + httpVersion: '1.1', + baseUrl: `http://localhost:${bsync.ctx.cfg.service.port}`, + }) + const tryScanMuteOperations1 = unauthedClient.scanMuteOperations({}) + await expect(tryScanMuteOperations1).rejects.toEqual( + new ConnectError('missing auth', Code.Unauthenticated), + ) + // bad auth + const badauthedClient = createClient({ + httpVersion: '1.1', + baseUrl: `http://localhost:${bsync.ctx.cfg.service.port}`, + interceptors: [authWithApiKey('key-bad')], + }) + const tryScanMuteOperations2 = badauthedClient.scanMuteOperations({}) + await expect(tryScanMuteOperations2).rejects.toEqual( + new ConnectError('invalid api key', Code.Unauthenticated), + ) + }) + + it('pages over created mute ops.', async () => { + // add 100 mute ops + for (let i = 0; i < 10; ++i) { + for (let j = 0; j < 8; ++j) { + await client.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: `did:example:${i}`, + subject: `did:example:${j}`, + }) + } + for (let j = 0; j < 2; ++j) { + await client.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: `did:example:${i}`, + subject: `at://did:example:0/app.bsky.graph.list/rkey${j}`, + }) + } + } + + let cursor: string | undefined + const operations: MuteOperation[] = [] + do { + const res = await client.scanMuteOperations({ + cursor, + limit: 30, + }) + operations.push(...res.operations) + cursor = res.operations.length ? res.cursor : undefined + } while (cursor) + + expect(operations.length).toEqual(100) + const operationIds = operations.map((op) => parseInt(op.id, 10)) + const ascending = (a: number, b: number) => a - b + expect(operationIds).toEqual([...operationIds].sort(ascending)) + }) + + it('supports long-poll, finding an operation.', async () => { + const scanPromise = client.scanMuteOperations({}) + await wait(100) // would be complete by now if it wasn't long-polling for an item + const { operation } = await client.addMuteOperation({ + type: MuteOperation_Type.ADD, + actorDid: 'did:example:a', + subject: 'did:example:b', + }) + const res = await scanPromise + expect(res.operations.length).toEqual(1) + expect(res.operations[0]).toEqual(operation) + expect(res.cursor).toEqual(operation?.id) + }) + + it('supports long-poll, not finding an operation.', async () => { + const res = await client.scanMuteOperations({}) + expect(res.cursor).toEqual('') + expect(res.operations).toEqual([]) + }) + }) +}) + +const dumpMuteState = async (db: Database) => { + const items = await db.db.selectFrom('mute_item').selectAll().execute() + const result: Record = {} + items.forEach((item) => { + result[item.actorDid] ??= [] + result[item.actorDid].push(item.subject) + }) + Object.values(result).forEach((subjects) => subjects.sort()) + return result +} + +const clearMutes = async (db: Database) => { + await db.db.deleteFrom('mute_item').execute() + await db.db.deleteFrom('mute_op').execute() +} diff --git a/packages/bsync/tsconfig.build.json b/packages/bsync/tsconfig.build.json new file mode 100644 index 00000000000..02a84823b65 --- /dev/null +++ b/packages/bsync/tsconfig.build.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["**/*.spec.ts", "**/*.test.ts"] +} diff --git a/packages/bsync/tsconfig.json b/packages/bsync/tsconfig.json new file mode 100644 index 00000000000..7e6792f9590 --- /dev/null +++ b/packages/bsync/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "./src", + "outDir": "./dist", + "emitDeclarationOnly": true + }, + "module": "nodenext", + "include": ["./src", "__tests__/**/**.ts"], + "references": [ + { "path": "../common/tsconfig.build.json" }, + { "path": "../common-web/tsconfig.build.json" } + ] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dc2fd699490..09aca984856 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -36,11 +36,11 @@ importers: specifier: ^18.0.0 version: 18.0.0 '@typescript-eslint/eslint-plugin': - specifier: ^5.38.1 - version: 5.38.1(@typescript-eslint/parser@5.38.1)(eslint@8.24.0)(typescript@4.8.4) + specifier: ^6.14.0 + version: 6.18.1(@typescript-eslint/parser@6.18.1)(eslint@8.24.0)(typescript@5.3.3) '@typescript-eslint/parser': - specifier: ^5.38.1 - version: 5.38.1(eslint@8.24.0)(typescript@4.8.4) + specifier: ^6.14.0 + version: 6.18.1(eslint@8.24.0)(typescript@5.3.3) babel-eslint: specifier: ^10.1.0 version: 10.1.0(eslint@8.24.0) @@ -85,10 +85,10 @@ importers: version: 5.0.0(prettier@2.7.1) ts-node: specifier: ^10.8.2 - version: 10.8.2(@swc/core@1.3.42)(@types/node@18.0.0)(typescript@4.8.4) + version: 10.8.2(@swc/core@1.3.42)(@types/node@18.0.0)(typescript@5.3.3) typescript: - specifier: ^4.8.4 - version: 4.8.4 + specifier: ^5.3.3 + version: 5.3.3 packages/api: dependencies: @@ -281,6 +281,58 @@ importers: specifier: ^0.27.2 version: 0.27.2 + packages/bsync: + dependencies: + '@atproto/common': + specifier: workspace:^ + version: link:../common + '@atproto/syntax': + specifier: workspace:^ + version: link:../syntax + '@bufbuild/protobuf': + specifier: ^1.5.0 + version: 1.6.0 + '@connectrpc/connect': + specifier: ^1.1.4 + version: 1.3.0(@bufbuild/protobuf@1.6.0) + '@connectrpc/connect-express': + specifier: ^1.1.4 + version: 1.3.0(@bufbuild/protobuf@1.6.0)(@connectrpc/connect-node@1.3.0)(@connectrpc/connect@1.3.0) + '@connectrpc/connect-node': + specifier: ^1.1.4 + version: 1.3.0(@bufbuild/protobuf@1.6.0)(@connectrpc/connect@1.3.0) + http-terminator: + specifier: ^3.2.0 + version: 3.2.0 + kysely: + specifier: ^0.22.0 + version: 0.22.0 + pg: + specifier: ^8.10.0 + version: 8.10.0 + pino: + specifier: ^8.15.0 + version: 8.15.0 + pino-http: + specifier: ^8.2.1 + version: 8.4.0 + typed-emitter: + specifier: ^2.1.0 + version: 2.1.0 + devDependencies: + '@bufbuild/buf': + specifier: ^1.28.1 + version: 1.28.1 + '@bufbuild/protoc-gen-es': + specifier: ^1.5.0 + version: 1.6.0(@bufbuild/protobuf@1.6.0) + '@connectrpc/protoc-gen-connect-es': + specifier: ^1.1.4 + version: 1.3.0(@bufbuild/protoc-gen-es@1.6.0)(@connectrpc/connect@1.3.0) + '@types/pg': + specifier: ^8.6.6 + version: 8.6.6 + packages/common: dependencies: '@atproto/common-web': @@ -404,7 +456,7 @@ importers: devDependencies: ts-node: specifier: ^10.8.1 - version: 10.8.2(@swc/core@1.3.42)(@types/node@18.17.8)(typescript@4.8.4) + version: 10.8.2(@swc/core@1.3.42)(@types/node@18.17.8)(typescript@5.3.3) packages/identity: dependencies: @@ -866,6 +918,15 @@ importers: specifier: 3.13.2 version: 3.13.2 + services/bsync: + dependencies: + '@atproto/bsync': + specifier: workspace:^ + version: link:../../packages/bsync + dd-trace: + specifier: 3.13.2 + version: 3.13.2 + services/ozone: dependencies: '@atproto/ozone': @@ -4450,6 +4511,103 @@ packages: resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==} dev: true + /@bufbuild/buf-darwin-arm64@1.28.1: + resolution: {integrity: sha512-nAyvwKkcd8qQTExCZo5MtSRhXLK7e3vzKFKHjXfkveRakSUST2HFlFZAHfErZimN4wBrPTN0V0hNRU8PPjkMpQ==} + engines: {node: '>=12'} + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@bufbuild/buf-darwin-x64@1.28.1: + resolution: {integrity: sha512-b0eT3xd3vX5a5lWAbo5h7FPuf9MsOJI4I39qs4TZnrlZ8BOuPfqzwzijiFf9UCwaX2vR1NQXexIoQ80Ci+fCHw==} + engines: {node: '>=12'} + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@bufbuild/buf-linux-aarch64@1.28.1: + resolution: {integrity: sha512-p5h9bZCVLMh8No9/7k7ulXzsFx5P7Lu6DiUMjSJ6aBXPMYo6Xl7r/6L2cQkpsZ53HMtIxCgMYS9a7zoS4K8wIw==} + engines: {node: '>=12'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@bufbuild/buf-linux-x64@1.28.1: + resolution: {integrity: sha512-fVJ3DiRigIso06jgEl+JNp59Y5t2pxDHd10d3SA4r+14sXbZ2J7Gy/wBqVXPry4x/jW567KKlvmhg7M5ZBgCQQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@bufbuild/buf-win32-arm64@1.28.1: + resolution: {integrity: sha512-KJiRJpugQRK/jXC46Xjlb68UydWhCZj2jHdWLIwNtgXd1WTJ3LngChZV7Y6pPK08pwBAVz0JYeVbD5IlTCD4TQ==} + engines: {node: '>=12'} + cpu: [arm64] + os: [win32] + requiresBuild: true + dev: true + optional: true + + /@bufbuild/buf-win32-x64@1.28.1: + resolution: {integrity: sha512-vMnc+7OVCkmlRWQsgYHgUqiBPRIjD8XeoRyApJ07YZzGs7DkRH4LhvmacJbLd3wORylbn6gLz3pQa8J/M61mzg==} + engines: {node: '>=12'} + cpu: [x64] + os: [win32] + requiresBuild: true + dev: true + optional: true + + /@bufbuild/buf@1.28.1: + resolution: {integrity: sha512-WRDagrf0uBjfV9s5eyrSPJDcdI4A5Q7JMCA4aMrHRR8fo/TTjniDBjJprszhaguqsDkn/LS4QIu92HVFZCrl9A==} + engines: {node: '>=12'} + hasBin: true + requiresBuild: true + optionalDependencies: + '@bufbuild/buf-darwin-arm64': 1.28.1 + '@bufbuild/buf-darwin-x64': 1.28.1 + '@bufbuild/buf-linux-aarch64': 1.28.1 + '@bufbuild/buf-linux-x64': 1.28.1 + '@bufbuild/buf-win32-arm64': 1.28.1 + '@bufbuild/buf-win32-x64': 1.28.1 + dev: true + + /@bufbuild/protobuf@1.6.0: + resolution: {integrity: sha512-hp19vSFgNw3wBBcVBx5qo5pufCqjaJ0Cfk5H/pfjNOfNWU+4/w0QVOmfAOZNRrNWRrVuaJWxcN8P2vhOkkzbBQ==} + + /@bufbuild/protoc-gen-es@1.6.0(@bufbuild/protobuf@1.6.0): + resolution: {integrity: sha512-m0akOPWeD5UBfGdZyudrbnmdjI8l/ZHlP8TyEIcj7qMCR4kh68tMtGvrjRzj5ynIpavrr6G7P06XP9F9f2MDRw==} + engines: {node: '>=14'} + hasBin: true + peerDependencies: + '@bufbuild/protobuf': 1.6.0 + peerDependenciesMeta: + '@bufbuild/protobuf': + optional: true + dependencies: + '@bufbuild/protobuf': 1.6.0 + '@bufbuild/protoplugin': 1.6.0 + transitivePeerDependencies: + - supports-color + dev: true + + /@bufbuild/protoplugin@1.6.0: + resolution: {integrity: sha512-o53ZsvojHQkAPoC9v5sJifY2OfXdRU8DO3QpPoJ+QuvYcfB9Zb3DZkNMQRyfEbF4TVYiaQ0mZzZl1mESDdyCxA==} + dependencies: + '@bufbuild/protobuf': 1.6.0 + '@typescript/vfs': 1.5.0 + typescript: 4.5.2 + transitivePeerDependencies: + - supports-color + dev: true + /@cbor-extract/cbor-extract-darwin-arm64@2.1.1: resolution: {integrity: sha512-blVBy5MXz6m36Vx0DfLd7PChOQKEs8lK2bD1WJn/vVgG4FXZiZmZb2GECHFvVPA5T7OnODd9xZiL3nMCv6QUhA==} cpu: [arm64] @@ -4701,6 +4859,60 @@ packages: prettier: 2.7.1 dev: true + /@connectrpc/connect-express@1.3.0(@bufbuild/protobuf@1.6.0)(@connectrpc/connect-node@1.3.0)(@connectrpc/connect@1.3.0): + resolution: {integrity: sha512-6wbaQheD9cb4DnU1PvgVQdB1XPfA0bhlA0V0ZKx6oJJnTgGEYBzPrQztmqs5XW36/r+qJRfMgaVzZfX8MLafgA==} + engines: {node: '>=16.0.0'} + peerDependencies: + '@bufbuild/protobuf': ^1.4.2 + '@connectrpc/connect': 1.3.0 + '@connectrpc/connect-node': 1.3.0 + dependencies: + '@bufbuild/protobuf': 1.6.0 + '@connectrpc/connect': 1.3.0(@bufbuild/protobuf@1.6.0) + '@connectrpc/connect-node': 1.3.0(@bufbuild/protobuf@1.6.0)(@connectrpc/connect@1.3.0) + '@types/express': 4.17.21 + dev: false + + /@connectrpc/connect-node@1.3.0(@bufbuild/protobuf@1.6.0)(@connectrpc/connect@1.3.0): + resolution: {integrity: sha512-2fV/z/8MUFOkTn2Gbm7T/qvRfkpt/D/w7ykYqACZRH6VNG/faY4lH2wUZiNkwv9tzTrECKOJFyPsaGs3nRYK3w==} + engines: {node: '>=16.0.0'} + peerDependencies: + '@bufbuild/protobuf': ^1.4.2 + '@connectrpc/connect': 1.3.0 + dependencies: + '@bufbuild/protobuf': 1.6.0 + '@connectrpc/connect': 1.3.0(@bufbuild/protobuf@1.6.0) + undici: 5.28.2 + dev: false + + /@connectrpc/connect@1.3.0(@bufbuild/protobuf@1.6.0): + resolution: {integrity: sha512-kTeWxJnLLtxKc2ZSDN0rIBgwfP8RwcLknthX4AKlIAmN9ZC4gGnCbwp+3BKcP/WH5c8zGBAWqSY3zeqCM+ah7w==} + peerDependencies: + '@bufbuild/protobuf': ^1.4.2 + dependencies: + '@bufbuild/protobuf': 1.6.0 + + /@connectrpc/protoc-gen-connect-es@1.3.0(@bufbuild/protoc-gen-es@1.6.0)(@connectrpc/connect@1.3.0): + resolution: {integrity: sha512-UbQN48c0zafo5EFSsh3POIJP6ofYiAgKE1aFOZ2Er4W3flUYihydZdM6TQauPkn7jDj4w9jjLSTTZ9//ecUbPA==} + engines: {node: '>=16.0.0'} + hasBin: true + peerDependencies: + '@bufbuild/protoc-gen-es': ^1.6.0 + '@connectrpc/connect': 1.3.0 + peerDependenciesMeta: + '@bufbuild/protoc-gen-es': + optional: true + '@connectrpc/connect': + optional: true + dependencies: + '@bufbuild/protobuf': 1.6.0 + '@bufbuild/protoc-gen-es': 1.6.0(@bufbuild/protobuf@1.6.0) + '@bufbuild/protoplugin': 1.6.0 + '@connectrpc/connect': 1.3.0(@bufbuild/protobuf@1.6.0) + transitivePeerDependencies: + - supports-color + dev: true + /@cspotcode/source-map-support@0.8.1: resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} @@ -4847,6 +5059,21 @@ packages: - pg-native - supports-color + /@eslint-community/eslint-utils@4.4.0(eslint@8.24.0): + resolution: {integrity: sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + peerDependencies: + eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 + dependencies: + eslint: 8.24.0 + eslint-visitor-keys: 3.4.3 + dev: true + + /@eslint-community/regexpp@4.10.0: + resolution: {integrity: sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA==} + engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} + dev: true + /@eslint/eslintrc@1.4.1: resolution: {integrity: sha512-XXrH9Uarn0stsyldqDYq8r++mROmWRI1xKMXa640Bb//SY1+ECYX6VzT6Lcx5frD0V30XieqJ0oX9I2Xj5aoMA==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -4864,6 +5091,11 @@ packages: - supports-color dev: true + /@fastify/busboy@2.1.0: + resolution: {integrity: sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==} + engines: {node: '>=14'} + dev: false + /@fastify/deepmerge@1.3.0: resolution: {integrity: sha512-J8TOSBq3SoZbDhM9+R/u77hP93gz/rajSA+K2kGyijPpORPWUXHUpTaleoj+92As0S9uPRP7Oi8IqMf0u+ro6A==} @@ -5594,13 +5826,11 @@ packages: dependencies: '@types/connect': 3.4.35 '@types/node': 18.17.8 - dev: true /@types/connect@3.4.35: resolution: {integrity: sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==} dependencies: '@types/node': 18.17.8 - dev: true /@types/cors@2.8.12: resolution: {integrity: sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==} @@ -5622,7 +5852,6 @@ packages: '@types/qs': 6.9.7 '@types/range-parser': 1.2.4 '@types/send': 0.17.1 - dev: true /@types/express@4.17.13: resolution: {integrity: sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==} @@ -5633,6 +5862,15 @@ packages: '@types/serve-static': 1.15.2 dev: true + /@types/express@4.17.21: + resolution: {integrity: sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==} + dependencies: + '@types/body-parser': 1.19.2 + '@types/express-serve-static-core': 4.17.36 + '@types/qs': 6.9.7 + '@types/serve-static': 1.15.2 + dev: false + /@types/graceful-fs@4.1.6: resolution: {integrity: sha512-Sig0SNORX9fdW+bQuTEovKj3uHcUL6LQKbCrrqb1X7J6/ReAbhCXRAhc+SMejhLELFj2QcyuxmUooZ4bt5ReSw==} dependencies: @@ -5641,7 +5879,6 @@ packages: /@types/http-errors@2.0.1: resolution: {integrity: sha512-/K3ds8TRAfBvi5vfjuz8y6+GiAYBZ0x4tXv1Av6CWBWn0IlADc+ZX9pMq7oU0fNQPnBwIZl3rmeLp6SBApbxSQ==} - dev: true /@types/is-ci@3.0.0: resolution: {integrity: sha512-Q0Op0hdWbYd1iahB+IFNQcWXFq4O0Q5MwQP7uN0souuQ4rPg1vEYcnIOfr1gY+M+6rc8FGoRaBO1mOOvL29sEQ==} @@ -5678,11 +5915,9 @@ packages: /@types/mime@1.3.2: resolution: {integrity: sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==} - dev: true /@types/mime@3.0.1: resolution: {integrity: sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==} - dev: true /@types/minimist@1.2.2: resolution: {integrity: sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==} @@ -5723,11 +5958,9 @@ packages: /@types/qs@6.9.7: resolution: {integrity: sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==} - dev: true /@types/range-parser@1.2.4: resolution: {integrity: sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==} - dev: true /@types/semver@7.5.0: resolution: {integrity: sha512-G8hZ6XJiHnuhQKR7ZmysCeJWE08o8T0AXtk5darsCaTVsYZhhgUrq53jizaR2FvsoeCwJhlmwTjkXBY5Pn/ZHw==} @@ -5738,7 +5971,6 @@ packages: dependencies: '@types/mime': 1.3.2 '@types/node': 18.17.8 - dev: true /@types/serve-static@1.15.2: resolution: {integrity: sha512-J2LqtvFYCzaj8pVYKw8klQXrLLk7TBZmQ4ShlcdkELFKGwGMfevMLneMMRkMgZxotOD9wg497LpC7O8PcvAmfw==} @@ -5746,7 +5978,6 @@ packages: '@types/http-errors': 2.0.1 '@types/mime': 3.0.1 '@types/node': 18.17.8 - dev: true /@types/shimmer@1.0.5: resolution: {integrity: sha512-9Hp0ObzwwO57DpLFF0InUjUm/II8GmKAvzbefxQTihCb7KI6yc9yzf0nLc4mVdby5N4DRCgQM2wCup9KTieeww==} @@ -5778,132 +6009,146 @@ packages: '@types/yargs-parser': 21.0.0 dev: true - /@typescript-eslint/eslint-plugin@5.38.1(@typescript-eslint/parser@5.38.1)(eslint@8.24.0)(typescript@4.8.4): - resolution: {integrity: sha512-ky7EFzPhqz3XlhS7vPOoMDaQnQMn+9o5ICR9CPr/6bw8HrFkzhMSxuA3gRfiJVvs7geYrSeawGJjZoZQKCOglQ==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + /@typescript-eslint/eslint-plugin@6.18.1(@typescript-eslint/parser@6.18.1)(eslint@8.24.0)(typescript@5.3.3): + resolution: {integrity: sha512-nISDRYnnIpk7VCFrGcu1rnZfM1Dh9LRHnfgdkjcbi/l7g16VYRri3TjXi9Ir4lOZSw5N/gnV/3H7jIPQ8Q4daA==} + engines: {node: ^16.0.0 || >=18.0.0} peerDependencies: - '@typescript-eslint/parser': ^5.0.0 - eslint: ^6.0.0 || ^7.0.0 || ^8.0.0 + '@typescript-eslint/parser': ^6.0.0 || ^6.0.0-alpha + eslint: ^7.0.0 || ^8.0.0 typescript: '*' peerDependenciesMeta: typescript: optional: true dependencies: - '@typescript-eslint/parser': 5.38.1(eslint@8.24.0)(typescript@4.8.4) - '@typescript-eslint/scope-manager': 5.38.1 - '@typescript-eslint/type-utils': 5.38.1(eslint@8.24.0)(typescript@4.8.4) - '@typescript-eslint/utils': 5.38.1(eslint@8.24.0)(typescript@4.8.4) + '@eslint-community/regexpp': 4.10.0 + '@typescript-eslint/parser': 6.18.1(eslint@8.24.0)(typescript@5.3.3) + '@typescript-eslint/scope-manager': 6.18.1 + '@typescript-eslint/type-utils': 6.18.1(eslint@8.24.0)(typescript@5.3.3) + '@typescript-eslint/utils': 6.18.1(eslint@8.24.0)(typescript@5.3.3) + '@typescript-eslint/visitor-keys': 6.18.1 debug: 4.3.4 eslint: 8.24.0 + graphemer: 1.4.0 ignore: 5.2.4 - regexpp: 3.2.0 + natural-compare: 1.4.0 semver: 7.5.4 - tsutils: 3.21.0(typescript@4.8.4) - typescript: 4.8.4 + ts-api-utils: 1.0.3(typescript@5.3.3) + typescript: 5.3.3 transitivePeerDependencies: - supports-color dev: true - /@typescript-eslint/parser@5.38.1(eslint@8.24.0)(typescript@4.8.4): - resolution: {integrity: sha512-LDqxZBVFFQnQRz9rUZJhLmox+Ep5kdUmLatLQnCRR6523YV+XhRjfYzStQ4MheFA8kMAfUlclHSbu+RKdRwQKw==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + /@typescript-eslint/parser@6.18.1(eslint@8.24.0)(typescript@5.3.3): + resolution: {integrity: sha512-zct/MdJnVaRRNy9e84XnVtRv9Vf91/qqe+hZJtKanjojud4wAVy/7lXxJmMyX6X6J+xc6c//YEWvpeif8cAhWA==} + engines: {node: ^16.0.0 || >=18.0.0} peerDependencies: - eslint: ^6.0.0 || ^7.0.0 || ^8.0.0 + eslint: ^7.0.0 || ^8.0.0 typescript: '*' peerDependenciesMeta: typescript: optional: true dependencies: - '@typescript-eslint/scope-manager': 5.38.1 - '@typescript-eslint/types': 5.38.1 - '@typescript-eslint/typescript-estree': 5.38.1(typescript@4.8.4) + '@typescript-eslint/scope-manager': 6.18.1 + '@typescript-eslint/types': 6.18.1 + '@typescript-eslint/typescript-estree': 6.18.1(typescript@5.3.3) + '@typescript-eslint/visitor-keys': 6.18.1 debug: 4.3.4 eslint: 8.24.0 - typescript: 4.8.4 + typescript: 5.3.3 transitivePeerDependencies: - supports-color dev: true - /@typescript-eslint/scope-manager@5.38.1: - resolution: {integrity: sha512-BfRDq5RidVU3RbqApKmS7RFMtkyWMM50qWnDAkKgQiezRtLKsoyRKIvz1Ok5ilRWeD9IuHvaidaLxvGx/2eqTQ==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + /@typescript-eslint/scope-manager@6.18.1: + resolution: {integrity: sha512-BgdBwXPFmZzaZUuw6wKiHKIovms97a7eTImjkXCZE04TGHysG+0hDQPmygyvgtkoB/aOQwSM/nWv3LzrOIQOBw==} + engines: {node: ^16.0.0 || >=18.0.0} dependencies: - '@typescript-eslint/types': 5.38.1 - '@typescript-eslint/visitor-keys': 5.38.1 + '@typescript-eslint/types': 6.18.1 + '@typescript-eslint/visitor-keys': 6.18.1 dev: true - /@typescript-eslint/type-utils@5.38.1(eslint@8.24.0)(typescript@4.8.4): - resolution: {integrity: sha512-UU3j43TM66gYtzo15ivK2ZFoDFKKP0k03MItzLdq0zV92CeGCXRfXlfQX5ILdd4/DSpHkSjIgLLLh1NtkOJOAw==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + /@typescript-eslint/type-utils@6.18.1(eslint@8.24.0)(typescript@5.3.3): + resolution: {integrity: sha512-wyOSKhuzHeU/5pcRDP2G2Ndci+4g653V43gXTpt4nbyoIOAASkGDA9JIAgbQCdCkcr1MvpSYWzxTz0olCn8+/Q==} + engines: {node: ^16.0.0 || >=18.0.0} peerDependencies: - eslint: '*' + eslint: ^7.0.0 || ^8.0.0 typescript: '*' peerDependenciesMeta: typescript: optional: true dependencies: - '@typescript-eslint/typescript-estree': 5.38.1(typescript@4.8.4) - '@typescript-eslint/utils': 5.38.1(eslint@8.24.0)(typescript@4.8.4) + '@typescript-eslint/typescript-estree': 6.18.1(typescript@5.3.3) + '@typescript-eslint/utils': 6.18.1(eslint@8.24.0)(typescript@5.3.3) debug: 4.3.4 eslint: 8.24.0 - tsutils: 3.21.0(typescript@4.8.4) - typescript: 4.8.4 + ts-api-utils: 1.0.3(typescript@5.3.3) + typescript: 5.3.3 transitivePeerDependencies: - supports-color dev: true - /@typescript-eslint/types@5.38.1: - resolution: {integrity: sha512-QTW1iHq1Tffp9lNfbfPm4WJabbvpyaehQ0SrvVK2yfV79SytD9XDVxqiPvdrv2LK7DGSFo91TB2FgWanbJAZXg==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + /@typescript-eslint/types@6.18.1: + resolution: {integrity: sha512-4TuMAe+tc5oA7wwfqMtB0Y5OrREPF1GeJBAjqwgZh1lEMH5PJQgWgHGfYufVB51LtjD+peZylmeyxUXPfENLCw==} + engines: {node: ^16.0.0 || >=18.0.0} dev: true - /@typescript-eslint/typescript-estree@5.38.1(typescript@4.8.4): - resolution: {integrity: sha512-99b5e/Enoe8fKMLdSuwrfH/C0EIbpUWmeEKHmQlGZb8msY33qn1KlkFww0z26o5Omx7EVjzVDCWEfrfCDHfE7g==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + /@typescript-eslint/typescript-estree@6.18.1(typescript@5.3.3): + resolution: {integrity: sha512-fv9B94UAhywPRhUeeV/v+3SBDvcPiLxRZJw/xZeeGgRLQZ6rLMG+8krrJUyIf6s1ecWTzlsbp0rlw7n9sjufHA==} + engines: {node: ^16.0.0 || >=18.0.0} peerDependencies: typescript: '*' peerDependenciesMeta: typescript: optional: true dependencies: - '@typescript-eslint/types': 5.38.1 - '@typescript-eslint/visitor-keys': 5.38.1 + '@typescript-eslint/types': 6.18.1 + '@typescript-eslint/visitor-keys': 6.18.1 debug: 4.3.4 globby: 11.1.0 is-glob: 4.0.3 + minimatch: 9.0.3 semver: 7.5.4 - tsutils: 3.21.0(typescript@4.8.4) - typescript: 4.8.4 + ts-api-utils: 1.0.3(typescript@5.3.3) + typescript: 5.3.3 transitivePeerDependencies: - supports-color dev: true - /@typescript-eslint/utils@5.38.1(eslint@8.24.0)(typescript@4.8.4): - resolution: {integrity: sha512-oIuUiVxPBsndrN81oP8tXnFa/+EcZ03qLqPDfSZ5xIJVm7A9V0rlkQwwBOAGtrdN70ZKDlKv+l1BeT4eSFxwXA==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + /@typescript-eslint/utils@6.18.1(eslint@8.24.0)(typescript@5.3.3): + resolution: {integrity: sha512-zZmTuVZvD1wpoceHvoQpOiewmWu3uP9FuTWo8vqpy2ffsmfCE8mklRPi+vmnIYAIk9t/4kOThri2QCDgor+OpQ==} + engines: {node: ^16.0.0 || >=18.0.0} peerDependencies: - eslint: ^6.0.0 || ^7.0.0 || ^8.0.0 + eslint: ^7.0.0 || ^8.0.0 dependencies: + '@eslint-community/eslint-utils': 4.4.0(eslint@8.24.0) '@types/json-schema': 7.0.12 - '@typescript-eslint/scope-manager': 5.38.1 - '@typescript-eslint/types': 5.38.1 - '@typescript-eslint/typescript-estree': 5.38.1(typescript@4.8.4) + '@types/semver': 7.5.0 + '@typescript-eslint/scope-manager': 6.18.1 + '@typescript-eslint/types': 6.18.1 + '@typescript-eslint/typescript-estree': 6.18.1(typescript@5.3.3) eslint: 8.24.0 - eslint-scope: 5.1.1 - eslint-utils: 3.0.0(eslint@8.24.0) + semver: 7.5.4 transitivePeerDependencies: - supports-color - typescript dev: true - /@typescript-eslint/visitor-keys@5.38.1: - resolution: {integrity: sha512-bSHr1rRxXt54+j2n4k54p4fj8AHJ49VDWtjpImOpzQj4qjAiOpPni+V1Tyajh19Api1i844F757cur8wH3YvOA==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + /@typescript-eslint/visitor-keys@6.18.1: + resolution: {integrity: sha512-/kvt0C5lRqGoCfsbmm7/CwMqoSkY3zzHLIjdhHZQW3VFrnz7ATecOHR7nb7V+xn4286MBxfnQfQhAmCI0u+bJA==} + engines: {node: ^16.0.0 || >=18.0.0} dependencies: - '@typescript-eslint/types': 5.38.1 + '@typescript-eslint/types': 6.18.1 eslint-visitor-keys: 3.4.3 dev: true + /@typescript/vfs@1.5.0: + resolution: {integrity: sha512-AJS307bPgbsZZ9ggCT3wwpg3VbTKMFNHfaY/uF0ahSkYYrPF2dSSKDNIDIQAHm9qJqbLvCsSJH7yN4Vs/CsMMg==} + dependencies: + debug: 4.3.4 + transitivePeerDependencies: + - supports-color + dev: true + /abbrev@1.1.1: resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==} dev: true @@ -7447,14 +7692,6 @@ packages: prettier-linter-helpers: 1.0.0 dev: true - /eslint-scope@5.1.1: - resolution: {integrity: sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==} - engines: {node: '>=8.0.0'} - dependencies: - esrecurse: 4.3.0 - estraverse: 4.3.0 - dev: true - /eslint-scope@7.2.2: resolution: {integrity: sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -7565,11 +7802,6 @@ packages: estraverse: 5.3.0 dev: true - /estraverse@4.3.0: - resolution: {integrity: sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==} - engines: {node: '>=4.0'} - dev: true - /estraverse@5.3.0: resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} engines: {node: '>=4.0'} @@ -8088,7 +8320,6 @@ packages: /graphemer@1.4.0: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} - dev: false /handlebars-jest@1.0.0: resolution: {integrity: sha512-giA9RSHNLKOqFU2dJ3QapELUJmXb4wmQWIEPc5cYp3Sx4Nwo01PBsTWrwo28cGWC8gRg+seMVMBi7wQtcaqw3g==} @@ -8698,7 +8929,7 @@ packages: pretty-format: 28.1.3 slash: 3.0.0 strip-json-comments: 3.1.1 - ts-node: 10.8.2(@swc/core@1.3.42)(@types/node@18.0.0)(typescript@4.8.4) + ts-node: 10.8.2(@swc/core@1.3.42)(@types/node@18.0.0)(typescript@5.3.3) transitivePeerDependencies: - supports-color dev: true @@ -8738,7 +8969,7 @@ packages: pretty-format: 28.1.3 slash: 3.0.0 strip-json-comments: 3.1.1 - ts-node: 10.8.2(@swc/core@1.3.42)(@types/node@18.0.0)(typescript@4.8.4) + ts-node: 10.8.2(@swc/core@1.3.42)(@types/node@18.0.0)(typescript@5.3.3) transitivePeerDependencies: - supports-color dev: true @@ -9421,6 +9652,13 @@ packages: dependencies: brace-expansion: 2.0.1 + /minimatch@9.0.3: + resolution: {integrity: sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==} + engines: {node: '>=16 || 14 >=14.17'} + dependencies: + brace-expansion: 2.0.1 + dev: true + /minimist-options@4.1.0: resolution: {integrity: sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==} engines: {node: '>= 6'} @@ -11074,6 +11312,15 @@ packages: engines: {node: '>=8'} dev: true + /ts-api-utils@1.0.3(typescript@5.3.3): + resolution: {integrity: sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg==} + engines: {node: '>=16.13.0'} + peerDependencies: + typescript: '>=4.2.0' + dependencies: + typescript: 5.3.3 + dev: true + /ts-morph@16.0.0: resolution: {integrity: sha512-jGNF0GVpFj0orFw55LTsQxVYEUOCWBAbR5Ls7fTYE5pQsbW18ssTb/6UXx/GYAEjS+DQTp8VoTw0vqYMiaaQuw==} dependencies: @@ -11081,7 +11328,7 @@ packages: code-block-writer: 11.0.3 dev: false - /ts-node@10.8.2(@swc/core@1.3.42)(@types/node@18.0.0)(typescript@4.8.4): + /ts-node@10.8.2(@swc/core@1.3.42)(@types/node@18.0.0)(typescript@5.3.3): resolution: {integrity: sha512-LYdGnoGddf1D6v8REPtIH+5iq/gTDuZqv2/UJUU7tKjuEU8xVZorBM+buCGNjj+pGEud+sOoM4CX3/YzINpENA==} hasBin: true peerDependencies: @@ -11108,12 +11355,12 @@ packages: create-require: 1.1.1 diff: 4.0.2 make-error: 1.3.6 - typescript: 4.8.4 + typescript: 5.3.3 v8-compile-cache-lib: 3.0.1 yn: 3.1.1 dev: true - /ts-node@10.8.2(@swc/core@1.3.42)(@types/node@18.17.8)(typescript@4.8.4): + /ts-node@10.8.2(@swc/core@1.3.42)(@types/node@18.17.8)(typescript@5.3.3): resolution: {integrity: sha512-LYdGnoGddf1D6v8REPtIH+5iq/gTDuZqv2/UJUU7tKjuEU8xVZorBM+buCGNjj+pGEud+sOoM4CX3/YzINpENA==} hasBin: true peerDependencies: @@ -11140,13 +11387,14 @@ packages: create-require: 1.1.1 diff: 4.0.2 make-error: 1.3.6 - typescript: 4.8.4 + typescript: 5.3.3 v8-compile-cache-lib: 3.0.1 yn: 3.1.1 dev: true /tslib@1.14.1: resolution: {integrity: sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==} + dev: false /tslib@2.3.1: resolution: {integrity: sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==} @@ -11156,16 +11404,6 @@ packages: resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} requiresBuild: true - /tsutils@3.21.0(typescript@4.8.4): - resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==} - engines: {node: '>= 6'} - peerDependencies: - typescript: '>=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta' - dependencies: - tslib: 1.14.1 - typescript: 4.8.4 - dev: true - /tty-table@4.2.1: resolution: {integrity: sha512-xz0uKo+KakCQ+Dxj1D/tKn2FSyreSYWzdkL/BYhgN6oMW808g8QRMuh1atAV9fjTPbWBjfbkKQpI/5rEcnAc7g==} engines: {node: '>=8.0.0'} @@ -11276,12 +11514,18 @@ packages: optionalDependencies: rxjs: 7.8.1 - /typescript@4.8.4: - resolution: {integrity: sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==} + /typescript@4.5.2: + resolution: {integrity: sha512-5BlMof9H1yGt0P8/WF+wPNw6GfctgGjXp5hkblpyT+8rkASSmkUKMXrxR0Xg8ThVCi/JnHQiKXeBaEwCeQwMFw==} engines: {node: '>=4.2.0'} hasBin: true dev: true + /typescript@5.3.3: + resolution: {integrity: sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==} + engines: {node: '>=14.17'} + hasBin: true + dev: true + /uglify-js@3.17.4: resolution: {integrity: sha512-T9q82TJI9e/C1TAxYvfb16xO120tMVFZrGA3f9/P4424DNu6ypK103y0GPFVa17yotwSyZW5iYXgjYHkGrJW/g==} engines: {node: '>=0.8.0'} @@ -11303,6 +11547,13 @@ packages: which-boxed-primitive: 1.0.2 dev: true + /undici@5.28.2: + resolution: {integrity: sha512-wh1pHJHnUeQV5Xa8/kyQhO7WFa8M34l026L5P/+2TYiakvGy5Rdc8jWZVyG7ieht/0WgJLEd3kcU5gKx+6GC8w==} + engines: {node: '>=14.0'} + dependencies: + '@fastify/busboy': 2.1.0 + dev: false + /unicode-canonical-property-names-ecmascript@2.0.0: resolution: {integrity: sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==} engines: {node: '>=4'} diff --git a/services/bsync/Dockerfile b/services/bsync/Dockerfile new file mode 100644 index 00000000000..44b2704f26a --- /dev/null +++ b/services/bsync/Dockerfile @@ -0,0 +1,49 @@ +FROM node:18-alpine as build + +RUN npm install -g pnpm + +# Move files into the image and install +WORKDIR /app +COPY ./*.* ./ +# NOTE bsync's transitive dependencies go here: if that changes, this needs to be updated. +COPY ./packages/bsync ./packages/bsync +COPY ./packages/common ./packages/common +COPY ./packages/common-web ./packages/common-web +COPY ./packages/syntax ./packages/syntax + + +# install all deps +RUN pnpm install --frozen-lockfile > /dev/null +# build all packages with external node_modules +RUN ATP_BUILD_SHALLOW=true pnpm build > /dev/null +# update main with publishConfig +RUN pnpm update-main-to-dist > /dev/null +# clean up +RUN rm -rf node_modules +# install only prod deps, hoisted to root node_modules dir +RUN pnpm install --prod --shamefully-hoist --frozen-lockfile --prefer-offline > /dev/null + +WORKDIR services/bsync + +# Uses assets from build stage to reduce build size +FROM node:18-alpine + +RUN apk add --update dumb-init + +# Avoid zombie processes, handle signal forwarding +ENTRYPOINT ["dumb-init", "--"] + +WORKDIR /app/services/bsync +COPY --from=build /app /app + +EXPOSE 3000 +ENV PORT=3000 +ENV NODE_ENV=production + +# https://github.com/nodejs/docker-node/blob/master/docs/BestPractices.md#non-root-user +USER node +CMD ["node", "--enable-source-maps", "index.js"] + +LABEL org.opencontainers.image.source=https://github.com/bluesky-social/atproto +LABEL org.opencontainers.image.description="Bsync" +LABEL org.opencontainers.image.licenses=MIT diff --git a/services/bsync/index.js b/services/bsync/index.js new file mode 100644 index 00000000000..b9d22da8855 --- /dev/null +++ b/services/bsync/index.js @@ -0,0 +1,27 @@ +'use strict' /* eslint-disable */ + +require('dd-trace') // Only works with commonjs + .init({ logInjection: true }) + +// Tracer code above must come before anything else +const { + envToCfg, + readEnv, + httpLogger, + default: BsyncService, +} = require('@atproto/bsync') + +const main = async () => { + const env = readEnv() + const cfg = envToCfg(env) + const bsync = await BsyncService.create(cfg) + await bsync.start() + httpLogger.info('bsync is running') + process.on('SIGTERM', async () => { + httpLogger.info('bsync is stopping') + await bsync.destroy() + httpLogger.info('bsync is stopped') + }) +} + +main() diff --git a/services/bsync/package.json b/services/bsync/package.json new file mode 100644 index 00000000000..bf31aacb1a1 --- /dev/null +++ b/services/bsync/package.json @@ -0,0 +1,8 @@ +{ + "name": "bsync-service", + "private": true, + "dependencies": { + "@atproto/bsync": "workspace:^", + "dd-trace": "3.13.2" + } +} diff --git a/tsconfig.json b/tsconfig.json index 3307ccb7e4b..75a29d07106 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -25,6 +25,7 @@ "references": [ { "path": "./packages/pds/tsconfig.build.json" }, { "path": "./packages/bsky/tsconfig.build.json" }, + { "path": "./packages/bsync/tsconfig.build.json" }, { "path": "./packages/ozone/tsconfig.build.json" }, { "path": "./packages/api/tsconfig.build.json" }, { "path": "./packages/aws/tsconfig.build.json" },