From de398b790cfb14771765825c51ece9a3db71bf33 Mon Sep 17 00:00:00 2001 From: devin ivy Date: Thu, 18 Jan 2024 14:21:55 -0500 Subject: [PATCH] Prepare docker builds of bsync (#2053) * update bsync protos * support optional migrations in bsync service, docker build fixes * setup workflow for bsync build * fix bsync image name --- .../workflows/build-and-push-bsync-aws.yaml | 55 ++++++++++++++++++ .../workflows/build-and-push-bsync-ghcr.yaml | 57 +++++++++++++++++++ packages/bsync/proto/bsync.proto | 7 ++- packages/bsync/src/config.ts | 6 +- packages/bsync/src/gen/bsync_pb.ts | 28 +++++---- .../bsync/src/routes/add-mute-operation.ts | 19 ++++++- services/bsync/Dockerfile | 3 +- services/bsync/index.js | 5 ++ 8 files changed, 162 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/build-and-push-bsync-aws.yaml create mode 100644 .github/workflows/build-and-push-bsync-ghcr.yaml diff --git a/.github/workflows/build-and-push-bsync-aws.yaml b/.github/workflows/build-and-push-bsync-aws.yaml new file mode 100644 index 00000000000..de27a1fde9a --- /dev/null +++ b/.github/workflows/build-and-push-bsync-aws.yaml @@ -0,0 +1,55 @@ +name: build-and-push-bsync-aws +on: + push: + branches: + - bsync-build + - services/bsync +env: + REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }} + USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }} + PASSWORD: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_PASSWORD }} + IMAGE_NAME: bsync + +jobs: + bsync-container-aws: + if: github.repository == 'bluesky-social/atproto' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v2 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.USERNAME}} + password: ${{ env.PASSWORD }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + file: ./services/bsync/Dockerfile + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/build-and-push-bsync-ghcr.yaml b/.github/workflows/build-and-push-bsync-ghcr.yaml new file mode 100644 index 00000000000..8ecb3a5b8d5 --- /dev/null +++ b/.github/workflows/build-and-push-bsync-ghcr.yaml @@ -0,0 +1,57 @@ +name: build-and-push-bsync-ghcr +on: + push: + branches: + - bsync-build + - services/bsync +env: + REGISTRY: ghcr.io + USERNAME: ${{ github.actor }} + PASSWORD: ${{ secrets.GITHUB_TOKEN }} + + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + +jobs: + bsync-container-ghcr: + if: github.repository == 'bluesky-social/atproto' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v2 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.USERNAME }} + password: ${{ env.PASSWORD }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=bsync:,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + file: ./services/bsync/Dockerfile + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/packages/bsync/proto/bsync.proto b/packages/bsync/proto/bsync.proto index d8b49dc56f4..5ef37b607b6 100644 --- a/packages/bsync/proto/bsync.proto +++ b/packages/bsync/proto/bsync.proto @@ -10,9 +10,10 @@ option go_package = "./;bsync"; message MuteOperation { enum Type { - ADD = 0; - REMOVE = 1; - CLEAR = 2; + TYPE_UNSPECIFIED = 0; + TYPE_ADD = 1; + TYPE_REMOVE = 2; + TYPE_CLEAR = 3; } string id = 1; Type type = 2; diff --git a/packages/bsync/src/config.ts b/packages/bsync/src/config.ts index ee2385c0a7a..d5ea4454c2d 100644 --- a/packages/bsync/src/config.ts +++ b/packages/bsync/src/config.ts @@ -1,5 +1,5 @@ import assert from 'node:assert' -import { envInt, envStr, envList } from '@atproto/common' +import { envInt, envStr, envList, envBool } from '@atproto/common' export const envToCfg = (env: ServerEnvironment): ServerConfig => { const serviceCfg: ServerConfig['service'] = { @@ -15,6 +15,7 @@ export const envToCfg = (env: ServerEnvironment): ServerConfig => { poolSize: env.dbPoolSize, poolMaxUses: env.dbPoolMaxUses, poolIdleTimeoutMs: env.dbPoolIdleTimeoutMs, + migrate: env.dbMigrate, } assert(env.apiKeys.length > 0, 'missing api keys') @@ -47,6 +48,7 @@ type DatabaseConfig = { poolSize?: number poolMaxUses?: number poolIdleTimeoutMs?: number + migrate?: boolean } type AuthConfig = { @@ -65,6 +67,7 @@ export const readEnv = (): ServerEnvironment => { dbPoolSize: envInt('BSYNC_DB_POOL_SIZE'), dbPoolMaxUses: envInt('BSYNC_DB_POOL_MAX_USES'), dbPoolIdleTimeoutMs: envInt('BSYNC_DB_POOL_IDLE_TIMEOUT_MS'), + dbMigrate: envBool('BSYNC_DB_MIGRATE'), // secrets apiKeys: envList('BSYNC_API_KEYS'), } @@ -81,6 +84,7 @@ export type ServerEnvironment = { dbPoolSize?: number dbPoolMaxUses?: number dbPoolIdleTimeoutMs?: number + dbMigrate?: boolean // secrets apiKeys: string[] } diff --git a/packages/bsync/src/gen/bsync_pb.ts b/packages/bsync/src/gen/bsync_pb.ts index 932bb5337e0..aafd4012b25 100644 --- a/packages/bsync/src/gen/bsync_pb.ts +++ b/packages/bsync/src/gen/bsync_pb.ts @@ -25,7 +25,7 @@ export class MuteOperation extends Message { /** * @generated from field: bsync.MuteOperation.Type type = 2; */ - type = MuteOperation_Type.ADD + type = MuteOperation_Type.UNSPECIFIED /** * @generated from field: string actor_did = 3; @@ -90,25 +90,31 @@ export class MuteOperation extends Message { */ export enum MuteOperation_Type { /** - * @generated from enum value: ADD = 0; + * @generated from enum value: TYPE_UNSPECIFIED = 0; */ - ADD = 0, + UNSPECIFIED = 0, /** - * @generated from enum value: REMOVE = 1; + * @generated from enum value: TYPE_ADD = 1; */ - REMOVE = 1, + ADD = 1, /** - * @generated from enum value: CLEAR = 2; + * @generated from enum value: TYPE_REMOVE = 2; */ - CLEAR = 2, + REMOVE = 2, + + /** + * @generated from enum value: TYPE_CLEAR = 3; + */ + CLEAR = 3, } // Retrieve enum metadata with: proto3.getEnumType(MuteOperation_Type) proto3.util.setEnumType(MuteOperation_Type, 'bsync.MuteOperation.Type', [ - { no: 0, name: 'ADD' }, - { no: 1, name: 'REMOVE' }, - { no: 2, name: 'CLEAR' }, + { no: 0, name: 'TYPE_UNSPECIFIED' }, + { no: 1, name: 'TYPE_ADD' }, + { no: 2, name: 'TYPE_REMOVE' }, + { no: 3, name: 'TYPE_CLEAR' }, ]) /** @@ -118,7 +124,7 @@ export class AddMuteOperationRequest extends Message { /** * @generated from field: bsync.MuteOperation.Type type = 1; */ - type = MuteOperation_Type.ADD + type = MuteOperation_Type.UNSPECIFIED /** * @generated from field: string actor_did = 2; diff --git a/packages/bsync/src/routes/add-mute-operation.ts b/packages/bsync/src/routes/add-mute-operation.ts index 6e433c0aa4e..fd535cd824b 100644 --- a/packages/bsync/src/routes/add-mute-operation.ts +++ b/packages/bsync/src/routes/add-mute-operation.ts @@ -92,10 +92,16 @@ const clearMuteItems = async (db: Database, op: MuteOpInfo) => { .execute() } -const validMuteOp = (op: MuteOpInfo): MuteOpInfo => { +const validMuteOp = (op: MuteOpInfo): MuteOpInfoValid => { if (!Object.values(MuteOperation_Type).includes(op.type)) { throw new ConnectError('bad mute operation type', Code.InvalidArgument) } + if (op.type === MuteOperation_Type.UNSPECIFIED) { + throw new ConnectError( + 'unspecified mute operation type', + Code.InvalidArgument, + ) + } if (!isValidDid(op.actorDid)) { throw new ConnectError( 'actor_did must be a valid did', @@ -127,7 +133,7 @@ const validMuteOp = (op: MuteOpInfo): MuteOpInfo => { ) } } - return op + return op as MuteOpInfoValid // op.type has been checked } const isValidDid = (did: string) => { @@ -156,3 +162,12 @@ type MuteOpInfo = { actorDid: string subject: string } + +type MuteOpInfoValid = { + type: + | MuteOperation_Type.ADD + | MuteOperation_Type.REMOVE + | MuteOperation_Type.CLEAR + actorDid: string + subject: string +} diff --git a/services/bsync/Dockerfile b/services/bsync/Dockerfile index 44b2704f26a..515bcfa02c3 100644 --- a/services/bsync/Dockerfile +++ b/services/bsync/Dockerfile @@ -10,6 +10,7 @@ COPY ./packages/bsync ./packages/bsync COPY ./packages/common ./packages/common COPY ./packages/common-web ./packages/common-web COPY ./packages/syntax ./packages/syntax +COPY ./services/bsync ./services/bsync # install all deps @@ -37,7 +38,7 @@ WORKDIR /app/services/bsync COPY --from=build /app /app EXPOSE 3000 -ENV PORT=3000 +ENV BSYNC_PORT=3000 ENV NODE_ENV=production # https://github.com/nodejs/docker-node/blob/master/docs/BestPractices.md#non-root-user diff --git a/services/bsync/index.js b/services/bsync/index.js index b9d22da8855..c58cf839a3c 100644 --- a/services/bsync/index.js +++ b/services/bsync/index.js @@ -15,6 +15,11 @@ const main = async () => { const env = readEnv() const cfg = envToCfg(env) const bsync = await BsyncService.create(cfg) + if (bsync.ctx.cfg.db.migrate) { + httpLogger.info('bsync db is migrating') + await bsync.ctx.db.migrateToLatestOrThrow() + httpLogger.info('bsync db migration complete') + } await bsync.start() httpLogger.info('bsync is running') process.on('SIGTERM', async () => {