Prepare docker builds of bsync (#2053)

* update bsync protos

* support optional migrations in bsync service, docker build fixes

* setup workflow for bsync build

* fix bsync image name

.github/workflows/build-and-push-bsync-aws.yaml

@@ -0,0 +1,55 @@
+name: build-and-push-bsync-aws
+on:
+  push:
+    branches:
+      - bsync-build
+      - services/bsync
+env:
+  REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }}
+  USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }}
+  PASSWORD: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_PASSWORD }}
+  IMAGE_NAME: bsync
+
+jobs:
+  bsync-container-aws:
+    if: github.repository == 'bluesky-social/atproto'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ env.USERNAME}}
+          password: ${{ env.PASSWORD }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=sha,enable=true,priority=100,prefix=,suffix=,format=long
+
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ./services/bsync/Dockerfile
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/build-and-push-bsync-ghcr.yaml

@@ -0,0 +1,57 @@
+name: build-and-push-bsync-ghcr
+on:
+  push:
+    branches:
+      - bsync-build
+      - services/bsync
+env:
+  REGISTRY: ghcr.io
+  USERNAME: ${{ github.actor }}
+  PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  bsync-container-ghcr:
+    if: github.repository == 'bluesky-social/atproto'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ env.USERNAME }}
+          password: ${{ env.PASSWORD }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=sha,enable=true,priority=100,prefix=bsync:,suffix=,format=long
+
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ./services/bsync/Dockerfile
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

packages/bsync/proto/bsync.proto

@@ -10,9 +10,10 @@ option go_package = "./;bsync";
 
 message MuteOperation {
   enum Type {
-    ADD = 0;
-    REMOVE = 1;
-    CLEAR = 2;
+    TYPE_UNSPECIFIED = 0;
+    TYPE_ADD = 1;
+    TYPE_REMOVE = 2;
+    TYPE_CLEAR = 3;
   }
   string id = 1;
   Type type = 2;

packages/bsync/src/config.ts

@@ -1,5 +1,5 @@
 import assert from 'node:assert'
-import { envInt, envStr, envList } from '@atproto/common'
+import { envInt, envStr, envList, envBool } from '@atproto/common'
 
 export const envToCfg = (env: ServerEnvironment): ServerConfig => {
   const serviceCfg: ServerConfig['service'] = {
@@ -15,6 +15,7 @@ export const envToCfg = (env: ServerEnvironment): ServerConfig => {
     poolSize: env.dbPoolSize,
     poolMaxUses: env.dbPoolMaxUses,
     poolIdleTimeoutMs: env.dbPoolIdleTimeoutMs,
+    migrate: env.dbMigrate,
   }
 
   assert(env.apiKeys.length > 0, 'missing api keys')
@@ -47,6 +48,7 @@ type DatabaseConfig = {
   poolSize?: number
   poolMaxUses?: number
   poolIdleTimeoutMs?: number
+  migrate?: boolean
 }
 
 type AuthConfig = {
@@ -65,6 +67,7 @@ export const readEnv = (): ServerEnvironment => {
     dbPoolSize: envInt('BSYNC_DB_POOL_SIZE'),
     dbPoolMaxUses: envInt('BSYNC_DB_POOL_MAX_USES'),
     dbPoolIdleTimeoutMs: envInt('BSYNC_DB_POOL_IDLE_TIMEOUT_MS'),
+    dbMigrate: envBool('BSYNC_DB_MIGRATE'),
     // secrets
     apiKeys: envList('BSYNC_API_KEYS'),
   }
@@ -81,6 +84,7 @@ export type ServerEnvironment = {
   dbPoolSize?: number
   dbPoolMaxUses?: number
   dbPoolIdleTimeoutMs?: number
+  dbMigrate?: boolean
   // secrets
   apiKeys: string[]
 }

packages/bsync/src/gen/bsync_pb.ts

@@ -25,7 +25,7 @@ export class MuteOperation extends Message<MuteOperation> {
   /**
    * @generated from field: bsync.MuteOperation.Type type = 2;
    */
-  type = MuteOperation_Type.ADD
+  type = MuteOperation_Type.UNSPECIFIED
 
   /**
    * @generated from field: string actor_did = 3;
@@ -90,25 +90,31 @@ export class MuteOperation extends Message<MuteOperation> {
  */
 export enum MuteOperation_Type {
   /**
-   * @generated from enum value: ADD = 0;
+   * @generated from enum value: TYPE_UNSPECIFIED = 0;
    */
-  ADD = 0,
+  UNSPECIFIED = 0,
 
   /**
-   * @generated from enum value: REMOVE = 1;
+   * @generated from enum value: TYPE_ADD = 1;
    */
-  REMOVE = 1,
+  ADD = 1,
 
   /**
-   * @generated from enum value: CLEAR = 2;
+   * @generated from enum value: TYPE_REMOVE = 2;
    */
-  CLEAR = 2,
+  REMOVE = 2,
+
+  /**
+   * @generated from enum value: TYPE_CLEAR = 3;
+   */
+  CLEAR = 3,
 }
 // Retrieve enum metadata with: proto3.getEnumType(MuteOperation_Type)
 proto3.util.setEnumType(MuteOperation_Type, 'bsync.MuteOperation.Type', [
-  { no: 0, name: 'ADD' },
-  { no: 1, name: 'REMOVE' },
-  { no: 2, name: 'CLEAR' },
+  { no: 0, name: 'TYPE_UNSPECIFIED' },
+  { no: 1, name: 'TYPE_ADD' },
+  { no: 2, name: 'TYPE_REMOVE' },
+  { no: 3, name: 'TYPE_CLEAR' },
 ])
 
 /**
@@ -118,7 +124,7 @@ export class AddMuteOperationRequest extends Message<AddMuteOperationRequest> {
   /**
    * @generated from field: bsync.MuteOperation.Type type = 1;
    */
-  type = MuteOperation_Type.ADD
+  type = MuteOperation_Type.UNSPECIFIED
 
   /**
    * @generated from field: string actor_did = 2;

packages/bsync/src/routes/add-mute-operation.ts

@@ -92,10 +92,16 @@ const clearMuteItems = async (db: Database, op: MuteOpInfo) => {
     .execute()
 }
 
-const validMuteOp = (op: MuteOpInfo): MuteOpInfo => {
+const validMuteOp = (op: MuteOpInfo): MuteOpInfoValid => {
   if (!Object.values(MuteOperation_Type).includes(op.type)) {
     throw new ConnectError('bad mute operation type', Code.InvalidArgument)
   }
+  if (op.type === MuteOperation_Type.UNSPECIFIED) {
+    throw new ConnectError(
+      'unspecified mute operation type',
+      Code.InvalidArgument,
+    )
+  }
   if (!isValidDid(op.actorDid)) {
     throw new ConnectError(
       'actor_did must be a valid did',
@@ -127,7 +133,7 @@ const validMuteOp = (op: MuteOpInfo): MuteOpInfo => {
       )
     }
   }
-  return op
+  return op as MuteOpInfoValid // op.type has been checked
 }
 
 const isValidDid = (did: string) => {
@@ -156,3 +162,12 @@ type MuteOpInfo = {
   actorDid: string
   subject: string
 }
+
+type MuteOpInfoValid = {
+  type:
+    | MuteOperation_Type.ADD
+    | MuteOperation_Type.REMOVE
+    | MuteOperation_Type.CLEAR
+  actorDid: string
+  subject: string
+}

services/bsync/Dockerfile

@@ -10,6 +10,7 @@ COPY ./packages/bsync ./packages/bsync
 COPY ./packages/common ./packages/common
 COPY ./packages/common-web ./packages/common-web
 COPY ./packages/syntax ./packages/syntax
+COPY ./services/bsync ./services/bsync
 
 
 # install all deps
@@ -37,7 +38,7 @@ WORKDIR /app/services/bsync
 COPY --from=build /app /app
 
 EXPOSE 3000
-ENV PORT=3000
+ENV BSYNC_PORT=3000
 ENV NODE_ENV=production
 
 # https://github.com/nodejs/docker-node/blob/master/docs/BestPractices.md#non-root-user

services/bsync/index.js

@@ -15,6 +15,11 @@ const main = async () => {
   const env = readEnv()
   const cfg = envToCfg(env)
   const bsync = await BsyncService.create(cfg)
+  if (bsync.ctx.cfg.db.migrate) {
+    httpLogger.info('bsync db is migrating')
+    await bsync.ctx.db.migrateToLatestOrThrow()
+    httpLogger.info('bsync db migration complete')
+  }
   await bsync.start()
   httpLogger.info('bsync is running')
   process.on('SIGTERM', async () => {