feat: add cron job for copilot test

.github/actions/deploy/deploy.mjs

@@ -18,6 +18,8 @@ const {
   COPILOT_OPENAI_API_KEY,
   COPILOT_FAL_API_KEY,
   COPILOT_UNSPLASH_API_KEY,
+  SLACK_BOT_TOKEN,
+  RELEASE_SLACK_CHNNEL_ID,
   MAILER_SENDER,
   MAILER_USER,
   MAILER_PASSWORD,
@@ -148,6 +150,8 @@ const createHelmCommand = ({ isDryRun }) => {
     `--set-string graphql.app.copilot.openai.key="${COPILOT_OPENAI_API_KEY}"`,
     `--set-string graphql.app.copilot.fal.key="${COPILOT_FAL_API_KEY}"`,
     `--set-string graphql.app.copilot.unsplash.key="${COPILOT_UNSPLASH_API_KEY}"`,
+    `--set-string graphql.app.copilot.slack.botToken="${SLACK_BOT_TOKEN}"`,
+    `--set-string graphql.app.copilot.slack.channelId="${RELEASE_SLACK_CHNNEL_ID}"`,
     `--set-string graphql.app.mailer.sender="${MAILER_SENDER}"`,
     `--set-string graphql.app.mailer.user="${MAILER_USER}"`,
     `--set-string graphql.app.mailer.password="${MAILER_PASSWORD}"`,

.github/helm/affine/charts/graphql/templates/copilot-secret.yaml

@@ -8,4 +8,6 @@ data:
   openaiSecret: {{ .Values.app.copilot.openai.key | b64enc }}
   falSecret: {{ .Values.app.copilot.fal.key | b64enc }}
   unsplashSecret: {{ .Values.app.copilot.unsplash.key | b64enc }}
+  slackBotToken: {{ .Values.app.copilot.slack.botToken | b64enc }}
+  slackChannelId: {{ .Values.app.copilot.slack.channelId | b64enc }}
 {{- end }}

.github/helm/affine/charts/graphql/templates/copilot-test.yaml

@@ -0,0 +1,66 @@
+{{ if .Values.app.copilot.enabled }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "graphql.fullname" . }}-copilot-test
+  labels:
+    {{- include "graphql.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "1"
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  schedule: "0 8 * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          serviceAccountName: {{ include "graphql.serviceAccountName" . }}
+          containers:
+          - name: {{ .Chart.Name }}
+            image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+            command: ["yarn", "test:copilot:e2e:cron"]
+            env:
+              - name: AFFINE_ENV
+                value: "{{ .Release.Namespace }}"
+              - name: SLACK_BOT_TOKEN
+                valueFrom:
+                  secretKeyRef:
+                    name: "{{ .Values.app.copilot.secretName }}"
+                    key: slackBotToken
+              - name: CHANNEL_ID
+                valueFrom:
+                  secretKeyRef:
+                    name: "{{ .Values.app.copilot.secretName }}"
+                    key: slackChannelId
+              - name: COPILOT_E2E_ENDPOINT
+                value: "http://{{ include "graphql.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:3000"
+              - name: DATABASE_PASSWORD
+                valueFrom:
+                  secretKeyRef:
+                    name: pg-postgresql
+                    key: postgres-password
+              - name: DATABASE_URL
+                value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.url }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }}
+              - name: COPILOT_OPENAI_API_KEY
+                valueFrom:
+                  secretKeyRef:
+                    name: "{{ .Values.app.copilot.secretName }}"
+                    key: openaiSecret
+              - name: COPILOT_FAL_API_KEY
+                valueFrom:
+                  secretKeyRef:
+                    name: "{{ .Values.app.copilot.secretName }}"
+                    key: falSecret
+              - name: COPILOT_UNSPLASH_API_KEY
+                valueFrom:
+                  secretKeyRef:
+                    name: "{{ .Values.app.copilot.secretName }}"
+                    key: unsplashSecret
+            resources:
+              requests:
+                cpu: '100m'
+                memory: '200Mi'
+          restartPolicy: Never
+      backoffLimit: 1
+{{ end }}

.github/workflows/deploy.yml

@@ -101,6 +101,9 @@ jobs:
           COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }}
           COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }}
           COPILOT_UNSPLASH_API_KEY: ${{ secrets.COPILOT_UNSPLASH_API_KEY }}
+          # used for slack notifications
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          RELEASE_SLACK_CHNNEL_ID: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
           METRICS_CUSTOMER_IO_TOKEN: ${{ secrets.METRICS_CUSTOMER_IO_TOKEN }}
           MAILER_SENDER: ${{ secrets.OAUTH_EMAIL_SENDER }}
           MAILER_USER: ${{ secrets.OAUTH_EMAIL_LOGIN }}

packages/backend/native/index.d.ts

@@ -19,4 +19,3 @@ export declare function mergeUpdatesInApplyWay(updates: Array<Buffer>): Buffer
 export declare function mintChallengeResponse(resource: string, bits?: number | undefined | null): Promise<string>
 
 export declare function verifyChallengeResponse(response: string, bits: number, resource: string): Promise<boolean>
-

packages/backend/server/package.json

@@ -12,11 +12,12 @@
     "start": "node --loader ts-node/esm/transpile-only.mjs ./src/index.ts",
     "dev": "nodemon ./src/index.ts",
     "test": "ava --concurrency 1 --serial",
-    "test:copilot:e2e": "ava \"e2e/copilot.e2e.ts\"",
+    "test:copilot:e2e": "ava \"tests/**/copilot-*.e2e.ts\"",
     "test:copilot:spec": "ava \"tests/**/copilot-*.spec.ts\"",
     "test:coverage": "c8 ava --concurrency 1 --serial",
-    "test:copilot:e2e:coverage": "c8 ava --timeout=5m \"e2e/copilot.e2e.ts\"",
+    "test:copilot:e2e:coverage": "c8 ava --timeout=5m \"tests/**/copilot-*.e2e.ts\"",
     "test:copilot:spec:coverage": "c8 ava --timeout=5m \"tests/**/copilot-*.spec.ts\"",
+    "test:copilot:e2e:cron": "node ./scripts/copilot-cron-test.js",
     "postinstall": "prisma generate",
     "data-migration": "node --loader ts-node/esm/transpile-only.mjs ./src/data/index.ts",
     "predeploy": "yarn prisma migrate deploy && node --import ./scripts/register.js ./dist/data/index.js run",
@@ -59,7 +60,9 @@
     "@opentelemetry/semantic-conventions": "^1.25.0",
     "@prisma/client": "^5.15.0",
     "@prisma/instrumentation": "^5.15.0",
+    "@slack/web-api": "^7.3.4",
     "@socket.io/redis-adapter": "^8.3.0",
+    "ava": "^6.1.2",
     "cookie-parser": "^1.4.6",
     "dotenv": "^16.4.5",
     "express": "^4.19.2",
@@ -71,8 +74,10 @@
     "html-validate": "^8.20.1",
     "ioredis": "^5.3.2",
     "is-mobile": "^5.0.0",
+    "jsx-slack": "^6.1.1",
     "keyv": "^5.0.0",
     "lodash-es": "^4.17.21",
+    "marked": "^15.0.0",
     "mixpanel": "^0.18.0",
     "mustache": "^4.2.0",
     "nanoid": "^5.0.7",
@@ -88,6 +93,8 @@
     "ses": "^1.4.1",
     "socket.io": "^4.7.5",
     "stripe": "^17.0.0",
+    "supertest": "^7.0.0",
+    "tap-parser": "^18.0.0",
     "ts-node": "^10.9.2",
     "typescript": "^5.6.3",
     "yjs": "patch:yjs@npm%3A13.6.18#~/.yarn/patches/yjs-npm-13.6.18-ad0d5f7c43.patch",
@@ -109,11 +116,9 @@
     "@types/on-headers": "^1.0.3",
     "@types/sinon": "^17.0.3",
     "@types/supertest": "^6.0.2",
-    "ava": "^6.1.2",
     "c8": "^10.0.0",
     "nodemon": "^3.1.0",
-    "sinon": "^19.0.0",
-    "supertest": "^7.0.0"
+    "sinon": "^19.0.0"
   },
   "ava": {
     "timeout": "1m",

packages/backend/server/scripts/copilot-cron-test.js

@@ -0,0 +1,99 @@
+// start process
+
+import { spawn } from 'node:child_process';
+
+import { WebClient } from '@slack/web-api';
+import { jsxslack } from 'jsx-slack';
+import { marked, Renderer } from 'marked';
+import { Parser } from 'tap-parser';
+
+async function runTest() {
+  const tester = new Promise(resolve => {
+    const test = spawn(
+      'npx',
+      [
+        'ava',
+        '--config',
+        'tests/ava.docker.config.js',
+        'tests/**/copilot-*.e2e.ts',
+        '--tap',
+      ],
+      { env: { ...process.env, NODE_NO_WARNINGS: 1 } }
+    );
+
+    const parser = new Parser();
+    test.stdout.on('data', data => {
+      console.log(data.toString());
+      parser.write(data);
+    });
+
+    test.on('close', _ => {
+      const failures = parser?.failures.filter(f => !!f.fullname);
+      const timeouts = parser?.failures.filter(f => !f.fullname);
+      const result = [
+        `${parser.results.pass} passed`,
+        `${parser.results.fail - timeouts.length} failed`,
+        `${timeouts.length} timeouts`,
+        `${parser.results.skip} skipped`,
+      ];
+      const report = [
+        `Test finished with ${result.join(', ')}.`,
+        failures?.length > 0
+          ? `Failed tests: \n\n${failures.map(failure => `- ${failure.fullname}`).join('\n')}`
+          : '',
+      ];
+      resolve(report.join('\n\n'));
+    });
+  });
+
+  try {
+    return await tester;
+  } catch (e) {
+    return e.message;
+  }
+}
+
+function render(markdown) {
+  const rendered = marked(markdown, {
+    renderer: new (class CustomRenderer extends Renderer {
+      heading({ tokens }) {
+        return `
+            <Fragment>
+              <Section><b>${tokens[0].text}</b></Section>
+              <Divider />
+            </Fragment>`;
+      }
+
+      paragraph({ tokens }) {
+        return `<Section><p>${tokens[0].text}</p></Section>`;
+      }
+
+      list(token) {
+        return `<Section>${super.list(token)}</Section>`;
+      }
+
+      hr() {
+        return `<Divider />`;
+      }
+    })(),
+  });
+  return jsxslack([`<Blocks>${rendered}</Blocks>`]);
+}
+
+async function main() {
+  const { CHANNEL_ID, SLACK_BOT_TOKEN, AFFINE_ENV } = process.env;
+
+  const report = await runTest();
+  const blocks = render(
+    [`# AFFiNE Copilot Test ${AFFINE_ENV} Env Test Result`, report].join('\n\n')
+  );
+  const { ok } = await new WebClient(SLACK_BOT_TOKEN).chat.postMessage({
+    channel: CHANNEL_ID,
+    text: `AFFiNE Copilot Test ${AFFINE_ENV} Env Test Result`,
+    blocks,
+  });
+
+  console.assert(ok, 'Failed to send a message to Slack');
+}
+
+await main();

packages/backend/server/tests/ava.docker.config.js

@@ -0,0 +1,9 @@
+import packageJson from '../package.json' with { type: 'json' };
+
+export default {
+  ...packageJson.ava,
+  environmentVariables: {
+    ...packageJson.ava.environmentVariables,
+    TS_NODE_PROJECT: './tests/tsconfig.docker.json',
+  },
+};