Replace BullMQ with Graphile Worker + merge Webhooks and Emails into Transmission service

deployment/index.ts

@@ -8,7 +8,6 @@ import { deployCloudFlareSecurityTransform } from './services/cloudflare-securit
 import { deployDatabaseCleanupJob } from './services/database-cleanup';
 import { deployDbMigrations } from './services/db-migrations';
 import { configureDocker } from './services/docker';
-import { deployEmails } from './services/emails';
 import { prepareEnvironment } from './services/environment';
 import { configureGithubApp } from './services/github';
 import { deployGraphQL } from './services/graphql';
@@ -26,10 +25,10 @@ import { deploySentryEventsMonitor } from './services/sentry-events';
 import { configureSlackApp } from './services/slack-app';
 import { deploySuperTokens } from './services/supertokens';
 import { deployTokens } from './services/tokens';
+import { deployTransmission } from './services/transmission';
 import { deployUsage } from './services/usage';
 import { deployUsageEstimation } from './services/usage-estimation';
 import { deployUsageIngestor } from './services/usage-ingestor';
-import { deployWebhooks } from './services/webhooks';
 import { configureZendesk } from './services/zendesk';
 import { optimizeAzureCluster } from './utils/azure-helpers';
 import { isDefined } from './utils/helpers';
@@ -127,22 +126,12 @@ const tokens = deployTokens({
   observability,
 });
 
-const webhooks = deployWebhooks({
-  image: docker.factory.getImageId('webhooks', imagesTag),
+const transmission = deployTransmission({
+  image: docker.factory.getImageId('transmission', imagesTag),
   environment,
-  heartbeat: heartbeatsConfig.get('webhooks'),
+  heartbeat: heartbeatsConfig.get('transmission'),
   broker,
   docker,
-  redis,
-  sentry,
-  observability,
-});
-
-const emails = deployEmails({
-  image: docker.factory.getImageId('emails', imagesTag),
-  docker,
-  environment,
-  redis,
   sentry,
   observability,
 });
@@ -174,7 +163,7 @@ const rateLimit = deployRateLimit({
   environment,
   dbMigrations,
   usageEstimator,
-  emails,
+  transmission,
   postgres,
   sentry,
   observability,
@@ -232,7 +221,7 @@ const graphql = deployGraphQL({
   image: docker.factory.getImageId('server', imagesTag),
   docker,
   tokens,
-  webhooks,
+  transmission,
   schema,
   schemaPolicy,
   dbMigrations,
@@ -242,7 +231,6 @@ const graphql = deployGraphQL({
   usageEstimator,
   rateLimit,
   billing,
-  emails,
   supertokens,
   s3,
   s3Mirror,
@@ -330,7 +318,6 @@ export const usageApiServiceId = usage.service.id;
 export const usageIngestorApiServiceId = usageIngestor.service.id;
 export const tokensApiServiceId = tokens.service.id;
 export const schemaApiServiceId = schema.service.id;
-export const webhooksApiServiceId = webhooks.service.id;
 
 export const appId = app.deployment.id;
 export const publicIp = proxy!.status.loadBalancer.ingress[0].ip;

deployment/services/graphql.ts

@@ -7,7 +7,6 @@ import { CDN } from './cf-cdn';
 import { Clickhouse } from './clickhouse';
 import { DbMigrations } from './db-migrations';
 import { Docker } from './docker';
-import { Emails } from './emails';
 import { Environment } from './environment';
 import { GitHubApp } from './github';
 import { Observability } from './observability';
@@ -20,9 +19,9 @@ import { Schema } from './schema';
 import { Sentry } from './sentry';
 import { Supertokens } from './supertokens';
 import { Tokens } from './tokens';
+import { Transmission } from './transmission';
 import { Usage } from './usage';
 import { UsageEstimator } from './usage-estimation';
-import { Webhooks } from './webhooks';
 import { Zendesk } from './zendesk';
 
 export type GraphQL = ReturnType<typeof deployGraphQL>;
@@ -37,7 +36,6 @@ export function deployGraphQL({
   image,
   environment,
   tokens,
-  webhooks,
   schema,
   schemaPolicy,
   cdn,
@@ -47,7 +45,7 @@ export function deployGraphQL({
   dbMigrations,
   rateLimit,
   billing,
-  emails,
+  transmission,
   supertokens,
   s3,
   s3Mirror,
@@ -65,7 +63,6 @@ export function deployGraphQL({
   clickhouse: Clickhouse;
   environment: Environment;
   tokens: Tokens;
-  webhooks: Webhooks;
   schema: Schema;
   schemaPolicy: SchemaPolicy;
   redis: Redis;
@@ -77,7 +74,7 @@ export function deployGraphQL({
   dbMigrations: DbMigrations;
   rateLimit: RateLimitService;
   billing: StripeBillingService;
-  emails: Emails;
+  transmission: Transmission;
   supertokens: Supertokens;
   zendesk: Zendesk;
   docker: Docker;
@@ -125,12 +122,11 @@ export function deployGraphQL({
           REQUEST_LOGGING: '0', // disabled
           BILLING_ENDPOINT: serviceLocalEndpoint(billing.service),
           TOKENS_ENDPOINT: serviceLocalEndpoint(tokens.service),
-          WEBHOOKS_ENDPOINT: serviceLocalEndpoint(webhooks.service),
           SCHEMA_ENDPOINT: serviceLocalEndpoint(schema.service),
           SCHEMA_POLICY_ENDPOINT: serviceLocalEndpoint(schemaPolicy.service),
           HIVE_USAGE_ENDPOINT: serviceLocalEndpoint(usage.service),
           RATE_LIMIT_ENDPOINT: serviceLocalEndpoint(rateLimit.service),
-          EMAILS_ENDPOINT: serviceLocalEndpoint(emails.service),
+          TRANSMISSION_ENDPOINT: serviceLocalEndpoint(transmission.service),
           USAGE_ESTIMATOR_ENDPOINT: serviceLocalEndpoint(usageEstimator.service),
           WEB_APP_URL: `https://${environment.appDns}`,
           GRAPHQL_PUBLIC_ORIGIN: `https://${environment.appDns}`,

deployment/services/rate-limit.ts

@@ -2,11 +2,11 @@ import { serviceLocalEndpoint } from '../utils/local-endpoint';
 import { ServiceDeployment } from '../utils/service-deployment';
 import { DbMigrations } from './db-migrations';
 import { Docker } from './docker';
-import { Emails } from './emails';
 import { Environment } from './environment';
 import { Observability } from './observability';
 import { Postgres } from './postgres';
 import { Sentry } from './sentry';
+import { Transmission } from './transmission';
 import { UsageEstimator } from './usage-estimation';
 
 export type RateLimitService = ReturnType<typeof deployRateLimit>;
@@ -15,7 +15,7 @@ export function deployRateLimit({
   environment,
   dbMigrations,
   usageEstimator,
-  emails,
+  transmission,
   image,
   docker,
   postgres,
@@ -26,7 +26,7 @@ export function deployRateLimit({
   usageEstimator: UsageEstimator;
   environment: Environment;
   dbMigrations: DbMigrations;
-  emails: Emails;
+  transmission: Transmission;
   image: string;
   docker: Docker;
   postgres: Postgres;
@@ -45,7 +45,7 @@ export function deployRateLimit({
         SENTRY: sentry.enabled ? '1' : '0',
         LIMIT_CACHE_UPDATE_INTERVAL_MS: environment.isProduction ? '60000' : '86400000',
         USAGE_ESTIMATOR_ENDPOINT: serviceLocalEndpoint(usageEstimator.service),
-        EMAILS_ENDPOINT: serviceLocalEndpoint(emails.service),
+        TRANSMISSION_ENDPOINT: serviceLocalEndpoint(transmission.service),
         WEB_APP_URL: `https://${environment.appDns}/`,
         OPENTELEMETRY_COLLECTOR_ENDPOINT:
           observability.enabled && observability.tracingEndpoint

deployment/services/emails.ts → deployment/services/transmission.ts

@@ -2,33 +2,33 @@ import * as pulumi from '@pulumi/pulumi';
 import { serviceLocalEndpoint } from '../utils/local-endpoint';
 import { ServiceSecret } from '../utils/secrets';
 import { ServiceDeployment } from '../utils/service-deployment';
+import type { Broker } from './cf-broker';
 import { Docker } from './docker';
 import { Environment } from './environment';
 import { Observability } from './observability';
-import { Redis } from './redis';
 import { Sentry } from './sentry';
 
-export type Emails = ReturnType<typeof deployEmails>;
+export type Transmission = ReturnType<typeof deployTransmission>;
 
 class PostmarkSecret extends ServiceSecret<{
   token: pulumi.Output<string> | string;
   from: string;
   messageStream: string;
 }> {}
 
-export function deployEmails({
+export function deployTransmission({
   environment,
-  redis,
   heartbeat,
   image,
   docker,
   sentry,
   observability,
+  broker,
 }: {
   observability: Observability;
   environment: Environment;
   image: string;
-  redis: Redis;
+  broker: Broker;
   docker: Docker;
   heartbeat?: string;
   sentry: Sentry;
@@ -41,12 +41,13 @@ export function deployEmails({
   });
 
   const { deployment, service } = new ServiceDeployment(
-    'emails-service',
+    'transmission-service',
     {
       imagePullSecret: docker.secret,
       env: {
         ...environment.envVars,
         SENTRY: sentry.enabled ? '1' : '0',
+        REQUEST_BROKER: '1',
         EMAIL_PROVIDER: 'postmark',
         HEARTBEAT_ENDPOINT: heartbeat ?? '',
         OPENTELEMETRY_COLLECTOR_ENDPOINT:
@@ -61,14 +62,13 @@ export function deployEmails({
       image,
       replicas: environment.isProduction ? 3 : 1,
     },
-    [redis.deployment, redis.service],
+    [],
   )
-    .withSecret('REDIS_HOST', redis.secret, 'host')
-    .withSecret('REDIS_PORT', redis.secret, 'port')
-    .withSecret('REDIS_PASSWORD', redis.secret, 'password')
     .withSecret('EMAIL_FROM', postmarkSecret, 'from')
     .withSecret('EMAIL_PROVIDER_POSTMARK_TOKEN', postmarkSecret, 'token')
     .withSecret('EMAIL_PROVIDER_POSTMARK_MESSAGE_STREAM', postmarkSecret, 'messageStream')
+    .withSecret('REQUEST_BROKER_ENDPOINT', broker.secret, 'baseUrl')
+    .withSecret('REQUEST_BROKER_SIGNATURE', broker.secret, 'secretSignature')
     .withConditionalSecret(sentry.enabled, 'SENTRY_DSN', sentry.secret, 'dsn')
     .deploy();
 

docker/docker-compose.community.yml

@@ -189,9 +189,7 @@ services:
         condition: service_completed_successfully
       tokens:
         condition: service_healthy
-      webhooks:
-        condition: service_healthy
-      emails:
+      transmission:
         condition: service_healthy
       schema:
         condition: service_healthy
@@ -215,10 +213,9 @@ services:
       REDIS_PORT: 6379
       REDIS_PASSWORD: '${REDIS_PASSWORD}'
       TOKENS_ENDPOINT: http://tokens:3003
-      WEBHOOKS_ENDPOINT: http://webhooks:3005
       SCHEMA_ENDPOINT: http://schema:3002
       SCHEMA_POLICY_ENDPOINT: http://policy:3012
-      EMAILS_ENDPOINT: http://emails:3011
+      TRANSMISSION_ENDPOINT: http://transmission:3005
       ENCRYPTION_SECRET: '${HIVE_ENCRYPTION_SECRET}'
       WEB_APP_URL: '${HIVE_APP_BASE_URL}'
       PORT: 3001
@@ -299,41 +296,28 @@ services:
       SENTRY_DSN: '${SENTRY_DSN:-}'
       PROMETHEUS_METRICS: '${PROMETHEUS_METRICS:-}'
 
-  webhooks:
-    image: '${DOCKER_REGISTRY}webhooks${DOCKER_TAG}'
+  transmission:
+    image: '${DOCKER_REGISTRY}transmission${DOCKER_TAG}'
     networks:
       - 'stack'
     depends_on:
-      redis:
+      db:
+        condition: service_healthy
+      clickhouse:
         condition: service_healthy
+    ports:
+      - 3005:3005
     environment:
       NODE_ENV: production
       PORT: 3005
-      REDIS_HOST: redis
-      REDIS_PORT: 6379
-      REDIS_PASSWORD: '${REDIS_PASSWORD}'
       LOG_LEVEL: '${LOG_LEVEL:-debug}'
-      OPENTELEMETRY_COLLECTOR_ENDPOINT: '${OPENTELEMETRY_COLLECTOR_ENDPOINT:-}'
-      SENTRY: '${SENTRY:-0}'
-      SENTRY_DSN: '${SENTRY_DSN:-}'
-      PROMETHEUS_METRICS: '${PROMETHEUS_METRICS:-}'
-
-  emails:
-    image: '${DOCKER_REGISTRY}emails${DOCKER_TAG}'
-    networks:
-      - 'stack'
-    depends_on:
-      redis:
-        condition: service_healthy
-    environment:
-      NODE_ENV: production
-      PORT: 3011
-      REDIS_HOST: redis
-      REDIS_PORT: 6379
-      REDIS_PASSWORD: '${REDIS_PASSWORD}'
+      POSTGRES_HOST: db
+      POSTGRES_USER: '${POSTGRES_USER}'
+      POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
+      POSTGRES_PORT: 5432
+      POSTGRES_DB: '${POSTGRES_DB}'
       EMAIL_FROM: [email protected]
       EMAIL_PROVIDER: sendmail
-      LOG_LEVEL: '${LOG_LEVEL:-debug}'
       OPENTELEMETRY_COLLECTOR_ENDPOINT: '${OPENTELEMETRY_COLLECTOR_ENDPOINT:-}'
       SENTRY: '${SENTRY:-0}'
       SENTRY_DSN: '${SENTRY_DSN:-}'