fix/notifications not working (#16)

* fix * fix build * fix build * fix build * enable real sinks instead of mocks
dialectlabs · Apr 22, 2024 · 5319a80 · 5319a80
1 parent 36cf2f4
commit 5319a80
Show file tree

Hide file tree

Showing 23 changed files with 1,200 additions and 640 deletions.
diff --git a/.env.production b/.env.production
@@ -0,0 +1,2 @@
+ENVIRONMENT=production
+DIALECT_SDK_ENVIRONMENT=production
diff --git a/.github/create-pr.sh b/.github/create-pr.sh
@@ -0,0 +1,55 @@
+#!/bin/sh
+
+set -e
+set -x
+
+if [ -z "$INPUT_SOURCE_FOLDER" ]
+then
+  echo "Source folder must be defined"
+  return -1
+fi
+
+if [ $INPUT_DESTINATION_HEAD_BRANCH == "main" ] || [ $INPUT_DESTINATION_HEAD_BRANCH == "master"]
+then
+  echo "Destination head branch cannot be 'main' nor 'master'"
+  return -1
+fi
+
+if [ -z "$INPUT_PULL_REQUEST_REVIEWERS" ]
+then
+  PULL_REQUEST_REVIEWERS=$INPUT_PULL_REQUEST_REVIEWERS
+else
+  PULL_REQUEST_REVIEWERS='-r '$INPUT_PULL_REQUEST_REVIEWERS
+fi
+
+CLONE_DIR=$(mktemp -d)
+
+echo "Setting git variables"
+export GITHUB_TOKEN=$API_TOKEN_GITHUB
+git config --global user.email "$INPUT_USER_EMAIL"
+git config --global user.name "$INPUT_USER_NAME"
+
+echo "Cloning destination git repository"
+git clone "https://$API_TOKEN_GITHUB@github.com/$INPUT_DESTINATION_REPO.git" "$CLONE_DIR"
+
+echo "Copying contents to git repo"-r $INPUT_USER_NAME
+cp -R $INPUT_SOURCE_FOLDER "$CLONE_DIR/$INPUT_DESTINATION_FOLDER"
+cd "$CLONE_DIR"
+git checkout -b "$INPUT_DESTINATION_HEAD_BRANCH"
+
+echo "Adding git commit"
+git add .
+if git status | grep -q "Changes to be committed"
+then
+  git commit --message "Update from https://github.com/$GITHUB_REPOSITORY/commit/$GITHUB_SHA"
+  echo "Pushing git commit"
+  git push -u origin HEAD:$INPUT_DESTINATION_HEAD_BRANCH
+  echo "Creating a pull request"
+  gh pr create -t $INPUT_DESTINATION_HEAD_BRANCH \
+               -b $INPUT_DESTINATION_HEAD_BRANCH \
+               -B $INPUT_DESTINATION_BASE_BRANCH \
+               -H $INPUT_DESTINATION_HEAD_BRANCH \
+                  $PULL_REQUEST_REVIEWERS
+else
+  echo "No changes detected"
+fi
diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml
@@ -21,7 +21,7 @@ env:
 
 jobs:
   deployment:
-    name: Deployment
+    name: Create deployment PR
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -50,14 +50,13 @@ jobs:
           cat ${{ env.DEPLOYMENT_SCRIPTS_REPO }}/${{ env.SERVICE_RESOURCES_FILE }} |\
           ./hcledit attribute set locals.svc_${{ steps.service-name.outputs.service_name }}_image_version '"${{ inputs.service_image_tag }}"' \
           > ${{ env.SERVICE_RESOURCES_FILE }}
-      - name: Create pull request
-        uses: paygoc6/[email protected]
+      - name: Create deployment pull request
         env:
           API_TOKEN_GITHUB: ${{ secrets.api_github_token }}
-        with:
-          source_folder: "terraform-plans"
-          destination_repo: ${{ env.DEPLOYMENT_SCRIPTS_REPO }}
-          destination_base_branch: 'main'
-          destination_head_branch: cd/${{ inputs.environment }}/${{ github.event.repository.name }}/${{ inputs.service_image_tag }}
-          user_email: ${{ github.event.pusher.email }}
-          user_name: ${{ github.actor }}
+          INPUT_SOURCE_FOLDER: "terraform-plans"
+          INPUT_DESTINATION_REPO: ${{ env.DEPLOYMENT_SCRIPTS_REPO }}
+          INPUT_DESTINATION_BASE_BRANCH: "main"
+          INPUT_DESTINATION_HEAD_BRANCH: cd/${{ inputs.environment }}/${{ github.event.repository.name }}/${{ inputs.service_image_tag }}
+          INPUT_USER_EMAIL: ${{ github.event.pusher.email }}
+          INPUT_USER_NAME: ${{ github.actor }}
+        run: "${GITHUB_WORKSPACE}/.github/create-pr.sh"
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -6,9 +6,9 @@ on:
       - '**'
     tags:
       - '*'
-
 env:
   AWS_REGION: us-west-2
+  NPM_GITHUB_TOKEN: ${{ secrets.API_TOKEN_GITHUB_PRIVATE_PACKAGES_DOWNLOAD }}
 
 jobs:
   integration:
@@ -17,11 +17,16 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+      - name: Use Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '18'
       - name: Install deps
         run: yarn
+      - name: Lint
+        run: yarn lint
       - name: Build
         run: yarn build
-
   docker-image:
     if: ${{ !contains(github.ref, 'heads/dependabot') }}
     needs:
@@ -56,14 +61,14 @@ jobs:
           ECR_REPOSITORY: ${{ github.event.repository.name }}
           IMAGE_TAG: ${{ steps.get-image-tag.outputs.image_tag }}
         run: |
-          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -t $ECR_REGISTRY/$ECR_REPOSITORY:latest .
+          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -t $ECR_REGISTRY/$ECR_REPOSITORY:latest --build-arg NPM_GITHUB_TOKEN=${{ secrets.API_TOKEN_GITHUB_PRIVATE_PACKAGES_DOWNLOAD }} .
           docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
           docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest
     outputs:
       image_tag: ${{ steps.get-image-tag.outputs.image_tag }}
 
   cd_production:
-    if: ${{ contains(github.ref, 'heads/master') }}
+    if: ${{ contains(github.ref, 'heads/main') || contains(github.ref, 'fix')}}
     needs:
       - docker-image
     uses: ./.github/workflows/cd.yaml

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:16-alpine
+FROM node:20-alpine3.18
 
 WORKDIR /app
 

diff --git a/package.json b/package.json
@@ -13,6 +13,7 @@
     "start:dev": "dotenv -e .env.dev -e .env.service -- nest start --watch",
     "start:dev:debug": "dotenv -e .env.dev -e .env.service -- nest start --debug --watch",
     "start:local-dev": "dotenv -e .env.local-dev -e .env.service -- nest start --watch",
+    "start:production": "dotenv -e .env.production -e .env.service -- nest start --watch",
     "start:local-dev:debug": "dotenv -e .env.local-dev -e .env.service -- nest start --debug --watch",
     "client:start:dev": "dotenv -e .env.dev -e .env.client -- ts-node test/client.ts",
     "client:start:local-dev": "dotenv -e .env.local-dev -e .env.client -- ts-node test/client.ts"
@@ -24,14 +25,16 @@
     "@nestjs/common": "^9.0.0",
     "@nestjs/config": "^2.2.0",
     "@nestjs/core": "^9.0.0",
+    "@nestjs/event-emitter": "^1.3.1",
     "@nestjs/platform-express": "^9.0.0",
     "@nestjs/schedule": "^2.1.0",
     "@nestjs/terminus": "^9.1.0",
-    "@nestjs/event-emitter": "^1.3.1",
-    "@solana/spl-governance": "^0.0.34",
-    "@solana/spl-token": "0.1.8",
+    "@solana/spl-governance": "^0.3.28",
+    "@solana/spl-token": "^0.4.3",
     "@solana/spl-token-registry": "^0.2.3775",
-    "bn.js": "^5.2.1",
+    "bn.js": "^5.1.3",
+    "borsh": "^0.3.1",
+    "bs58": "^4.0.1",
     "lodash": "^4.17.21",
     "luxon": "^3.0.1",
     "nestjs-pino": "^2.6.0",
@@ -47,6 +50,7 @@
     "@nestjs/schematics": "^9.0.0",
     "@nestjs/testing": "^9.0.0",
     "@types/bn.js": "^5.1.0",
+    "@types/bs58": "^4.0.4",
     "@types/cron": "^2.0.0",
     "@types/express": "^4.17.13",
     "@types/jest": "28.1.4",
@@ -56,6 +60,7 @@
     "@types/supertest": "^2.0.11",
     "@typescript-eslint/eslint-plugin": "^5.0.0",
     "@typescript-eslint/parser": "^5.0.0",
+    "dotenv-cli": "^7.4.1",
     "eslint": "^8.0.1",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-prettier": "^4.0.0",

diff --git a/scripts/twitter.ts b/scripts/twitter.ts
diff --git a/src/app.module.ts b/src/app.module.ts
@@ -11,7 +11,7 @@ import { DialectSdk } from './dialect-sdk';
 import { ConfigModule } from '@nestjs/config';
 import { RealmsRestService } from './realms-rest-service';
 import { RealmsService } from './realms.service';
-import { RealmsRepository } from './realms-repository';
+import { RealmsCache } from './realms-cache';
 import { ScheduleModule } from '@nestjs/schedule';
 import { NewProposalsMonitoringService } from './new-proposals-monitoring.service';
 import { ProposalStateChangeMonitoringService } from './proposal-state-monitoring.service';
@@ -45,9 +45,9 @@ import { EventEmitterModule } from '@nestjs/event-emitter';
   ],
   controllers: [HealthController],
   providers: [
+    RealmsCache,
     CachingHealth,
     RealmsRestService,
-    RealmsRepository,
     RealmsService,
     NewProposalsMonitoringService,
     ProposalStateChangeMonitoringService,

diff --git a/src/caching.health.ts b/src/caching.health.ts
@@ -4,61 +4,80 @@ import {
   HealthIndicatorResult,
 } from '@nestjs/terminus';
 import { Injectable, Logger } from '@nestjs/common';
-import { OnEvent } from '@nestjs/event-emitter';
-
-export interface CachingEvent {
-  type: CachingEventType;
-}
-
-export interface CachingStartedEvent extends CachingEvent {
-  type: CachingEventType.Started;
-  timeStarted: number;
-}
-
-export interface CachingFinishedEvent extends CachingEvent {
-  type: CachingEventType.Finished;
-}
-
-export enum CachingEventType {
-  Started = 'caching.started',
-  Finished = 'caching.finished',
-}
+import { RealmsCache } from './realms-cache';
 
 @Injectable()
 export class CachingHealth extends HealthIndicator {
-  private static readonly MAX_CACHING_EXECUTION_TIME_MILLIS = process.env
-    .MAX_CACHING_EXECUTION_TIME_MILLIS
-    ? parseInt(process.env.MAX_CACHING_EXECUTION_TIME_MILLIS, 10)
-    : 600000;
+  private static MAX_STATIC_ACCOUNT_CACHE_AGE_MILLIS =
+    process.env.MAX_CACHE_AGE_MILLIS ?? 3 * 60 * 60 * 1000;
+
+  private static MAX_DYNAMIC_ACCOUNT_CACHE_AGE_MILLIS =
+    process.env.MAX_CACHE_AGE_MILLIS ?? 30 * 60 * 1000;
+
   private readonly logger = new Logger(CachingHealth.name);
-  private lastStartedCaching: number;
-  private cachingInProgress = false;
+
+  constructor(private readonly realmsCache: RealmsCache) {
+    super();
+  }
 
   public isHealthy(): HealthIndicatorResult {
-    const isHealthy = this.cachingInProgress
-      ? Date.now() - this.lastStartedCaching <
-        CachingHealth.MAX_CACHING_EXECUTION_TIME_MILLIS
-      : true;
-    if (isHealthy) {
-      return this.getStatus('caching', isHealthy);
+    if (this.realmsCache.initializationError) {
+      this.logger.error(
+        'Caching health check failed. Service needs to be restarted, because initialization failed',
+      );
+      throw new HealthCheckError(
+        'Caching failed',
+        this.getStatus('caching', false),
+      );
     }
-    this.logger.error(
-      'Caching health check failed. Service needs to be restarted',
-    );
-    throw new HealthCheckError(
-      'Caching failed',
-      this.getStatus('caching', isHealthy),
-    );
-  }
 
-  @OnEvent(CachingEventType.Started)
-  onCachingStarted({ timeStarted }: CachingStartedEvent) {
-    this.lastStartedCaching = timeStarted;
-    this.cachingInProgress = true;
-  }
+    if (!this.realmsCache.isInitialized) {
+      return this.getStatus('caching', true);
+    }
+
+    if (
+      !this.realmsCache.lastDynamicAccountCachingSuccessFinishedAt ||
+      !this.realmsCache.lastStaticAccountCachingSuccessFinishedAt
+    ) {
+      this.logger.error(
+        `Some of the cache ages are not initialized, this should not happen. 
+  Static cache age: ${this.realmsCache.lastStaticAccountCachingSuccessFinishedAt}, dynamic cache age: ${this.realmsCache.lastDynamicAccountCachingSuccessFinishedAt}
+  Service needs to be restarted`,
+      );
+      throw new HealthCheckError(
+        'Caching failed',
+        this.getStatus('caching', false),
+      );
+    }
+
+    const staticCacheAge =
+      Date.now() -
+      this.realmsCache.lastStaticAccountCachingSuccessFinishedAt.getTime();
+
+    if (staticCacheAge > CachingHealth.MAX_STATIC_ACCOUNT_CACHE_AGE_MILLIS) {
+      this.logger.error(
+        `Static cache age is too old: ${staticCacheAge}, service needs to be restarted`,
+      );
+      throw new HealthCheckError(
+        'Caching failed',
+        this.getStatus('caching', false),
+      );
+    }
+
+    const dynamicCacheAge =
+      Date.now() -
+      this.realmsCache.lastDynamicAccountCachingSuccessFinishedAt.getTime();
+
+    if (dynamicCacheAge > CachingHealth.MAX_DYNAMIC_ACCOUNT_CACHE_AGE_MILLIS) {
+      this.logger.error(
+        `Dynamic cache age is too old: ${dynamicCacheAge}, service needs to be restarted`,
+      );
+      throw new HealthCheckError(
+        'Caching failed',
+        this.getStatus('caching', false),
+      );
+    }
 
-  @OnEvent(CachingEventType.Finished)
-  onCachingFinished() {
-    this.cachingInProgress = false;
+    return this.getStatus('caching', true);
   }
 }
diff --git a/src/dialect-sdk.ts b/src/dialect-sdk.ts
@@ -8,9 +8,9 @@ import {
 } from '@dialectlabs/sdk';
 
 export abstract class DialectSdk implements IDialectSdk {
-  readonly dapps: Dapps;
-  readonly info: DialectSdkInfo;
-  readonly threads: Messaging;
-  readonly wallet: Wallets;
-  readonly identity: IdentityResolver;
+  readonly dapps!: Dapps;
+  readonly info!: DialectSdkInfo;
+  readonly threads!: Messaging;
+  readonly wallet!: Wallets;
+  readonly identity!: IdentityResolver;
 }
diff --git a/src/health.controller.ts b/src/health.controller.ts
@@ -10,12 +10,12 @@ import { CachingHealth } from './caching.health';
 export class HealthController {
   constructor(
     private health: HealthCheckService,
-    private readonly dataIngestionHealth: CachingHealth,
+    private readonly cachingHealth: CachingHealth,
   ) {}
 
   @Get()
   @HealthCheck()
   check() {
-    return this.health.check([() => this.dataIngestionHealth.isHealthy()]);
+    return this.health.check([() => this.cachingHealth.isHealthy()]);
   }
 }
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		ENVIRONMENT=production
		DIALECT_SDK_ENVIRONMENT=production