Rearchitect exporter to use BullMQ as a background job queueing system (

#5)
DA0-DA0 · Oct 2, 2023 · eba771d · eba771d
1 parent c1d0cd0
commit eba771d
Show file tree

Hide file tree

Showing 33 changed files with 2,690 additions and 1,094 deletions.
diff --git a/compose.dev.yml b/compose.dev.yml
@@ -12,12 +12,14 @@ services:
       NODE_ENV: development
       # contains db connection info that matches db service setup below
       CONFIG_FILE: config-dev.json
-    # waits for db to start
+    # waits for db and redis to start
     depends_on:
       db_accounts:
         condition: service_healthy
       db_data:
         condition: service_healthy
+      redis:
+        condition: service_healthy
     # colorizes output
     tty: true
 
@@ -47,5 +49,13 @@ services:
       timeout: 3s
       retries: 5
 
+  redis:
+    image: redis:7-alpine
+    healthcheck:
+      test: ["CMD-SHELL", "redis-cli ping"]
+      interval: 1s
+      timeout: 3s
+      retries: 5
+
 volumes:
   node_modules:
diff --git a/compose.test.yml b/compose.test.yml
@@ -12,10 +12,12 @@ services:
       NODE_ENV: test
       # contains db connection info that matches db service setup below
       CONFIG_FILE: config-test.json
-    # waits for db to start
+    # waits for db and redis to start
     depends_on:
       db:
         condition: service_healthy
+      redis:
+        condition: service_healthy
     # colorizes output
     tty: true
 
@@ -32,5 +34,13 @@ services:
       timeout: 3s
       retries: 5
 
+  redis:
+    image: redis:7-alpine
+    healthcheck:
+      test: ["CMD-SHELL", "redis-cli ping"]
+      interval: 1s
+      timeout: 3s
+      retries: 5
+
 volumes:
   node_modules:
diff --git a/config-dev.json b/config-dev.json
@@ -2,6 +2,9 @@
   "home": "./fake",
   "rpc": "https://uni-rpc.reece.sh",
   "bech32Prefix": "juno",
+  "redis": {
+    "host": "redis"
+  },
   "db": {
     "data": {
       "dialect": "postgres",

diff --git a/config-test.json b/config-test.json
@@ -1,4 +1,7 @@
 {
+  "redis": {
+    "host": "redis"
+  },
   "db": {
     "data": {
       "dialect": "postgres",

diff --git a/config.json.example b/config.json.example
@@ -2,6 +2,10 @@
   "home": "/Users/noah/.juno/indexer",
   "rpc": "https://uni-rpc.reece.sh",
   "bech32Prefix": "juno",
+  "redis": {
+    "host": "127.0.0.1",
+    "password": ""
+  },
   "db": {
     "data": {
       "dialect": "postgres",

diff --git a/ecosystem.config.js b/ecosystem.config.js
@@ -1,15 +1,17 @@
 module.exports = {
   apps: [
     {
-      name: 'exporter',
-      script: 'dist/scripts/export/index.js',
+      name: 'export-tracer',
+      script: 'dist/scripts/export/trace.js',
       wait_ready: true,
       listen_timeout: 30000,
       kill_timeout: 30000,
     },
     {
-      name: 'webhooks',
-      script: 'dist/scripts/webhooks.js',
+      name: 'export-processor',
+      script: 'dist/scripts/export/process.js',
+      wait_ready: true,
+      listen_timeout: 30000,
       kill_timeout: 30000,
     },
     {