Rearchitect exporter to use BullMQ as a background job queueing system

compose.dev.yml

@@ -12,12 +12,14 @@ services:
       NODE_ENV: development
       # contains db connection info that matches db service setup below
       CONFIG_FILE: config-dev.json
-    # waits for db to start
+    # waits for db and redis to start
     depends_on:
       db_accounts:
         condition: service_healthy
       db_data:
         condition: service_healthy
+      redis:
+        condition: service_healthy
     # colorizes output
     tty: true
 
@@ -47,5 +49,13 @@ services:
       timeout: 3s
       retries: 5
 
+  redis:
+    image: redis:7-alpine
+    healthcheck:
+      test: ["CMD-SHELL", "redis-cli ping"]
+      interval: 1s
+      timeout: 3s
+      retries: 5
+
 volumes:
   node_modules:

compose.test.yml

@@ -12,10 +12,12 @@ services:
       NODE_ENV: test
       # contains db connection info that matches db service setup below
       CONFIG_FILE: config-test.json
-    # waits for db to start
+    # waits for db and redis to start
     depends_on:
       db:
         condition: service_healthy
+      redis:
+        condition: service_healthy
     # colorizes output
     tty: true
 
@@ -32,5 +34,13 @@ services:
       timeout: 3s
       retries: 5
 
+  redis:
+    image: redis:7-alpine
+    healthcheck:
+      test: ["CMD-SHELL", "redis-cli ping"]
+      interval: 1s
+      timeout: 3s
+      retries: 5
+
 volumes:
   node_modules:

config-dev.json

@@ -2,6 +2,9 @@
   "home": "./fake",
   "rpc": "https://uni-rpc.reece.sh",
   "bech32Prefix": "juno",
+  "redis": {
+    "host": "redis"
+  },
   "db": {
     "data": {
       "dialect": "postgres",

config-test.json

@@ -1,4 +1,7 @@
 {
+  "redis": {
+    "host": "redis"
+  },
   "db": {
     "data": {
       "dialect": "postgres",

config.json.example

@@ -2,6 +2,10 @@
   "home": "/Users/noah/.juno/indexer",
   "rpc": "https://uni-rpc.reece.sh",
   "bech32Prefix": "juno",
+  "redis": {
+    "host": "127.0.0.1",
+    "password": ""
+  },
   "db": {
     "data": {
       "dialect": "postgres",

ecosystem.config.js

@@ -1,15 +1,17 @@
 module.exports = {
   apps: [
     {
-      name: 'exporter',
-      script: 'dist/scripts/export/index.js',
+      name: 'export-tracer',
+      script: 'dist/scripts/export/trace.js',
       wait_ready: true,
       listen_timeout: 30000,
       kill_timeout: 30000,
     },
     {
-      name: 'webhooks',
-      script: 'dist/scripts/webhooks.js',
+      name: 'export-processor',
+      script: 'dist/scripts/export/process.js',
+      wait_ready: true,
+      listen_timeout: 30000,
       kill_timeout: 30000,
     },
     {