feat!: fx and interscheme implementation

.ncurc.yaml

@@ -9,5 +9,7 @@ reject: [
   "get-port",
   # sinon v17.0.1 causes 58 tests to fail. This will need to be resolved in a future story.
   # Issue is tracked here: https://github.com/mojaloop/project/issues/3616
-  "sinon"
+  "sinon",
+  # glob >= 11 requires node >= 20
+  "glob"
 ]

.nvmrc

@@ -1 +1 @@
-18.17.1
+18.20.3

.nycrc.yml

@@ -17,5 +17,20 @@ exclude: [
   "**/node_modules/**",
   '**/migrations/**',
   '**/ddl/**',
-  '**/bulk*/**'
+  '**/bulk*/**',
+  'src/shared/logger/**',
+  'src/shared/loggingPlugin.js',
+  'src/shared/constants.js',
+  'src/domain/position/index.js',
+  'src/domain/position/binProcessor.js',
+  'src/handlers/positions/handler.js',
+  'src/handlers/transfers/createRemittanceEntity.js',
+  'src/handlers/transfers/FxFulfilService.js',
+  'src/models/position/batch.js',
+  'src/models/fxTransfer/**',
+  'src/models/participant/externalParticipantCached.js', # todo: figure out why it shows only 50% coverage in Branch
+  'src/models/transfer/facade.js', ## add more test coverage
+  'src/shared/fspiopErrorFactory.js',
+  'src/lib/proxyCache.js' # todo: remove this line after adding test coverage
 ]
+## todo: increase test coverage before merging feat/fx-impl to main branch

Dockerfile

@@ -3,26 +3,27 @@
 
 # NOTE: Ensure you set NODE_VERSION Build Argument as follows...
 #
-#  export NODE_VERSION="$(cat .nvmrc)-alpine" \
-#  docker build \
-#    --build-arg NODE_VERSION=$NODE_VERSION \
-#    -t mojaloop/central-ledger:local \
-#    . \
+# export NODE_VERSION="$(cat .nvmrc)-alpine"
+# docker build \
+#   --build-arg NODE_VERSION=$NODE_VERSION \
+#   -t mojaloop/central-ledger:local \
+#   .
 #
 
 # Build Image
-FROM node:${NODE_VERSION} as builder
+FROM node:${NODE_VERSION} AS builder
 
 WORKDIR /opt/app
 
 RUN apk --no-cache add git
-RUN apk add --no-cache -t build-dependencies make gcc g++ python3 libtool openssl-dev autoconf automake bash \
+RUN apk add --no-cache -t build-dependencies make gcc g++ python3 py3-setuptools libtool openssl-dev autoconf automake bash \
     && cd $(npm root -g)/npm \
     && npm install -g node-gyp
 
 COPY package.json package-lock.json* /opt/app/
 
 RUN npm ci
+RUN npm prune --omit=dev
 
 FROM node:${NODE_VERSION}
 WORKDIR /opt/app
@@ -32,7 +33,7 @@
 RUN ln -sf /dev/stdout ./logs/combined.log
 
 # Create a non-root user: ml-user
-RUN adduser -D ml-user 
+RUN adduser -D ml-user
 USER ml-user
 
 COPY --chown=ml-user --from=builder /opt/app .
@@ -43,7 +44,5 @@
 COPY seeds /opt/app/seeds
 COPY test /opt/app/test
 
-RUN npm prune --production
-
 EXPOSE 3001
 CMD ["npm", "run", "start"]

README.md

@@ -56,7 +56,7 @@ Or via docker build directly:
 
 ```bash
 docker build \
-  --build-arg NODE_VERSION="$(cat .nvmrc)-alpine" \
+  --build-arg NODE_VERSION="$(cat .nvmrc)-alpine3.19" \
   -t mojaloop/ml-api-adapter:local \
   .
 ```
@@ -113,21 +113,27 @@ NOTE: Only POSITION.PREPARE and POSITION.COMMIT is supported at this time, with
 
 Batch processing can be enabled in the transfer execution flow. Follow the steps below to enable batch processing for a more efficient transfer execution:
 
+Note: The position messages with action 'FX_PREPARE', 'FX_COMMIT' and 'FX_TIMEOUT_RESERVED' are only supported in batch processing.
+
 - **Step 1:** **Create a New Kafka Topic**
 
   Create a new Kafka topic named `topic-transfer-position-batch` to handle batch processing events.
 - **Step 2:** **Configure Action Type Mapping**
 
-  Point the prepare handler to the newly created topic for the action type `prepare` using the `KAFKA.EVENT_TYPE_ACTION_TOPIC_MAP` configuration as shown below:
+  Point the prepare handler to the newly created topic for the action types those are supported in batch processing using the `KAFKA.EVENT_TYPE_ACTION_TOPIC_MAP` configuration as shown below:
   ```
     "KAFKA": {
       "EVENT_TYPE_ACTION_TOPIC_MAP" : {
         "POSITION":{
           "PREPARE": "topic-transfer-position-batch",
           "BULK_PREPARE": "topic-transfer-position",
           "COMMIT": "topic-transfer-position-batch",
+          "FX_COMMIT": "topic-transfer-position-batch",
           "BULK_COMMIT": "topic-transfer-position",
           "RESERVE": "topic-transfer-position",
+          "FX_PREPARE": "topic-transfer-position-batch",
+          "TIMEOUT_RESERVED": "topic-transfer-position-batch",
+          "FX_TIMEOUT_RESERVED": "topic-transfer-position-batch"
         }
       }
     }
@@ -185,7 +191,8 @@ If you want to run integration tests in a repetitive manner, you can startup the
     Start containers required for Integration Tests
 
     ```bash
-    docker-compose -f docker-compose.yml up -d mysql kafka init-kafka kafka-debug-console
+    source ./docker/env.sh
+    docker compose up -d mysql kafka init-kafka redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5
     ```
 
     Run wait script which will report once all required containers are up and running
@@ -220,7 +227,8 @@ If you want to run integration tests in a repetitive manner, you can startup the
     Start containers required for Integration Tests, including a `central-ledger` container which will be used as a proxy shell.
 
     ```bash
-    docker-compose -f docker-compose.yml -f docker-compose.integration.yml up -d kafka mysql central-ledger
+    source ./docker/env.sh
+    docker-compose -f docker-compose.yml -f docker-compose.integration.yml up -d kafka mysql central-ledger init-kafka redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5
     ```
 
     Run the Integration Tests from the `central-ledger` container
@@ -235,24 +243,42 @@ If you want to run override position topic tests you can repeat the above and us
 
 #### For running integration tests for batch processing interactively
 - Run dependecies
-```
-docker-compose up -d mysql kafka init-kafka kafka-debug-console
+```bash
+source ./docker/env.sh
+docker compose up -d mysql kafka init-kafka redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5
 npm run wait-4-docker
 ```
 - Run central-ledger services
 ```
 nvm use
 npm run migrate
-env "CLEDG_KAFKA__EVENT_TYPE_ACTION_TOPIC_MAP__POSITION__PREPARE=topic-transfer-position-batch" npm start
+export CLEDG_KAFKA__EVENT_TYPE_ACTION_TOPIC_MAP__POSITION__PREPARE=topic-transfer-position-batch
+export CLEDG_KAFKA__EVENT_TYPE_ACTION_TOPIC_MAP__POSITION__COMMIT=topic-transfer-position-batch
+export CLEDG_KAFKA__EVENT_TYPE_ACTION_TOPIC_MAP__POSITION__RESERVE=topic-transfer-position-batch
+export CLEDG_KAFKA__EVENT_TYPE_ACTION_TOPIC_MAP__POSITION__TIMEOUT_RESERVED=topic-transfer-position-batch
+export CLEDG_KAFKA__EVENT_TYPE_ACTION_TOPIC_MAP__POSITION__FX_TIMEOUT_RESERVED=topic-transfer-position-batch
+export CLEDG_KAFKA__EVENT_TYPE_ACTION_TOPIC_MAP__POSITION__ABORT=topic-transfer-position-batch
+export CLEDG_KAFKA__EVENT_TYPE_ACTION_TOPIC_MAP__POSITION__FX_ABORT=topic-transfer-position-batch
+npm start
 ```
 - Additionally, run position batch handler in a new terminal
 ```
+nvm use
 export CLEDG_KAFKA__EVENT_TYPE_ACTION_TOPIC_MAP__POSITION__PREPARE=topic-transfer-position-batch
+export CLEDG_KAFKA__EVENT_TYPE_ACTION_TOPIC_MAP__POSITION__FX_PREPARE=topic-transfer-position-batch
 export CLEDG_KAFKA__EVENT_TYPE_ACTION_TOPIC_MAP__POSITION__COMMIT=topic-transfer-position-batch
+export CLEDG_KAFKA__EVENT_TYPE_ACTION_TOPIC_MAP__POSITION__TIMEOUT_RESERVED=topic-transfer-position-batch
+export CLEDG_KAFKA__EVENT_TYPE_ACTION_TOPIC_MAP__POSITION__FX_TIMEOUT_RESERVED=topic-transfer-position-batch
+export CLEDG_KAFKA__EVENT_TYPE_ACTION_TOPIC_MAP__POSITION__ABORT=topic-transfer-position-batch
+export CLEDG_KAFKA__EVENT_TYPE_ACTION_TOPIC_MAP__POSITION__FX_ABORT=topic-transfer-position-batch
 export CLEDG_HANDLERS__API__DISABLED=true
 node src/handlers/index.js handler --positionbatch
 ```
-- Run tests using `npx tape 'test/integration-override/**/handlerBatch.test.js'`
+- Run tests using the following commands in a new terminal
+```
+nvm use
+npm run test:int-override
+```
 
 
 If you want to just run all of the integration suite non-interactively then use npm run `test:integration`.
@@ -263,7 +289,11 @@ It will handle docker start up, migration, service starting and testing. Be sure
 If you want to run functional tests locally utilizing the [ml-core-test-harness](https://github.com/mojaloop/ml-core-test-harness), you can run the following commands:
 
 ```bash
-docker build -t mojaloop/central-ledger:local .
+export NODE_VERSION="$(cat .nvmrc)-alpine"
+docker build \
+  --build-arg NODE_VERSION=$NODE_VERSION \
+  -t mojaloop/central-ledger:local \
+  .
 ```
 
 ```bash

audit-ci.jsonc

@@ -4,6 +4,19 @@
   // Only use one of ["low": true, "moderate": true, "high": true, "critical": true]
   "moderate": true,
   "allowlist": [ // NOTE: Please add as much information as possible to any items added to the allowList
-  "GHSA-w5p7-h5w8-2hfq" // tap-spec>tap-out>trim; This has been analyzed and this is acceptable as it is used to run tests.
+    "GHSA-w5p7-h5w8-2hfq", // tap-spec>tap-out>trim; This has been analyzed and this is acceptable as it is used to run tests.
+    "GHSA-2mvq-xp48-4c77", // https://github.com/advisories/GHSA-2mvq-xp48-4c77
+    "GHSA-5854-jvxx-2cg9", // https://github.com/advisories/GHSA-5854-jvxx-2cg9
+    "GHSA-7hx8-2rxv-66xv", // https://github.com/advisories/GHSA-7hx8-2rxv-66xv
+    "GHSA-c429-5p7v-vgjp", // https://github.com/advisories/GHSA-c429-5p7v-vgjp
+    "GHSA-g64q-3vg8-8f93", // https://github.com/advisories/GHSA-g64q-3vg8-8f93
+    "GHSA-mg85-8mv5-ffjr", // https://github.com/advisories/GHSA-mg85-8mv5-ffjr
+    "GHSA-8hc4-vh64-cxmj", // https://github.com/advisories/GHSA-8hc4-vh64-cxmj
+    "GHSA-952p-6rrq-rcjv", // https://github.com/advisories/GHSA-952p-6rrq-rcjv
+    "GHSA-9wv6-86v2-598j", // https://github.com/advisories/GHSA-9wv6-86v2-598j
+    "GHSA-qwcr-r2fm-qrc7", // https://github.com/advisories/GHSA-qwcr-r2fm-qrc7
+    "GHSA-cm22-4g7w-348p", // https://github.com/advisories/GHSA-cm22-4g7w-348p
+    "GHSA-m6fv-jmcg-4jfg", // https://github.com/advisories/GHSA-m6fv-jmcg-4jfg
+    "GHSA-qw6h-vgh9-j6wx" // https://github.com/advisories/GHSA-qw6h-vgh9-j6wx
   ]
-}
+}

config/default.json

@@ -78,20 +78,36 @@
   },
   "INTERNAL_TRANSFER_VALIDITY_SECONDS": "432000",
   "ENABLE_ON_US_TRANSFERS": false,
+  "PAYEE_PARTICIPANT_CURRENCY_VALIDATION_ENABLED": false,
   "CACHE": {
     "CACHE_ENABLED": false,
     "MAX_BYTE_SIZE": 10000000,
     "EXPIRES_IN_MS": 1000
   },
+  "PROXY_CACHE": {
+    "enabled": true,
+    "type": "redis-cluster",
+    "proxyConfig": {
+      "cluster": [
+        { "host": "localhost", "port": 6379 }
+      ]
+    }
+  },
   "API_DOC_ENDPOINTS_ENABLED": true,
   "KAFKA": {
     "EVENT_TYPE_ACTION_TOPIC_MAP" : {
       "POSITION":{
         "PREPARE": null,
+        "FX_PREPARE": "topic-transfer-position-batch",
         "BULK_PREPARE": null,
         "COMMIT": null,
         "BULK_COMMIT": null,
-        "RESERVE": null
+        "RESERVE": null,
+        "FX_RESERVE": "topic-transfer-position-batch",
+        "TIMEOUT_RESERVED": null,
+        "FX_TIMEOUT_RESERVED": "topic-transfer-position-batch",
+        "ABORT": null,
+        "FX_ABORT": "topic-transfer-position-batch"
       }
     },
     "TOPIC_TEMPLATES": {

docker-compose.yml

@@ -1,9 +1,22 @@
-version: "3.7"
-
 networks:
   cl-mojaloop-net:
     name: cl-mojaloop-net
 
+
+# @see https://uninterrupted.tech/blog/hassle-free-redis-cluster-deployment-using-docker/
+x-redis-node: &REDIS_NODE
+  image: docker.io/bitnami/redis-cluster:6.2.14
+  environment: &REDIS_ENVS
+    ALLOW_EMPTY_PASSWORD: yes
+    REDIS_CLUSTER_DYNAMIC_IPS: no
+    REDIS_CLUSTER_ANNOUNCE_IP: ${REDIS_CLUSTER_ANNOUNCE_IP}
+    REDIS_NODES: redis-node-0:6379 redis-node-1:9301 redis-node-2:9302 redis-node-3:9303 redis-node-4:9304 redis-node-5:9305
+  healthcheck:
+    test: [ "CMD", "redis-cli", "ping" ]
+    timeout: 2s
+  networks:
+    - cl-mojaloop-net
+
 services:
   central-ledger:
     image: mojaloop/central-ledger:local
@@ -31,10 +44,14 @@ services:
       - CLEDG_MONGODB__DISABLED=false
     networks:
       - cl-mojaloop-net
+    extra_hosts:
+      - "redis-node-0:host-gateway"
     depends_on:
       - mysql
       - kafka
       - objstore
+      - redis-node-0
+      # - redis
     healthcheck:
       test: ["CMD", "sh", "-c" ,"apk --no-cache add curl", "&&", "curl", "http://localhost:3001/health"]
       timeout: 20s
@@ -94,6 +111,77 @@ services:
       retries: 10
       start_period: 40s
       interval: 30s
+
+  redis-node-0:
+    <<: *REDIS_NODE
+    environment:
+      <<: *REDIS_ENVS
+      REDIS_CLUSTER_CREATOR: yes
+      REDIS_PORT_NUMBER: 6379
+    depends_on:
+      - redis-node-1
+      - redis-node-2
+    ports:
+      - "6379:6379"
+      - "16379:16379"
+  redis-node-1:
+    <<: *REDIS_NODE
+    environment:
+      <<: *REDIS_ENVS
+      REDIS_PORT_NUMBER: 9301
+    ports:
+      - "9301:9301"
+      - "19301:19301"
+  redis-node-2:
+    <<: *REDIS_NODE
+    environment:
+      <<: *REDIS_ENVS
+      REDIS_PORT_NUMBER: 9302
+    ports:
+      - "9302:9302"
+      - "19302:19302"
+  redis-node-3:
+    <<: *REDIS_NODE
+    environment:
+      <<: *REDIS_ENVS
+      REDIS_PORT_NUMBER: 9303
+    ports:
+      - "9303:9303"
+      - "19303:19303"
+  redis-node-4:
+    <<: *REDIS_NODE
+    environment:
+      <<: *REDIS_ENVS
+      REDIS_PORT_NUMBER: 9304
+    ports:
+      - "9304:9304"
+      - "19304:19304"
+  redis-node-5:
+    <<: *REDIS_NODE
+    environment:
+      <<: *REDIS_ENVS
+      REDIS_PORT_NUMBER: 9305
+    ports:
+      - "9305:9305"
+      - "19305:19305"
+
+##  To be used with proxyCache.type === 'redis'
+#  redis:
+#    image: redis:6.2.4-alpine
+#    restart: "unless-stopped"
+#    environment:
+#      <<: *REDIS_ENVS
+#      REDIS_CLUSTER_CREATOR: yes
+#    depends_on:
+#      - redis-node-1
+#      - redis-node-2
+#      - redis-node-3
+#      - redis-node-4
+#      - redis-node-5
+#    ports:
+#      - "6379:6379"
+#    networks:
+#      - cl-mojaloop-net
 
   mockserver:
     image: jamesdbloom/mockserver

docker/central-ledger/default.json

@@ -82,6 +82,15 @@
     "MAX_BYTE_SIZE": 10000000,
     "EXPIRES_IN_MS": 1000
   },
+  "PROXY_CACHE": {
+    "enabled": true,
+    "type": "redis-cluster",
+    "proxyConfig": {
+      "cluster": [
+        { "host": "redis-node-0", "port": 6379 }
+      ]
+    }
+  },
   "KAFKA": {
     "TOPIC_TEMPLATES": {
       "PARTICIPANT_TOPIC_TEMPLATE": {