WIP Post-extraction date filtering

package.json

@@ -27,6 +27,7 @@
     "commander": "^6.2.0",
     "csv-parse": "^4.8.8",
     "fhir-crud-client": "^1.2.2",
+    "fhir-mapper": "git+https://github.com/standardhealth/fhir-mapper.git",
     "fhirpath": "2.1.5",
     "lodash": "^4.17.21",
     "moment": "^2.29.4",

src/application/app.js

@@ -5,6 +5,7 @@ const { sendEmailNotification, zipErrors } = require('./tools/emailNotifications
 const { extractDataForPatients } = require('./tools/mcodeExtraction');
 const { parsePatientIds } = require('../helpers/appUtils');
 const { validateConfig } = require('../helpers/configUtils');
+const { DateFilterMapper } = require('../helpers/mapperUtils');
 
 function checkInputAndConfig(config, fromDate, toDate) {
   // Check input args and needed config variables based on client being used
@@ -43,6 +44,25 @@ async function mcodeApp(Client, fromDate, toDate, config, pathToRunLogs, debug,
   logger.info(`Extracting data for ${patientIds.length} patients`);
   const { extractedData, successfulExtraction, totalExtractionErrors } = await extractDataForPatients(patientIds, mcodeClient, effectiveFromDate, effectiveToDate);
 
+  // Perform post-extraction processes
+  if (config.postExtraction) {
+    logger.info('Running post-extraction processes');
+    // If dateFilter is in the config file, apply date filtering to specified resourceTypes
+    if (config.postExtraction.dateFilter) {
+      const filter = config.postExtraction.dateFilter;
+      logger.info(`Filtering ${filter.resourceTypes} resources to be within ${filter.startDate} and ${filter.endDate}`);
+      // generate a date filter mapper for each resource type
+      const mappers = filter.resourceTypes.map((type) => new DateFilterMapper(type, filter.startDate, filter.endDate));
+      mappers.forEach((mapper) => {
+        extractedData.map((bundle) => {
+          const mappedBundle = bundle;
+          mappedBundle.entry = mapper.execute(bundle.entry);
+          return bundle;
+        });
+      });
+    }
+  }
+
   // If we have notification information, send an emailNotification
   const { notificationInfo } = config;
   if (notificationInfo && Object.keys(notificationInfo).length > 0) {

src/helpers/mapperUtils.js

@@ -0,0 +1,41 @@
+const { AggregateMapper } = require('fhir-mapper');
+const fhirpath = require('fhirpath');
+
+// Mapping of resource tyoe to FHIR date property / format
+// This is based on what date type our CSV extractors output
+// for example, Observation could have any of the effectiveX date types, but we output effectiveDateTime
+const DateFormatByResourceType = {
+  Observation: 'effectiveDateTime',
+  Condition: 'extension.valueDateTime',
+  // TODO add other resource types and their associate dates
+  // OR write code to check for all date formats
+};
+
+
+// Excludes resources of a specified type from a bundle if they are outside the given date range
+class DateFilterMapper extends AggregateMapper {
+  constructor(resourceType, startDate = null, endDate = null) {
+    const resourceMapping = {
+      filter: (r) => r.resource.resourceType === this.resourceType,
+      exclude: (r) => {
+        const date = fhirpath.evaluate(r.resource, DateFormatByResourceType[this.resourceType])[0];
+        // const date = resource[DateFormatByResourceType[this.resourceType]];
+        const time = new Date(date).getTime();
+        const start = (new Date(this.startDate)).getTime();
+        const end = (new Date(this.endDate)).getTime();
+        return (this.endDate && (time > end)) || (this.startDate && time < start);
+      },
+      exec: (resource) => resource,
+    };
+
+    super(resourceMapping, {});
+
+    this.startDate = startDate;
+    this.endDate = endDate;
+    this.resourceType = resourceType;
+  }
+}
+
+module.exports = {
+  DateFilterMapper,
+};

src/helpers/schemas/config.schema.json

@@ -14,6 +14,9 @@
     "notificationInfo": {
       "$ref": "#/$defs/notificationInfo"
     },
+    "postExtraction": {
+      "$ref": "#/$defs/postExtraction"
+    },
     "extractors": {
       "title": "Extractors",
       "type": "array",
@@ -41,7 +44,7 @@
           "title": "Request Headers",
           "type": "object"
         },
-        "csvParse" : {
+        "csvParse": {
           "title": "CSV Parse",
           "type": "object",
           "properties": {
@@ -96,11 +99,61 @@
         }
       },
       "dependencies": {
-        "host": { "required": ["to"] },
-        "to": { "required": ["host"] },
-        "from": { "required": ["host", "to"] },
-        "port": { "required": ["host", "to"] },
-        "tlsRejectUnauthorized": { "required": ["host", "to"] }
+        "host": {
+          "required": [
+            "to"
+          ]
+        },
+        "to": {
+          "required": [
+            "host"
+          ]
+        },
+        "from": {
+          "required": [
+            "host",
+            "to"
+          ]
+        },
+        "port": {
+          "required": [
+            "host",
+            "to"
+          ]
+        },
+        "tlsRejectUnauthorized": {
+          "required": [
+            "host",
+            "to"
+          ]
+        }
+      }
+    },
+    "postExtraction": {
+      "title": "Post Extraction Processes",
+      "type": "object",
+      "properties": {
+        "dateFilter": {
+          "$ref": "#/$defs/postExtraction"
+        }
+      }
+    },
+    "dateFilter": {
+      "title": "Date Filter",
+      "type": "object",
+      "properties": {
+        "resourceTypes": {
+          "title": "Resource Types",
+          "type": "string"
+        },
+        "startDate": {
+          "title": "Start Date",
+          "type": "string"
+        },
+        "endDate": {
+          "title": "End Date",
+          "type": "string"
+        }
       }
     },
     "extractor": {