From 843f7ab15aa44cc53e4c3642b5f1edf1bd22ca1a Mon Sep 17 00:00:00 2001 From: Dylan Mendelowitz Date: Thu, 26 Oct 2023 14:53:45 -0400 Subject: [PATCH 1/5] WIP Post-extraction date filtering --- package-lock.json | 34 +++++++++++++- package.json | 1 + src/application/app.js | 20 ++++++++ src/helpers/mapperUtils.js | 41 ++++++++++++++++ src/helpers/schemas/config.schema.json | 65 +++++++++++++++++++++++--- 5 files changed, 154 insertions(+), 7 deletions(-) create mode 100644 src/helpers/mapperUtils.js diff --git a/package-lock.json b/package-lock.json index fde4bb9b..8ba334b6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2412,7 +2412,8 @@ }, "ansi-regex": { "version": "5.0.0", - "resolved": "", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", "dev": true }, "ansi-styles": { @@ -3145,6 +3146,11 @@ "which": "^2.0.1" } }, + "crypto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/crypto/-/crypto-1.0.1.tgz", + "integrity": "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig==" + }, "cssom": { "version": "0.4.4", "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz", @@ -4052,6 +4058,32 @@ "url-join": "^4.0.1" } }, + "fhir-mapper": { + "version": "git+https://github.com/standardhealth/fhir-mapper.git#531dcdb9d4673b14aa4fafe8eccfcdeaa1dd370e", + "from": "git+https://github.com/standardhealth/fhir-mapper.git", + "requires": { + "crypto": "^1.0.1", + "fhirpath": "^0.10.3", + "lodash": "^4.17.21" + }, + "dependencies": { + "commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + }, + "fhirpath": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/fhirpath/-/fhirpath-0.10.3.tgz", + "integrity": "sha512-QxASSupObeYe8IAfX/qbSJoU57iitln8sN3W716G4gk6NJcLcbDLuiINibPqQ1MyugUEpUH/trl8Kb9i/l3Mhw==", + "requires": { + "antlr4": "^4.7.1", + "commander": "^2.18.0", + "js-yaml": "^3.12.0" + } + } + } + }, "fhirpath": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/fhirpath/-/fhirpath-2.1.5.tgz", diff --git a/package.json b/package.json index 8b00d500..3ce76412 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "commander": "^6.2.0", "csv-parse": "^4.8.8", "fhir-crud-client": "^1.2.2", + "fhir-mapper": "git+https://github.com/standardhealth/fhir-mapper.git", "fhirpath": "2.1.5", "lodash": "^4.17.21", "moment": "^2.29.4", diff --git a/src/application/app.js b/src/application/app.js index 0c83f5d8..251f7816 100644 --- a/src/application/app.js +++ b/src/application/app.js @@ -5,6 +5,7 @@ const { sendEmailNotification, zipErrors } = require('./tools/emailNotifications const { extractDataForPatients } = require('./tools/mcodeExtraction'); const { parsePatientIds } = require('../helpers/appUtils'); const { validateConfig } = require('../helpers/configUtils'); +const { DateFilterMapper } = require('../helpers/mapperUtils'); function checkInputAndConfig(config, fromDate, toDate) { // Check input args and needed config variables based on client being used @@ -43,6 +44,25 @@ async function mcodeApp(Client, fromDate, toDate, config, pathToRunLogs, debug, logger.info(`Extracting data for ${patientIds.length} patients`); const { extractedData, successfulExtraction, totalExtractionErrors } = await extractDataForPatients(patientIds, mcodeClient, effectiveFromDate, effectiveToDate); + // Perform post-extraction processes + if (config.postExtraction) { + logger.info('Running post-extraction processes'); + // If dateFilter is in the config file, apply date filtering to specified resourceTypes + if (config.postExtraction.dateFilter) { + const filter = config.postExtraction.dateFilter; + logger.info(`Filtering ${filter.resourceTypes} resources to be within ${filter.startDate} and ${filter.endDate}`); + // generate a date filter mapper for each resource type + const mappers = filter.resourceTypes.map((type) => new DateFilterMapper(type, filter.startDate, filter.endDate)); + mappers.forEach((mapper) => { + extractedData.map((bundle) => { + const mappedBundle = bundle; + mappedBundle.entry = mapper.execute(bundle.entry); + return bundle; + }); + }); + } + } + // If we have notification information, send an emailNotification const { notificationInfo } = config; if (notificationInfo && Object.keys(notificationInfo).length > 0) { diff --git a/src/helpers/mapperUtils.js b/src/helpers/mapperUtils.js new file mode 100644 index 00000000..d4116c29 --- /dev/null +++ b/src/helpers/mapperUtils.js @@ -0,0 +1,41 @@ +const { AggregateMapper } = require('fhir-mapper'); +const fhirpath = require('fhirpath'); + +// Mapping of resource tyoe to FHIR date property / format +// This is based on what date type our CSV extractors output +// for example, Observation could have any of the effectiveX date types, but we output effectiveDateTime +const DateFormatByResourceType = { + Observation: 'effectiveDateTime', + Condition: 'extension.valueDateTime', + // TODO add other resource types and their associate dates + // OR write code to check for all date formats +}; + + +// Excludes resources of a specified type from a bundle if they are outside the given date range +class DateFilterMapper extends AggregateMapper { + constructor(resourceType, startDate = null, endDate = null) { + const resourceMapping = { + filter: (r) => r.resource.resourceType === this.resourceType, + exclude: (r) => { + const date = fhirpath.evaluate(r.resource, DateFormatByResourceType[this.resourceType])[0]; + // const date = resource[DateFormatByResourceType[this.resourceType]]; + const time = new Date(date).getTime(); + const start = (new Date(this.startDate)).getTime(); + const end = (new Date(this.endDate)).getTime(); + return (this.endDate && (time > end)) || (this.startDate && time < start); + }, + exec: (resource) => resource, + }; + + super(resourceMapping, {}); + + this.startDate = startDate; + this.endDate = endDate; + this.resourceType = resourceType; + } +} + +module.exports = { + DateFilterMapper, +}; diff --git a/src/helpers/schemas/config.schema.json b/src/helpers/schemas/config.schema.json index d31e41c3..e617cf0d 100644 --- a/src/helpers/schemas/config.schema.json +++ b/src/helpers/schemas/config.schema.json @@ -14,6 +14,9 @@ "notificationInfo": { "$ref": "#/$defs/notificationInfo" }, + "postExtraction": { + "$ref": "#/$defs/postExtraction" + }, "extractors": { "title": "Extractors", "type": "array", @@ -41,7 +44,7 @@ "title": "Request Headers", "type": "object" }, - "csvParse" : { + "csvParse": { "title": "CSV Parse", "type": "object", "properties": { @@ -96,11 +99,61 @@ } }, "dependencies": { - "host": { "required": ["to"] }, - "to": { "required": ["host"] }, - "from": { "required": ["host", "to"] }, - "port": { "required": ["host", "to"] }, - "tlsRejectUnauthorized": { "required": ["host", "to"] } + "host": { + "required": [ + "to" + ] + }, + "to": { + "required": [ + "host" + ] + }, + "from": { + "required": [ + "host", + "to" + ] + }, + "port": { + "required": [ + "host", + "to" + ] + }, + "tlsRejectUnauthorized": { + "required": [ + "host", + "to" + ] + } + } + }, + "postExtraction": { + "title": "Post Extraction Processes", + "type": "object", + "properties": { + "dateFilter": { + "$ref": "#/$defs/postExtraction" + } + } + }, + "dateFilter": { + "title": "Date Filter", + "type": "object", + "properties": { + "resourceTypes": { + "title": "Resource Types", + "type": "string" + }, + "startDate": { + "title": "Start Date", + "type": "string" + }, + "endDate": { + "title": "End Date", + "type": "string" + } } }, "extractor": { From eab1319210c28db9212b3ed0bfd11d2c98c18290 Mon Sep 17 00:00:00 2001 From: Dylan Mendelowitz Date: Mon, 30 Oct 2023 09:47:22 -0400 Subject: [PATCH 2/5] removing commented out code --- src/helpers/mapperUtils.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/helpers/mapperUtils.js b/src/helpers/mapperUtils.js index d4116c29..aec30072 100644 --- a/src/helpers/mapperUtils.js +++ b/src/helpers/mapperUtils.js @@ -19,7 +19,6 @@ class DateFilterMapper extends AggregateMapper { filter: (r) => r.resource.resourceType === this.resourceType, exclude: (r) => { const date = fhirpath.evaluate(r.resource, DateFormatByResourceType[this.resourceType])[0]; - // const date = resource[DateFormatByResourceType[this.resourceType]]; const time = new Date(date).getTime(); const start = (new Date(this.startDate)).getTime(); const end = (new Date(this.endDate)).getTime(); From 820851baa8b285d7ace3836d4db98e5073be5b0a Mon Sep 17 00:00:00 2001 From: Dylan Mendelowitz Date: Thu, 2 Nov 2023 12:24:58 -0400 Subject: [PATCH 3/5] Mapper now included in config file as .js file --- .gitignore | 2 +- src/application/app.js | 31 +++++++++++++---------------- src/helpers/mapperUtils.js | 40 -------------------------------------- 3 files changed, 14 insertions(+), 59 deletions(-) delete mode 100644 src/helpers/mapperUtils.js diff --git a/.gitignore b/.gitignore index a717e2a8..9f8e5364 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,5 @@ node_modules/ .DS_Store output/ logs/ -config/*.json +config/* !config/csv.config.example.json diff --git a/src/application/app.js b/src/application/app.js index 251f7816..362af874 100644 --- a/src/application/app.js +++ b/src/application/app.js @@ -1,11 +1,12 @@ const moment = require('moment'); +const fs = require('fs'); +const { AggregateMapper } = require('fhir-mapper'); const logger = require('../helpers/logger'); const { RunInstanceLogger } = require('./tools/RunInstanceLogger'); const { sendEmailNotification, zipErrors } = require('./tools/emailNotifications'); const { extractDataForPatients } = require('./tools/mcodeExtraction'); const { parsePatientIds } = require('../helpers/appUtils'); const { validateConfig } = require('../helpers/configUtils'); -const { DateFilterMapper } = require('../helpers/mapperUtils'); function checkInputAndConfig(config, fromDate, toDate) { // Check input args and needed config variables based on client being used @@ -44,23 +45,17 @@ async function mcodeApp(Client, fromDate, toDate, config, pathToRunLogs, debug, logger.info(`Extracting data for ${patientIds.length} patients`); const { extractedData, successfulExtraction, totalExtractionErrors } = await extractDataForPatients(patientIds, mcodeClient, effectiveFromDate, effectiveToDate); - // Perform post-extraction processes - if (config.postExtraction) { - logger.info('Running post-extraction processes'); - // If dateFilter is in the config file, apply date filtering to specified resourceTypes - if (config.postExtraction.dateFilter) { - const filter = config.postExtraction.dateFilter; - logger.info(`Filtering ${filter.resourceTypes} resources to be within ${filter.startDate} and ${filter.endDate}`); - // generate a date filter mapper for each resource type - const mappers = filter.resourceTypes.map((type) => new DateFilterMapper(type, filter.startDate, filter.endDate)); - mappers.forEach((mapper) => { - extractedData.map((bundle) => { - const mappedBundle = bundle; - mappedBundle.entry = mapper.execute(bundle.entry); - return bundle; - }); - }); - } + // Post-extraction mapping + if (fs.existsSync('./config/mapper.js')) { + logger.info('Applying post-extraction mapping'); + // eslint-disable-next-line global-require + const { resourceMapping, variables } = require('../../config/mapper.js'); + const mapper = new AggregateMapper(resourceMapping, variables); + extractedData.map((bundle) => { + const mappedBundle = bundle; + mappedBundle.entry = mapper.execute(bundle.entry); + return bundle; + }); } // If we have notification information, send an emailNotification diff --git a/src/helpers/mapperUtils.js b/src/helpers/mapperUtils.js deleted file mode 100644 index aec30072..00000000 --- a/src/helpers/mapperUtils.js +++ /dev/null @@ -1,40 +0,0 @@ -const { AggregateMapper } = require('fhir-mapper'); -const fhirpath = require('fhirpath'); - -// Mapping of resource tyoe to FHIR date property / format -// This is based on what date type our CSV extractors output -// for example, Observation could have any of the effectiveX date types, but we output effectiveDateTime -const DateFormatByResourceType = { - Observation: 'effectiveDateTime', - Condition: 'extension.valueDateTime', - // TODO add other resource types and their associate dates - // OR write code to check for all date formats -}; - - -// Excludes resources of a specified type from a bundle if they are outside the given date range -class DateFilterMapper extends AggregateMapper { - constructor(resourceType, startDate = null, endDate = null) { - const resourceMapping = { - filter: (r) => r.resource.resourceType === this.resourceType, - exclude: (r) => { - const date = fhirpath.evaluate(r.resource, DateFormatByResourceType[this.resourceType])[0]; - const time = new Date(date).getTime(); - const start = (new Date(this.startDate)).getTime(); - const end = (new Date(this.endDate)).getTime(); - return (this.endDate && (time > end)) || (this.startDate && time < start); - }, - exec: (resource) => resource, - }; - - super(resourceMapping, {}); - - this.startDate = startDate; - this.endDate = endDate; - this.resourceType = resourceType; - } -} - -module.exports = { - DateFilterMapper, -}; From 0580cac0a1259f2b8af44696368f5037c6d99a6c Mon Sep 17 00:00:00 2001 From: Dylan Mendelowitz Date: Thu, 2 Nov 2023 12:27:53 -0400 Subject: [PATCH 4/5] lint fix --- src/application/app.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/application/app.js b/src/application/app.js index 362af874..85630096 100644 --- a/src/application/app.js +++ b/src/application/app.js @@ -1,5 +1,6 @@ const moment = require('moment'); const fs = require('fs'); +const path = require('path'); const { AggregateMapper } = require('fhir-mapper'); const logger = require('../helpers/logger'); const { RunInstanceLogger } = require('./tools/RunInstanceLogger'); @@ -48,8 +49,8 @@ async function mcodeApp(Client, fromDate, toDate, config, pathToRunLogs, debug, // Post-extraction mapping if (fs.existsSync('./config/mapper.js')) { logger.info('Applying post-extraction mapping'); - // eslint-disable-next-line global-require - const { resourceMapping, variables } = require('../../config/mapper.js'); + // eslint-disable-next-line global-require, import/no-dynamic-require + const { resourceMapping, variables } = require(path.resolve('../../config/mapper.js')); const mapper = new AggregateMapper(resourceMapping, variables); extractedData.map((bundle) => { const mappedBundle = bundle; From 6a2c6ccb1659ee7d7c0dcb610dcc4156e716543d Mon Sep 17 00:00:00 2001 From: Dylan Mendelowitz Date: Mon, 13 Nov 2023 12:29:20 -0500 Subject: [PATCH 5/5] Restoring config schema --- src/helpers/schemas/config.schema.json | 65 +++----------------------- 1 file changed, 6 insertions(+), 59 deletions(-) diff --git a/src/helpers/schemas/config.schema.json b/src/helpers/schemas/config.schema.json index e617cf0d..d31e41c3 100644 --- a/src/helpers/schemas/config.schema.json +++ b/src/helpers/schemas/config.schema.json @@ -14,9 +14,6 @@ "notificationInfo": { "$ref": "#/$defs/notificationInfo" }, - "postExtraction": { - "$ref": "#/$defs/postExtraction" - }, "extractors": { "title": "Extractors", "type": "array", @@ -44,7 +41,7 @@ "title": "Request Headers", "type": "object" }, - "csvParse": { + "csvParse" : { "title": "CSV Parse", "type": "object", "properties": { @@ -99,61 +96,11 @@ } }, "dependencies": { - "host": { - "required": [ - "to" - ] - }, - "to": { - "required": [ - "host" - ] - }, - "from": { - "required": [ - "host", - "to" - ] - }, - "port": { - "required": [ - "host", - "to" - ] - }, - "tlsRejectUnauthorized": { - "required": [ - "host", - "to" - ] - } - } - }, - "postExtraction": { - "title": "Post Extraction Processes", - "type": "object", - "properties": { - "dateFilter": { - "$ref": "#/$defs/postExtraction" - } - } - }, - "dateFilter": { - "title": "Date Filter", - "type": "object", - "properties": { - "resourceTypes": { - "title": "Resource Types", - "type": "string" - }, - "startDate": { - "title": "Start Date", - "type": "string" - }, - "endDate": { - "title": "End Date", - "type": "string" - } + "host": { "required": ["to"] }, + "to": { "required": ["host"] }, + "from": { "required": ["host", "to"] }, + "port": { "required": ["host", "to"] }, + "tlsRejectUnauthorized": { "required": ["host", "to"] } } }, "extractor": {