From 9e34200e96be566c1fb8df587f7c295daab1a861 Mon Sep 17 00:00:00 2001 From: k-joseph Date: Tue, 20 Aug 2019 17:45:33 +0300 Subject: [PATCH 1/9] MEPTS-224: added merge person transformation --- .gitignore | 2 +- .../transformations/merge-person.ktr | 791 ++++++++++++++++++ 2 files changed, 792 insertions(+), 1 deletion(-) create mode 100644 src/main/resources/pdiresources/transformations/merge-person.ktr diff --git a/.gitignore b/.gitignore index fe1d7b6..3489826 100644 --- a/.gitignore +++ b/.gitignore @@ -45,5 +45,5 @@ out/ # End of https://www.gitignore.io/api/java,gradle # migrator files -pdiresources/ +/pdiresources/ settings.properties diff --git a/src/main/resources/pdiresources/transformations/merge-person.ktr b/src/main/resources/pdiresources/transformations/merge-person.ktr new file mode 100644 index 0000000..751e81a --- /dev/null +++ b/src/main/resources/pdiresources/transformations/merge-person.ktr @@ -0,0 +1,791 @@ + + + + merge-person + + + + Normal + 0 + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/06 09:23:50.203 + - + 2019/08/06 09:23:50.203 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + OpenMRS destination + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + OpenMRS source + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_SOURCE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + Load Person from Source + Persons In Both Source & Destination + Y + + + Persons In Both Source & Destination + Persons in Source Not Destination + Y + + + Persons in Source Not Destination + Save Missing persons into Destination + Y + + + + Persons in Source Not Destination + FilterRows + + Y + + 1 + + none + + + ok + + + + N + source_person_id + IS NULL + + + + + + + + + + + + + 256 + 144 + Y + + + + Load Person from Source + TableInput + + Y + + 1 + + none + + + OpenMRS source + SELECT * FROM person + 0 + + N + N + N + + + + + + + + + + 96 + 80 + Y + + + + Persons In Both Source & Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select person_id as source_person_id from person where uuid = ? + Y + N + + + uuid + String + + + + + + + + + + + + 96 + 256 + Y + + + + Save Missing persons into Destination + TableOutput + + Y + + 1 + + none + + + OpenMRS destination + +
person
+ 1000 + N + N + Y + Y + N + + N + Y + N + + Y + N + + + + uuid + uuid + + + person_id + source_person_id + + + birthdate + birthdate + + + gender + gender + + + birthdate_estimated + birthdate_estimated + + + dead + dead + + + death_date + death_date + + + cause_of_death + cause_of_death + + + creator + creator + + + date_created + date_created + + + changed_by + changed_by + + + date_changed + date_changed + + + voided + voided + + + voided_by + voided_by + + + date_voided + date_voided + + + void_reason + void_reason + + + deathdate_estimated + deathdate_estimated + + + birthtime + birthtime + + + + + + + + + + + + 544 + 144 + Y + + + + + + + N + + From 381888644c650df6caff0edb8edac1c13b5e2206 Mon Sep 17 00:00:00 2001 From: k-joseph Date: Thu, 22 Aug 2019 17:44:11 +0300 Subject: [PATCH 2/9] MEPTS-224: adding merge-person.ktr to migrator pdiresources --- src/main/java/com/openmrs/migrator/Migrator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/openmrs/migrator/Migrator.java b/src/main/java/com/openmrs/migrator/Migrator.java index 998a897..42ad77f 100644 --- a/src/main/java/com/openmrs/migrator/Migrator.java +++ b/src/main/java/com/openmrs/migrator/Migrator.java @@ -144,6 +144,7 @@ private void executeSetupCommand() throws IOException, SQLException, SettingsExc pdiFiles.add( SettingsService.PDI_RESOURCES_DIR + "/transformations/validate-visit-attribute-types.ktr"); pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/validate-visit-types.ktr"); + pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-person.ktr"); pdiFiles.add(SettingsService.SETTINGS_PROPERTIES); bootstrapService.createDirectoryStructure(dirList); From 39033d3224c9804e98c21057e7f11e7860f6658d Mon Sep 17 00:00:00 2001 From: k-joseph Date: Thu, 29 Aug 2019 16:48:54 +0300 Subject: [PATCH 3/9] MEPTS-224: Added functional person and user merge transformations --- .../resources/pdiresources/jobs/merging.kjb | 661 +++++++++ .../transformations/create-mappings.ktr | 546 ++++++++ .../transformations/drop-mappings.ktr | 532 ++++++++ .../transformations/merge-persons.ktr | 1044 ++++++++++++++ .../transformations/merge-users.ktr | 1211 +++++++++++++++++ ....ktr => update-mapped-person-creators.ktr} | 221 +-- .../update-mapped-user-creators.ktr | 718 ++++++++++ .../update-mapped-user-persons.ktr | 718 ++++++++++ 8 files changed, 5504 insertions(+), 147 deletions(-) create mode 100644 src/main/resources/pdiresources/jobs/merging.kjb create mode 100644 src/main/resources/pdiresources/transformations/create-mappings.ktr create mode 100644 src/main/resources/pdiresources/transformations/drop-mappings.ktr create mode 100644 src/main/resources/pdiresources/transformations/merge-persons.ktr create mode 100644 src/main/resources/pdiresources/transformations/merge-users.ktr rename src/main/resources/pdiresources/transformations/{merge-person.ktr => update-mapped-person-creators.ktr} (79%) create mode 100644 src/main/resources/pdiresources/transformations/update-mapped-user-creators.ktr create mode 100644 src/main/resources/pdiresources/transformations/update-mapped-user-persons.ktr diff --git a/src/main/resources/pdiresources/jobs/merging.kjb b/src/main/resources/pdiresources/jobs/merging.kjb new file mode 100644 index 0000000..fd30e58 --- /dev/null +++ b/src/main/resources/pdiresources/jobs/merging.kjb @@ -0,0 +1,661 @@ + + + merging + + + + / + - + 2019/08/26 18:05:32.913 + - + 2019/08/26 18:05:32.913 + + + + + + + + + + + + + ID_JOB + Y + ID_JOB + + + CHANNEL_ID + Y + CHANNEL_ID + + + JOBNAME + Y + JOBNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + START_JOB_ENTRY + N + START_JOB_ENTRY + + + CLIENT + N + CLIENT + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + JOBNAME + Y + TRANSNAME + + + JOBENTRYNAME + Y + STEPNAME + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + RESULT + Y + RESULT + + + NR_RESULT_ROWS + Y + NR_RESULT_ROWS + + + NR_RESULT_FILES + Y + NR_RESULT_FILES + + + LOG_FIELD + N + LOG_FIELD + + + COPY_NR + N + COPY_NR + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + N + + + + Start + + SPECIAL + + Y + N + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + Y + 0 + 32 + 32 + + + + create mappings table if missing + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/create-mappings.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 0 + 160 + 32 + + + + drop-mappings + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/drop-mappings.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 0 + 880 + 336 + + + + merge persons + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/merge-persons.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 0 + 352 + 32 + + + + update mapped person creators + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-person-creators.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 1 + 656 + 32 + + + + merge users + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/merge-users.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 496 + 32 + + + + update mapped user persons + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-user-persons.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 880 + 144 + + + + update mapped user creators + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-user-creators.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 880 + 32 + + + + + + Start + create mappings table if missing + 0 + 0 + Y + Y + Y + + + create mappings table if missing + merge persons + 0 + 0 + Y + Y + N + + + merge persons + merge users + 0 + 2 + Y + Y + N + + + merge users + update mapped person creators + 2 + 1 + Y + Y + N + + + update mapped person creators + update mapped user creators + 1 + 2 + Y + Y + N + + + update mapped user creators + update mapped user persons + 2 + 2 + Y + Y + N + + + update mapped user persons + drop-mappings + 2 + 0 + Y + Y + N + + + + + + + METASTORE.pentaho + + Default Run Configuration + {"namespace":"pentaho","id":"Default Run Configuration","name":"Default Run Configuration","description":"Defines a default run configuration","metaStoreName":null} + + + + {"_":"Embedded MetaStore Elements","namespace":"pentaho","type":"Default Run Configuration"} + + Pentaho local + {"children":[{"children":[],"id":"server","value":null},{"children":[],"id":"clustered","value":"N"},{"children":[],"id":"name","value":"Pentaho local"},{"children":[],"id":"description","value":null},{"children":[],"id":"pentaho","value":"N"},{"children":[],"id":"readOnly","value":"Y"},{"children":[],"id":"sendResources","value":"N"},{"children":[],"id":"logRemoteExecutionLocally","value":"N"},{"children":[],"id":"remote","value":"N"},{"children":[],"id":"local","value":"Y"},{"children":[],"id":"showTransformations","value":"N"}],"id":"Pentaho local","value":null,"name":"Pentaho local","owner":null,"ownerPermissionsList":[]} + + + + diff --git a/src/main/resources/pdiresources/transformations/create-mappings.ktr b/src/main/resources/pdiresources/transformations/create-mappings.ktr new file mode 100644 index 0000000..335ff14 --- /dev/null +++ b/src/main/resources/pdiresources/transformations/create-mappings.ktr @@ -0,0 +1,546 @@ + + + + create-mappings + + + + Normal + / + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/26 16:16:32.316 + - + 2019/08/26 16:16:32.316 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + destination/merge + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + + create mappings table if missing + ExecSQL + + Y + + 1 + + none + + + destination/merge + N + N + N + N + CREATE TABLE IF NOT EXISTS migrator_merge_mappings( + mapping_id int(11) NOT NULL AUTO_INCREMENT, + entity VARCHAR(38) NOT NULL, + attribute VARCHAR(38) NOT NULL, + answer int(11) NOT NULL, + uuid VARCHAR(38), + PRIMARY KEY (mapping_id) +) ENGINE=INNODB; + N + + + + + + + + + + + + + + + + 208 + 48 + Y + + + + + + + N + + diff --git a/src/main/resources/pdiresources/transformations/drop-mappings.ktr b/src/main/resources/pdiresources/transformations/drop-mappings.ktr new file mode 100644 index 0000000..3cf0de2 --- /dev/null +++ b/src/main/resources/pdiresources/transformations/drop-mappings.ktr @@ -0,0 +1,532 @@ + + + + drop-mappings + + + + Normal + / + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/28 23:33:23.668 + - + 2019/08/28 23:33:23.668 + + N + + + + + destination/merge + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + + Delete mappings table + Delete + + Y + + 1 + + none + + + destination/merge + 100 + + +
migrator_merge_mappings
+ + + + + + + + + + + 144 + 80 + Y + + + + + + + N + + diff --git a/src/main/resources/pdiresources/transformations/merge-persons.ktr b/src/main/resources/pdiresources/transformations/merge-persons.ktr new file mode 100644 index 0000000..c21c1a9 --- /dev/null +++ b/src/main/resources/pdiresources/transformations/merge-persons.ktr @@ -0,0 +1,1044 @@ + + + + merge-persons + + + + Normal + 0 + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/06 09:23:50.203 + - + 2019/08/06 09:23:50.203 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + OpenMRS destination + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + OpenMRS source + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_SOURCE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + Load Persons from Source + Persons In Both Source & Destination + Y + + + Persons In Both Source & Destination + Persons in Source Not Destination + Y + + + Creator Exists on Destination + Save Missing persons into Destination + Y + + + insert merge mapping for persons + Creator Exists on Destination + Y + + + Creator Exists on Destination + insert merge mapping for missing creator + Y + + + Person Creator uuid From Destination + Daemon user + Y + + + Daemon user + insert merge mapping for persons + Y + + + insert merge mapping for missing creator + Set creator to daemon user + Y + + + Set creator to daemon user + Save Missing persons into Destination + Y + + + Persons in Source Not Destination + Person Creator uuid From Destination + Y + + + + Creator Exists on Destination + FilterRows + + Y + + 1 + + none + + + Save Missing persons into Destination + insert merge mapping for missing creator + + + N + destination_creator_uuid + IS NOT NULL + + + + + + + + + + + + + 336 + 416 + Y + + + + Daemon user + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select user_id as daemon_user_id from users where system_id = 'daemon'; + Y + N + + + + + + + + + + + + 208 + 32 + Y + + + + Load Persons from Source + TableInput + + N + + 1 + + none + + + OpenMRS source + SELECT * FROM person + 0 + + N + N + N + + + + + + + + + + 80 + 16 + Y + + + + Person Creator uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_creator_uuid from users where user_id = ? + Y + N + + + creator + String + + + + + + + + + + + + 240 + 160 + Y + + + + Persons In Both Source & Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select person_id as destination_person_id from person where uuid = ? + Y + N + + + uuid + String + + + + + + + + + + + + 112 + 544 + Y + + + + Persons in Source Not Destination + FilterRows + + Y + + 1 + + none + + + Person Creator uuid From Source + + + + N + destination_person_id + IS NULL + + + + + + + + + + + + + 224 + 288 + Y + + + + Save Missing persons into Destination + TableOutput + + Y + + 1 + + none + + + OpenMRS destination + +
person
+ 1000 + N + N + Y + Y + N + + N + Y + N + + Y + N + + + + uuid + uuid + + + birthdate + birthdate + + + gender + gender + + + birthdate_estimated + birthdate_estimated + + + dead + dead + + + death_date + death_date + + + cause_of_death + cause_of_death + + + creator + creator + + + date_created + date_created + + + changed_by + changed_by + + + date_changed + date_changed + + + voided + voided + + + voided_by + voided_by + + + date_voided + date_voided + + + void_reason + void_reason + + + deathdate_estimated + deathdate_estimated + + + birthtime + birthtime + + + + + + + + + + + + 896 + 16 + Y + + + + Set creator to daemon user + SetValueField + + Y + + 1 + + none + + + + + creator + daemon_user_id + + + + + + + + + + + + 544 + 16 + Y + + + + insert merge mapping for missing creator + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('person', 'creator', ?, ?); + Y + + + + + + + creator + + + uuid + + + + + + + + + + + + 880 + 416 + Y + + + + insert merge mapping for persons + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('person', 'person_id', ?, ?); + Y + + + + + + + person_id + + + uuid + + + + + + + + + + + + 416 + 48 + Y + + + + + + + N + + diff --git a/src/main/resources/pdiresources/transformations/merge-users.ktr b/src/main/resources/pdiresources/transformations/merge-users.ktr new file mode 100644 index 0000000..bc2dd7d --- /dev/null +++ b/src/main/resources/pdiresources/transformations/merge-users.ktr @@ -0,0 +1,1211 @@ + + + + merge-users + + + + Normal + 0 + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/06 09:23:50.203 + - + 2019/08/06 09:23:50.203 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + OpenMRS destination + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + OpenMRS source + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_SOURCE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + Creator Exists on Destination + insert merge mapping for missing creator + Y + + + Load Users from Source + Users In Both Source & Destination + Y + + + Users In Both Source & Destination + Persons in Source Not Destination + Y + + + Persons in Source Not Destination + User Creator uuid From Destination + Y + + + insert merge mapping for missing creator + Set creator to daemon user + Y + + + insert merge mapping for users + Creator Exists on Destination + Y + + + Daemon user/person + insert merge mapping for users + Y + + + Set creator to daemon user + Person Exists on Destination + Y + + + Creator Exists on Destination + Person Exists on Destination + Y + + + Set person_id to daemon person id + Save Missing users into Destination + Y + + + Person Exists on Destination + Save Missing users into Destination + Y + + + User Creator uuid From Destination + User Person uuid From Destination + Y + + + User Person uuid From Destination + Daemon user/person + Y + + + Person Exists on Destination + insert merge mapping for missing person + Y + + + insert merge mapping for missing person + Set person_id to daemon person id + Y + + + + Creator Exists on Destination + FilterRows + + Y + + 1 + + none + + + Person Exists on Destination + insert merge mapping for missing creator + + + N + destination_creator_uuid + IS NOT NULL + + + + + + + + + + + + + 352 + 416 + Y + + + + Daemon user/person + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select user_id AS daemon_user_id, person_id AS daemon_person_id from users where system_id = 'daemon'; + Y + N + + + + + + + + + + + + 224 + 32 + Y + + + + Load Users from Source + TableInput + + N + + 1 + + none + + + OpenMRS source + SELECT * FROM users + 0 + + N + N + N + + + + + + + + + + 96 + 16 + Y + + + + User Creator uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_creator_uuid from users where user_id = ? + Y + N + + + creator + String + + + + + + + + + + + + 256 + 192 + Y + + + + Users In Both Source & Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select user_id as destination_user_id from users where uuid = ? + Y + N + + + uuid + String + + + + + + + + + + + + 128 + 544 + Y + + + + Persons in Source Not Destination + FilterRows + + Y + + 1 + + none + + + User Creator uuid From Destination + + + + N + destination_user_id + IS NULL + + + + + + + + + + + + + 240 + 336 + Y + + + + Save Missing users into Destination + TableOutput + + Y + + 1 + + none + + + OpenMRS destination + +
users
+ 1000 + N + N + Y + Y + N + + N + Y + N + + Y + N + + + + system_id + system_id + + + username + username + + + password + password + + + salt + salt + + + secret_question + secret_question + + + secret_answer + secret_answer + + + creator + creator + + + date_created + date_created + + + changed_by + changed_by + + + date_changed + date_changed + + + person_id + person_id + + + retired + retired + + + retired_by + retired_by + + + date_retired + date_retired + + + retire_reason + retire_reason + + + uuid + uuid + + + + + + + + + + + + 944 + 384 + Y + + + + Set creator to daemon user + SetValueField + + Y + + 1 + + none + + + + + creator + daemon_user_id + + + + + + + + + + + + 704 + 0 + Y + + + + insert merge mapping for missing creator + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('users', 'creator', ?, ?); + Y + + + + + + + creator + + + uuid + + + + + + + + + + + + 592 + 128 + Y + + + + insert merge mapping for users + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('users', 'user_id', ?, ?); + Y + + + + + + + user_id + + + uuid + + + + + + + + + + + + 496 + 32 + Y + + + + Person Exists on Destination + FilterRows + + Y + + 1 + + none + + + Save Missing users into Destination + insert merge mapping for missing person + + + N + destination_person_uuid + IS NOT NULL + + + + + + + + + + + + + 672 + 240 + Y + + + + Set person_id to daemon person id + SetValueField + + Y + + 1 + + none + + + + + person_id + daemon_person_id + + + + + + + + + + + + 672 + 448 + Y + + + + User Person uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_person_uuid from person where person_id = ? + Y + N + + + person_id + String + + + + + + + + + + + + 240 + 112 + Y + + + + insert merge mapping for missing person + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('users', 'person_id', ?, ?); + Y + + + + + + + person_id + + + uuid + + + + + + + + + + + + 624 + 336 + Y + + + + + + + N + + diff --git a/src/main/resources/pdiresources/transformations/merge-person.ktr b/src/main/resources/pdiresources/transformations/update-mapped-person-creators.ktr similarity index 79% rename from src/main/resources/pdiresources/transformations/merge-person.ktr rename to src/main/resources/pdiresources/transformations/update-mapped-person-creators.ktr index 751e81a..39e82d8 100644 --- a/src/main/resources/pdiresources/transformations/merge-person.ktr +++ b/src/main/resources/pdiresources/transformations/update-mapped-person-creators.ktr @@ -1,12 +1,11 @@ - merge-person + update-mapped-person-creators Normal - 0 / @@ -428,16 +427,16 @@ - - 2019/08/06 09:23:50.203 + 2019/08/29 02:16:38.102 - - 2019/08/06 09:23:50.203 + 2019/08/29 02:16:38.102 H4sIAAAAAAAAAAMAAAAAAAAAAAA= N - OpenMRS destination + destination/merge ${ETL_DATABASE_HOST} MYSQL Native @@ -492,7 +491,7 @@ - OpenMRS source + source ${ETL_DATABASE_HOST} MYSQL Native @@ -548,24 +547,24 @@ - Load Person from Source - Persons In Both Source & Destination + Read creators from mappings + Get creator Source user uuid Y - Persons In Both Source & Destination - Persons in Source Not Destination + Get creator Source user uuid + Get creator Destination user id Y - Persons in Source Not Destination - Save Missing persons into Destination + Get creator Destination user id + Update person creator to mapped in destination Y - Persons in Source Not Destination - FilterRows + Get creator Destination user id + DBJoin Y @@ -574,16 +573,17 @@ none - ok - - - - N - source_person_id - IS NULL - - - + destination/merge + 0 + SELECT user_id as mapping_destination_creator_user_id from users where uuid = ? + Y + N + + + source_creator_uuid + String + + @@ -593,14 +593,14 @@ - 256 - 144 + 544 + 176 Y - Load Person from Source - TableInput + Get creator Source user uuid + DBJoin Y @@ -609,13 +609,17 @@ none - OpenMRS source - SELECT * FROM person - 0 - - N - N - N + source + 0 + SELECT uuid as source_creator_uuid from users where user_id = ? + Y + N + + + mapping_source_creator_user_id + Integer + + @@ -625,14 +629,14 @@ - 96 - 80 + 336 + 32 Y - Persons In Both Source & Destination - DBJoin + Read creators from mappings + TableInput Y @@ -641,17 +645,13 @@ none - OpenMRS destination - 0 - select person_id as source_person_id from person where uuid = ? - Y - N - - - uuid - String - - + destination/merge + SELECT uuid AS mapping_source_person_uuid, answer AS mapping_source_creator_user_id FROM migrator_merge_mappings WHERE entity = 'person' AND attribute = 'creator'; + 0 + + N + N + N @@ -661,14 +661,14 @@ - 96 - 256 + 144 + 128 Y - Save Missing persons into Destination - TableOutput + Update person creator to mapped in destination + InsertUpdate Y @@ -677,97 +677,24 @@ none - OpenMRS destination - - person
- 1000 - N - N - Y - Y - N - - N - Y - N - - Y - N - - - - uuid - uuid - - - person_id - source_person_id - - - birthdate - birthdate - - - gender - gender - - - birthdate_estimated - birthdate_estimated - - - dead - dead - - - death_date - death_date - - - cause_of_death - cause_of_death - - - creator - creator - - - date_created - date_created - - - changed_by - changed_by - - - date_changed - date_changed - - - voided - voided - - - voided_by - voided_by - - - date_voided - date_voided - - - void_reason - void_reason - - - deathdate_estimated - deathdate_estimated - - - birthtime - birthtime - - + destination/merge + 100 + N + + + person
+ + mapping_source_person_uuid + uuid + = + + + + creator + mapping_destination_creator_user_id + Y + + @@ -777,8 +704,8 @@ - 544 - 144 + 816 + 176 Y diff --git a/src/main/resources/pdiresources/transformations/update-mapped-user-creators.ktr b/src/main/resources/pdiresources/transformations/update-mapped-user-creators.ktr new file mode 100644 index 0000000..9afec82 --- /dev/null +++ b/src/main/resources/pdiresources/transformations/update-mapped-user-creators.ktr @@ -0,0 +1,718 @@ + + + + update-mapped-user-creators + + + + Normal + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/29 02:16:38.102 + - + 2019/08/29 02:16:38.102 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + destination/merge + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + source + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_SOURCE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + Read creators from mappings + Get creator Source user uuid + Y + + + Get creator Source user uuid + Get creator Destination user id + Y + + + Get creator Destination user id + Update user creator to mapped in destination + Y + + + + Get creator Destination user id + DBJoin + + Y + + 1 + + none + + + destination/merge + 0 + SELECT user_id as mapping_destination_creator_user_id from users where uuid = ? + Y + N + + + source_creator_uuid + String + + + + + + + + + + + + 448 + 144 + Y + + + + Get creator Source user uuid + DBJoin + + Y + + 1 + + none + + + source + 0 + SELECT uuid as source_creator_uuid from users where user_id = ? + Y + N + + + mapping_source_creator_user_id + Integer + + + + + + + + + + + + 336 + 32 + Y + + + + Read creators from mappings + TableInput + + Y + + 1 + + none + + + destination/merge + SELECT uuid AS mapping_source_user_uuid, answer AS mapping_source_creator_user_id FROM migrator_merge_mappings WHERE entity = 'users' AND attribute = 'creator'; + 0 + + N + N + N + + + + + + + + + + 144 + 128 + Y + + + + Update user creator to mapped in destination + InsertUpdate + + Y + + 1 + + none + + + destination/merge + 100 + N + + +
users
+ + mapping_source_user_uuid + uuid + = + + + + creator + mapping_destination_creator_user_id + Y + + + + + + + + + + + + 832 + 96 + Y + + + + + + + N + + diff --git a/src/main/resources/pdiresources/transformations/update-mapped-user-persons.ktr b/src/main/resources/pdiresources/transformations/update-mapped-user-persons.ktr new file mode 100644 index 0000000..b762f3e --- /dev/null +++ b/src/main/resources/pdiresources/transformations/update-mapped-user-persons.ktr @@ -0,0 +1,718 @@ + + + + update-mapped-user-persons + + + + Normal + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/29 02:16:38.102 + - + 2019/08/29 02:16:38.102 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + destination/merge + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + source + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_SOURCE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + Read persons from mappings + Get Source User Person uuid + Y + + + Get Source User Person uuid + Get user person Destination person id + Y + + + Get user person Destination person id + Update user person to mapped in destination + Y + + + + Get user person Destination person id + DBJoin + + Y + + 1 + + none + + + destination/merge + 0 + SELECT person_id as mapping_destination_user_person_id from person where uuid = ? + Y + N + + + source_person_uuid + String + + + + + + + + + + + + 448 + 144 + Y + + + + Get Source User Person uuid + DBJoin + + Y + + 1 + + none + + + source + 0 + SELECT uuid as source_person_uuid from person where person_id = ? + Y + N + + + mapping_source_user_person_id + Integer + + + + + + + + + + + + 336 + 32 + Y + + + + Read persons from mappings + TableInput + + Y + + 1 + + none + + + destination/merge + SELECT uuid AS mapping_source_user_uuid, answer AS mapping_source_user_person_id FROM migrator_merge_mappings WHERE entity = 'users' AND attribute = 'person_id'; + 0 + + N + N + N + + + + + + + + + + 144 + 128 + Y + + + + Update user person to mapped in destination + InsertUpdate + + Y + + 1 + + none + + + destination/merge + 100 + N + + +
users
+ + mapping_source_user_uuid + uuid + = + + + + person_id + mapping_destination_user_person_id + Y + + + + + + + + + + + + 832 + 112 + Y + + + + + + + N + + From 3e5250dc935c9900be686cde1986db8663f8bae2 Mon Sep 17 00:00:00 2001 From: k-joseph Date: Thu, 29 Aug 2019 16:55:30 +0300 Subject: [PATCH 4/9] MEPTS-224: merged merging into merge transformation --- .../resources/pdiresources/jobs/merge.kjb | 344 ++++++++- .../resources/pdiresources/jobs/merging.kjb | 661 ------------------ 2 files changed, 336 insertions(+), 669 deletions(-) delete mode 100644 src/main/resources/pdiresources/jobs/merging.kjb diff --git a/src/main/resources/pdiresources/jobs/merge.kjb b/src/main/resources/pdiresources/jobs/merge.kjb index d68dc51..613510e 100644 --- a/src/main/resources/pdiresources/jobs/merge.kjb +++ b/src/main/resources/pdiresources/jobs/merge.kjb @@ -4,12 +4,11 @@ - 0 / - - 2019/08/26 13:13:15.739 + 2019/08/26 18:05:32.913 - - 2019/08/26 13:13:15.739 + 2019/08/26 18:05:32.913 @@ -305,8 +304,274 @@ N Y 0 + 32 + 32 + + + + create mappings table if missing + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/create-mappings.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 0 160 - 112 + 32 + + + + drop-mappings + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/drop-mappings.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 0 + 704 + 144 + + + + merge persons + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/merge-persons.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 0 + 352 + 32 + + + + update mapped person creators + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-person-creators.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 1 + 656 + 32 + + + + merge users + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/merge-users.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 496 + 32 + + + + update mapped user persons + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-user-persons.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 880 + 144 + + + + update mapped user creators + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-user-creators.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 880 + 32 @@ -316,7 +581,7 @@ filename - pdiresources/transformations/merge-encounter.ktr + ${Internal.Entry.Current.Directory}/../transformations/merge-encounter.ktr N N @@ -343,21 +608,84 @@ N Y 0 - 320 - 112 + 544 + 144 Start - merge encounter + create mappings table if missing 0 0 Y Y Y + + create mappings table if missing + merge persons + 0 + 0 + Y + Y + N + + + merge persons + merge users + 0 + 2 + Y + Y + N + + + merge users + update mapped person creators + 2 + 1 + Y + Y + N + + + update mapped person creators + update mapped user creators + 1 + 2 + Y + Y + N + + + update mapped user creators + update mapped user persons + 2 + 2 + Y + Y + N + + + update mapped user persons + drop-mappings + 2 + 0 + Y + Y + N + + + drop-mappings + merge encounter + 0 + 0 + Y + Y + N + diff --git a/src/main/resources/pdiresources/jobs/merging.kjb b/src/main/resources/pdiresources/jobs/merging.kjb deleted file mode 100644 index fd30e58..0000000 --- a/src/main/resources/pdiresources/jobs/merging.kjb +++ /dev/null @@ -1,661 +0,0 @@ - - - merging - - - - / - - - 2019/08/26 18:05:32.913 - - - 2019/08/26 18:05:32.913 - - - - - - - - - - - - - ID_JOB - Y - ID_JOB - - - CHANNEL_ID - Y - CHANNEL_ID - - - JOBNAME - Y - JOBNAME - - - STATUS - Y - STATUS - - - LINES_READ - Y - LINES_READ - - - LINES_WRITTEN - Y - LINES_WRITTEN - - - LINES_UPDATED - Y - LINES_UPDATED - - - LINES_INPUT - Y - LINES_INPUT - - - LINES_OUTPUT - Y - LINES_OUTPUT - - - LINES_REJECTED - Y - LINES_REJECTED - - - ERRORS - Y - ERRORS - - - STARTDATE - Y - STARTDATE - - - ENDDATE - Y - ENDDATE - - - LOGDATE - Y - LOGDATE - - - DEPDATE - Y - DEPDATE - - - REPLAYDATE - Y - REPLAYDATE - - - LOG_FIELD - Y - LOG_FIELD - - - EXECUTING_SERVER - N - EXECUTING_SERVER - - - EXECUTING_USER - N - EXECUTING_USER - - - START_JOB_ENTRY - N - START_JOB_ENTRY - - - CLIENT - N - CLIENT - - - - - -
- - - ID_BATCH - Y - ID_BATCH - - - CHANNEL_ID - Y - CHANNEL_ID - - - LOG_DATE - Y - LOG_DATE - - - JOBNAME - Y - TRANSNAME - - - JOBENTRYNAME - Y - STEPNAME - - - LINES_READ - Y - LINES_READ - - - LINES_WRITTEN - Y - LINES_WRITTEN - - - LINES_UPDATED - Y - LINES_UPDATED - - - LINES_INPUT - Y - LINES_INPUT - - - LINES_OUTPUT - Y - LINES_OUTPUT - - - LINES_REJECTED - Y - LINES_REJECTED - - - ERRORS - Y - ERRORS - - - RESULT - Y - RESULT - - - NR_RESULT_ROWS - Y - NR_RESULT_ROWS - - - NR_RESULT_FILES - Y - NR_RESULT_FILES - - - LOG_FIELD - N - LOG_FIELD - - - COPY_NR - N - COPY_NR - - - - - -
- - - ID_BATCH - Y - ID_BATCH - - - CHANNEL_ID - Y - CHANNEL_ID - - - LOG_DATE - Y - LOG_DATE - - - LOGGING_OBJECT_TYPE - Y - LOGGING_OBJECT_TYPE - - - OBJECT_NAME - Y - OBJECT_NAME - - - OBJECT_COPY - Y - OBJECT_COPY - - - REPOSITORY_DIRECTORY - Y - REPOSITORY_DIRECTORY - - - FILENAME - Y - FILENAME - - - OBJECT_ID - Y - OBJECT_ID - - - OBJECT_REVISION - Y - OBJECT_REVISION - - - PARENT_CHANNEL_ID - Y - PARENT_CHANNEL_ID - - - ROOT_CHANNEL_ID - Y - ROOT_CHANNEL_ID - - - N - - - - Start - - SPECIAL - - Y - N - N - 0 - 0 - 60 - 12 - 0 - 1 - 1 - N - Y - 0 - 32 - 32 - - - - create mappings table if missing - - TRANS - - filename - - ${Internal.Entry.Current.Directory}/../transformations/create-mappings.ktr - - N - N - N - N - N - N - - - N - N - Basic - N - - N - Y - N - N - N - Pentaho local - - Y - - N - Y - 0 - 160 - 32 - - - - drop-mappings - - TRANS - - filename - - ${Internal.Entry.Current.Directory}/../transformations/drop-mappings.ktr - - N - N - N - N - N - N - - - N - N - Basic - N - - N - Y - N - N - N - Pentaho local - - Y - - N - Y - 0 - 880 - 336 - - - - merge persons - - TRANS - - filename - - ${Internal.Entry.Current.Directory}/../transformations/merge-persons.ktr - - N - N - N - N - N - N - - - N - N - Basic - N - - N - Y - N - N - N - Pentaho local - - Y - - N - Y - 0 - 352 - 32 - - - - update mapped person creators - - TRANS - - filename - - ${Internal.Entry.Current.Directory}/../transformations/update-mapped-person-creators.ktr - - N - N - N - N - N - N - - - N - N - Basic - N - - N - Y - N - N - N - Pentaho local - - Y - - N - Y - 1 - 656 - 32 - - - - merge users - - TRANS - - filename - - ${Internal.Entry.Current.Directory}/../transformations/merge-users.ktr - - N - N - N - N - N - N - - - N - N - Basic - N - - N - Y - N - N - N - Pentaho local - - Y - - N - Y - 2 - 496 - 32 - - - - update mapped user persons - - TRANS - - filename - - ${Internal.Entry.Current.Directory}/../transformations/update-mapped-user-persons.ktr - - N - N - N - N - N - N - - - N - N - Basic - N - - N - Y - N - N - N - Pentaho local - - Y - - N - Y - 2 - 880 - 144 - - - - update mapped user creators - - TRANS - - filename - - ${Internal.Entry.Current.Directory}/../transformations/update-mapped-user-creators.ktr - - N - N - N - N - N - N - - - N - N - Basic - N - - N - Y - N - N - N - Pentaho local - - Y - - N - Y - 2 - 880 - 32 - - - - - - Start - create mappings table if missing - 0 - 0 - Y - Y - Y - - - create mappings table if missing - merge persons - 0 - 0 - Y - Y - N - - - merge persons - merge users - 0 - 2 - Y - Y - N - - - merge users - update mapped person creators - 2 - 1 - Y - Y - N - - - update mapped person creators - update mapped user creators - 1 - 2 - Y - Y - N - - - update mapped user creators - update mapped user persons - 2 - 2 - Y - Y - N - - - update mapped user persons - drop-mappings - 2 - 0 - Y - Y - N - - - - - - - METASTORE.pentaho - - Default Run Configuration - {"namespace":"pentaho","id":"Default Run Configuration","name":"Default Run Configuration","description":"Defines a default run configuration","metaStoreName":null} - - - - {"_":"Embedded MetaStore Elements","namespace":"pentaho","type":"Default Run Configuration"} - - Pentaho local - {"children":[{"children":[],"id":"server","value":null},{"children":[],"id":"clustered","value":"N"},{"children":[],"id":"name","value":"Pentaho local"},{"children":[],"id":"description","value":null},{"children":[],"id":"pentaho","value":"N"},{"children":[],"id":"readOnly","value":"Y"},{"children":[],"id":"sendResources","value":"N"},{"children":[],"id":"logRemoteExecutionLocally","value":"N"},{"children":[],"id":"remote","value":"N"},{"children":[],"id":"local","value":"Y"},{"children":[],"id":"showTransformations","value":"N"}],"id":"Pentaho local","value":null,"name":"Pentaho local","owner":null,"ownerPermissionsList":[]} - - - - From da250146e61a220587db8353358a20f9a7ea4d0a Mon Sep 17 00:00:00 2001 From: k-joseph Date: Thu, 29 Aug 2019 16:59:27 +0300 Subject: [PATCH 5/9] MEPTS-224: added merge users into pdiresources and made some few fixes --- src/main/java/com/openmrs/migrator/Migrator.java | 3 ++- src/main/resources/pdiresources/jobs/merge.kjb | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/openmrs/migrator/Migrator.java b/src/main/java/com/openmrs/migrator/Migrator.java index 7a109ce..7a8dadb 100644 --- a/src/main/java/com/openmrs/migrator/Migrator.java +++ b/src/main/java/com/openmrs/migrator/Migrator.java @@ -145,7 +145,8 @@ private void executeSetupCommand() throws IOException, SQLException, SettingsExc pdiFiles.add( SettingsService.PDI_RESOURCES_DIR + "/transformations/validate-visit-attribute-types.ktr"); pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/validate-visit-types.ktr"); - pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-person.ktr"); + pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-persons.ktr"); + pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-users.ktr"); pdiFiles.add(SettingsService.SETTINGS_PROPERTIES); pdiFiles.add(SettingsService.PDI_PLUGINS_DIR + "/pdi-core-plugins-impl-8.2.0.7-719.jar"); diff --git a/src/main/resources/pdiresources/jobs/merge.kjb b/src/main/resources/pdiresources/jobs/merge.kjb index 613510e..762b27c 100644 --- a/src/main/resources/pdiresources/jobs/merge.kjb +++ b/src/main/resources/pdiresources/jobs/merge.kjb @@ -6,7 +6,7 @@ / - - 2019/08/26 18:05:32.913 + 2019/08/26 13:13:15.739 - 2019/08/26 18:05:32.913 From 977923a96e8e8422ff1bd930961495a07a047643 Mon Sep 17 00:00:00 2001 From: k-joseph Date: Tue, 3 Sep 2019 11:31:36 +0300 Subject: [PATCH 6/9] MEPTS-224: Worked on concept_datatype, concept_class, concept, person, user merge transformations --- .../java/com/openmrs/migrator/Migrator.java | 16 +- .../resources/pdiresources/jobs/merge.kjb | 694 +++++- .../transformations/merge-concept-classes.ktr | 1183 +++++++++++ .../merge-concept-datatypes.ktr | 1187 +++++++++++ .../transformations/merge-concepts.ktr | 1858 +++++++++++++++++ .../transformations/merge-persons.ktr | 668 +++++- .../transformations/merge-users.ktr | 716 +++++-- .../update-mapped-foreign-cause-of-death.ktr | 718 +++++++ .../update-mapped-foreign-changed-by.ktr | 718 +++++++ .../update-mapped-foreign-concept-class.ktr | 718 +++++++ ...update-mapped-foreign-concept-datatype.ktr | 718 +++++++ .../update-mapped-foreign-creators.ktr | 718 +++++++ .../update-mapped-foreign-person-id.ktr | 718 +++++++ .../update-mapped-foreign-retired-by.ktr | 718 +++++++ .../update-mapped-foreign-voided-by.ktr | 718 +++++++ .../update-mapped-user-creators.ktr | 4 +- 16 files changed, 11824 insertions(+), 246 deletions(-) create mode 100644 src/main/resources/pdiresources/transformations/merge-concept-classes.ktr create mode 100644 src/main/resources/pdiresources/transformations/merge-concept-datatypes.ktr create mode 100644 src/main/resources/pdiresources/transformations/merge-concepts.ktr create mode 100644 src/main/resources/pdiresources/transformations/update-mapped-foreign-cause-of-death.ktr create mode 100644 src/main/resources/pdiresources/transformations/update-mapped-foreign-changed-by.ktr create mode 100644 src/main/resources/pdiresources/transformations/update-mapped-foreign-concept-class.ktr create mode 100644 src/main/resources/pdiresources/transformations/update-mapped-foreign-concept-datatype.ktr create mode 100644 src/main/resources/pdiresources/transformations/update-mapped-foreign-creators.ktr create mode 100644 src/main/resources/pdiresources/transformations/update-mapped-foreign-person-id.ktr create mode 100644 src/main/resources/pdiresources/transformations/update-mapped-foreign-retired-by.ktr create mode 100644 src/main/resources/pdiresources/transformations/update-mapped-foreign-voided-by.ktr diff --git a/src/main/java/com/openmrs/migrator/Migrator.java b/src/main/java/com/openmrs/migrator/Migrator.java index 7a8dadb..13f3b30 100644 --- a/src/main/java/com/openmrs/migrator/Migrator.java +++ b/src/main/java/com/openmrs/migrator/Migrator.java @@ -8,6 +8,13 @@ import com.openmrs.migrator.core.services.impl.MySQLProps; import com.openmrs.migrator.core.utilities.ConsoleUtils; import com.openmrs.migrator.core.utilities.FileIOUtilities; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + import java.io.Console; import java.io.File; import java.io.IOException; @@ -22,12 +29,6 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.Callable; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import picocli.CommandLine; -import picocli.CommandLine.Command; -import picocli.CommandLine.Option; @Command(name = "migrator", mixinStandardHelpOptions = true, helpCommand = true) public class Migrator implements Callable> { @@ -147,6 +148,9 @@ private void executeSetupCommand() throws IOException, SQLException, SettingsExc pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/validate-visit-types.ktr"); pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-persons.ktr"); pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-users.ktr"); + pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-concept-datatypes.ktr"); + pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-concept-classes.ktr"); + pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-concepts.ktr"); pdiFiles.add(SettingsService.SETTINGS_PROPERTIES); pdiFiles.add(SettingsService.PDI_PLUGINS_DIR + "/pdi-core-plugins-impl-8.2.0.7-719.jar"); diff --git a/src/main/resources/pdiresources/jobs/merge.kjb b/src/main/resources/pdiresources/jobs/merge.kjb index 762b27c..3606d68 100644 --- a/src/main/resources/pdiresources/jobs/merge.kjb +++ b/src/main/resources/pdiresources/jobs/merge.kjb @@ -304,8 +304,8 @@ N Y 0 - 32 - 32 + 16 + 16 @@ -342,7 +342,7 @@ N Y 0 - 160 + 64 32 @@ -380,8 +380,8 @@ N Y 0 - 704 - 144 + 560 + 304 @@ -418,18 +418,18 @@ N Y 0 - 352 - 32 + 336 + 96 - update mapped person creators + person changed by TRANS filename - ${Internal.Entry.Current.Directory}/../transformations/update-mapped-person-creators.ktr + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-changed-by.ktr N N @@ -456,8 +456,8 @@ N Y 1 - 656 - 32 + 784 + 16 @@ -494,18 +494,18 @@ N Y 2 - 496 - 32 + 144 + 16 - update mapped user persons + user creators TRANS filename - ${Internal.Entry.Current.Directory}/../transformations/update-mapped-user-persons.ktr + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-creators.ktr N N @@ -532,18 +532,110 @@ N Y 2 - 880 + 928 + 304 + + + + merge encounter + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/merge-encounter.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 0 + 32 + 304 + + + + Set person entities update variables + + SET_VARIABLES + + Y + + CURRENT_JOB + + + ENTITY + person + JVM + + + ENTITY_ID + person_id + JVM + + + N + Y + 0 + 416 144 - update mapped user creators + Set users entities update variables + + SET_VARIABLES + + Y + + CURRENT_JOB + + + ENTITY + users + JVM + + + ENTITY_ID + user_id + JVM + + + N + Y + 0 + 928 + 112 + + + + person voided by TRANS filename - ${Internal.Entry.Current.Directory}/../transformations/update-mapped-user-creators.ktr + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-voided-by.ktr N N @@ -570,18 +662,18 @@ N Y 2 - 880 - 32 + 688 + 48 - merge encounter + person creators TRANS filename - ${Internal.Entry.Current.Directory}/../transformations/merge-encounter.ktr + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-creators.ktr N N @@ -607,11 +699,407 @@ N Y - 0 - 544 + 3 + 560 + 80 + + + + user retired by + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-retired-by.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 816 + 288 + + + + user changed by + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-changed-by.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 1 + 688 + 256 + + + + user person id + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-person-id.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 1 + 576 + 208 + + + + merge concepts + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/merge-concepts.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 192 + 96 + + + + merge concept datatypes + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/merge-concept-datatypes.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 192 + 176 + + + + merge concept classes + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/merge-concept-classes.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 64 144 + + Set concept class entities update variables + + SET_VARIABLES + + Y + + CURRENT_JOB + + + ENTITY + concept_class + JVM + + + ENTITY_ID + concept_class_id + JVM + + + N + Y + 0 + 320 + 0 + + + + concept datatype + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-concept-datatype.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 608 + 0 + + + + concept class + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-concept-class.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 1 + 400 + 80 + + + + Set concept datatype entities update variables + + SET_VARIABLES + + Y + + CURRENT_JOB + + + ENTITY + concept_datatype + JVM + + + ENTITY_ID + concept_datatype_id + JVM + + + N + Y + 0 + 512 + 16 + + + + person cause_of_death + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-cause-of-death.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 1 + 944 + 16 + + @@ -624,8 +1112,8 @@ Y - create mappings table if missing - merge persons + drop-mappings + merge encounter 0 0 Y @@ -633,17 +1121,44 @@ N - merge persons - merge users + Set users entities update variables + user creators 0 2 Y Y + Y + + + person voided by + person changed by + 2 + 1 + Y + Y + Y + + + person creators + person voided by + 3 + 2 + Y + Y + Y + + + user creators + user retired by + 2 + 2 + Y + Y N - merge users - update mapped person creators + user retired by + user changed by 2 1 Y @@ -651,41 +1166,140 @@ N - update mapped person creators - update mapped user creators + user changed by + user person id 1 - 2 + 1 + Y + Y + N + + + user person id + drop-mappings + 1 + 0 Y Y N - update mapped user creators - update mapped user persons + merge concept classes + merge concept datatypes 2 2 Y Y + Y + + + merge concept datatypes + merge concepts + 2 + 2 + Y + Y + Y + + + create mappings table if missing + merge concept classes + 0 + 2 + Y + Y N - update mapped user persons - drop-mappings + merge concepts + merge persons 2 0 Y Y - N + Y - drop-mappings - merge encounter + merge persons + merge users 0 + 2 + Y + Y + Y + + + merge users + Set concept class entities update variables + 2 + 0 + Y + Y + Y + + + Set concept class entities update variables + concept class + 0 + 1 + Y + Y + Y + + + concept datatype + Set person entities update variables + 2 + 0 + Y + Y + Y + + + concept class + Set concept datatype entities update variables + 1 0 Y Y + Y + + + Set concept datatype entities update variables + concept datatype + 0 + 2 + Y + Y + Y + + + Set person entities update variables + person creators + 0 + 3 + Y + Y N + + person changed by + person cause_of_death + 1 + 1 + Y + Y + Y + + + person cause_of_death + Set users entities update variables + 1 + 0 + Y + Y + Y + diff --git a/src/main/resources/pdiresources/transformations/merge-concept-classes.ktr b/src/main/resources/pdiresources/transformations/merge-concept-classes.ktr new file mode 100644 index 0000000..b5ad914 --- /dev/null +++ b/src/main/resources/pdiresources/transformations/merge-concept-classes.ktr @@ -0,0 +1,1183 @@ + + + + merge-concept-classes + + + + Normal + 0 + / + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/06 09:23:50.203 + - + 2019/08/06 09:23:50.203 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + OpenMRS destination + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + OpenMRS source + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_SOURCE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + Load concept classes from Source + Concept classes In Both Source & Destination + Y + + + Concept classes In Both Source & Destination + Concept class in Source Not Destination + Y + + + insert merge mapping for concept class + Creator Exists on Destination + Y + + + Creator Exists on Destination + insert merge mapping for missing creator + Y + + + Daemon user + insert merge mapping for concept class + Y + + + insert merge mapping for missing creator + Set creator to daemon user + Y + + + Concept class in Source Not Destination + Concept class Creator uuid From Destination + Y + + + Concept class Creator uuid From Destination + User retired by uuid From Destination + Y + + + User retired by uuid From Destination + Daemon user + Y + + + Creator Exists on Destination + Retired by Exists on Destination + Y + + + Retired by Exists on Destination + Save Missing concept class into Destination + Y + + + Set creator to daemon user + Retired by Exists on Destination + Y + + + Retired by Exists on Destination + insert merge mapping for missing retired by + Y + + + insert merge mapping for missing retired by + Set retired by to daemon user + Y + + + Set retired by to daemon user + Save Missing concept class into Destination + Y + + + + Concept class Creator uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_creator_uuid from users where user_id = ? + Y + N + + + creator + Integer + + + + + + + + + + + + 208 + 240 + Y + + + + Concept class in Source Not Destination + FilterRows + + Y + + 1 + + none + + + + + + + N + destination_concept_class_id + IS NULL + + + + + + + + + + + + + 336 + 368 + Y + + + + Concept classes In Both Source & Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select concept_class_id as destination_concept_class_id from concept_class where uuid = ? + Y + N + + + uuid + String + + + + + + + + + + + + 144 + 336 + Y + + + + Creator Exists on Destination + FilterRows + + Y + + 1 + + none + + + Retired by Exists on Destination + insert merge mapping for missing creator + + + N + destination_creator_uuid + IS NOT NULL + + + + + + + + + + + + + 352 + 96 + Y + + + + Daemon user + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select user_id as daemon_user_id from users where system_id = 'daemon'; + Y + N + + + + + + + + + + + + 256 + 16 + Y + + + + Load concept classes from Source + TableInput + + N + + 1 + + none + + + OpenMRS source + SELECT * FROM concept_class; + 0 + + N + N + N + + + + + + + + + + 112 + 32 + Y + + + + Retired by Exists on Destination + FilterRows + + Y + + 1 + + none + + + Save Missing concept class into Destination + insert merge mapping for missing retired by + + + N + destination_retired_by_uuid + IS NOT NULL + + + + + + + + + + + + + 720 + 16 + Y + + + + Save Missing concept class into Destination + TableOutput + + Y + + 1 + + none + + + OpenMRS destination + +
concept_class
+ 1000 + N + N + Y + Y + N + + N + Y + N + + Y + N + + + + name + name + + + description + description + + + creator + creator + + + date_created + date_created + + + retired + retired + + + retired_by + retired_by + + + date_retired + date_retired + + + retire_reason + retire_reason + + + uuid + uuid + + + + + + + + + + + + 912 + 112 + Y + + + + Set creator to daemon user + SetValueField + + Y + + 1 + + none + + + + + creator + daemon_user_id + + + + + + + + + + + + 608 + 144 + Y + + + + Set retired by to daemon user + SetValueField + + Y + + 1 + + none + + + + + retired_by + daemon_user_id + + + + + + + + + + + + 768 + 176 + Y + + + + User retired by uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_retired_by_uuid from users where user_id = ? + Y + N + + + retired_by + Integer + + + + + + + + + + + + 224 + 160 + Y + + + + insert merge mapping for concept class + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('concept_class', 'concept_class_id', ?, ?); + Y + + + + + + + concept_class_id + + + uuid + + + + + + + + + + + + 464 + 16 + Y + + + + insert merge mapping for missing creator + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('concept_class', 'creator', ?, ?); + Y + + + + + + + creator + + + uuid + + + + + + + + + + + + 416 + 288 + Y + + + + insert merge mapping for missing retired by + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('concept_class', 'retired_by', ?, ?); + Y + + + + + + + retired_by + + + uuid + + + + + + + + + + + + 592 + 240 + Y + + + + + + + N + + diff --git a/src/main/resources/pdiresources/transformations/merge-concept-datatypes.ktr b/src/main/resources/pdiresources/transformations/merge-concept-datatypes.ktr new file mode 100644 index 0000000..3c2a890 --- /dev/null +++ b/src/main/resources/pdiresources/transformations/merge-concept-datatypes.ktr @@ -0,0 +1,1187 @@ + + + + merge-concept-datatypes + + + + Normal + 0 + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/06 09:23:50.203 + - + 2019/08/06 09:23:50.203 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + OpenMRS destination + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + OpenMRS source + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_SOURCE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + Load concept datatypes from Source + Concept datatypes In Both Source & Destination + Y + + + Concept datatypes In Both Source & Destination + Concept datatype in Source Not Destination + Y + + + insert merge mapping for concept datatype + Creator Exists on Destination + Y + + + Creator Exists on Destination + insert merge mapping for missing creator + Y + + + Daemon user + insert merge mapping for concept datatype + Y + + + insert merge mapping for missing creator + Set creator to daemon user + Y + + + Concept datatype in Source Not Destination + Concept datatype Creator uuid From Destination + Y + + + Concept datatype Creator uuid From Destination + User retired by uuid From Destination + Y + + + User retired by uuid From Destination + Daemon user + Y + + + Creator Exists on Destination + Retired by Exists on Destination + Y + + + Retired by Exists on Destination + Save Missing concept datatype into Destination + Y + + + Set creator to daemon user + Retired by Exists on Destination + Y + + + Retired by Exists on Destination + insert merge mapping for missing retired by + Y + + + insert merge mapping for missing retired by + Set retired by to daemon user + Y + + + Set retired by to daemon user + Save Missing concept datatype into Destination + Y + + + + Creator Exists on Destination + FilterRows + + Y + + 1 + + none + + + Retired by Exists on Destination + insert merge mapping for missing creator + + + N + destination_creator_uuid + IS NOT NULL + + + + + + + + + + + + + 352 + 96 + Y + + + + Daemon user + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select user_id as daemon_user_id from users where system_id = 'daemon'; + Y + N + + + + + + + + + + + + 256 + 16 + Y + + + + Load concept datatypes from Source + TableInput + + N + + 1 + + none + + + OpenMRS source + SELECT * FROM concept_datatype + 0 + + N + N + N + + + + + + + + + + 112 + 32 + Y + + + + Concept datatype Creator uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_creator_uuid from users where user_id = ? + Y + N + + + creator + Integer + + + + + + + + + + + + 208 + 240 + Y + + + + Concept datatypes In Both Source & Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select concept_datatype_id as destination_concept_datatype_id from concept_datatype where uuid = ? + Y + N + + + uuid + String + + + + + + + + + + + + 144 + 336 + Y + + + + Concept datatype in Source Not Destination + FilterRows + + Y + + 1 + + none + + + + + + + N + destination_concept_datatype_id + IS NULL + + + + + + + + + + + + + 336 + 368 + Y + + + + Save Missing concept datatype into Destination + TableOutput + + Y + + 1 + + none + + + OpenMRS destination + +
concept_datatype
+ 1000 + N + N + Y + Y + N + + N + Y + N + + Y + N + + + + name + name + + + hl7_abbreviation + hl7_abbreviation + + + description + description + + + creator + creator + + + date_created + date_created + + + retired + retired + + + retired_by + retired_by + + + date_retired + date_retired + + + retire_reason + retire_reason + + + uuid + uuid + + + + + + + + + + + + 944 + 48 + Y + + + + Set creator to daemon user + SetValueField + + Y + + 1 + + none + + + + + creator + daemon_user_id + + + + + + + + + + + + 608 + 144 + Y + + + + insert merge mapping for missing creator + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('concept_datatype', 'creator', ?, ?); + Y + + + + + + + creator + + + uuid + + + + + + + + + + + + 416 + 288 + Y + + + + insert merge mapping for concept datatype + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('concept_datatype', 'concept_datatype_id', ?, ?); + Y + + + + + + + concept_datatype_id + + + uuid + + + + + + + + + + + + 464 + 16 + Y + + + + User retired by uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_retired_by_uuid from users where user_id = ? + Y + N + + + retired_by + Integer + + + + + + + + + + + + 224 + 160 + Y + + + + Retired by Exists on Destination + FilterRows + + Y + + 1 + + none + + + Save Missing concept datatype into Destination + insert merge mapping for missing retired by + + + N + destination_retired_by_uuid + IS NOT NULL + + + + + + + + + + + + + 720 + 16 + Y + + + + insert merge mapping for missing retired by + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('concept_datatype', 'retired_by', ?, ?); + Y + + + + + + + retired_by + + + uuid + + + + + + + + + + + + 656 + 240 + Y + + + + Set retired by to daemon user + SetValueField + + Y + + 1 + + none + + + + + retired_by + daemon_user_id + + + + + + + + + + + + 809 + 174 + Y + + + + + + + N + + diff --git a/src/main/resources/pdiresources/transformations/merge-concepts.ktr b/src/main/resources/pdiresources/transformations/merge-concepts.ktr new file mode 100644 index 0000000..0e695c5 --- /dev/null +++ b/src/main/resources/pdiresources/transformations/merge-concepts.ktr @@ -0,0 +1,1858 @@ + + + + merge-concepts + + + + Normal + 0 + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/06 09:23:50.203 + - + 2019/08/06 09:23:50.203 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + OpenMRS destination + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + OpenMRS source + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_SOURCE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + Creator on Destination + insert merge mapping for missing creator + Y + + + Load Concepts from Source + Concepts In Both Source & Destination + Y + + + Concepts In Both Source & Destination + Concepts in Source Not Destination + Y + + + Concepts in Source Not Destination + Concept Creator uuid From Destination + Y + + + insert merge mapping for missing creator + Set creator to daemon user + Y + + + insert merge mapping for concept + Creator on Destination + Y + + + Concept Changed by uuid From Destination + Concept retired by uuid From Destination + Y + + + Changed by on Destination + Retired By on Destination + Y + + + Changed by on Destination + insert merge mapping for missing changed by + Y + + + insert merge mapping for missing changed by + Set changed by to daemon id + Y + + + Set changed by to daemon id + Retired By on Destination + Y + + + Retired By on Destination + insert merge mapping for missing retired by + Y + + + insert merge mapping for missing retired by + Set retired by to daemon id + Y + + + Concept Creator uuid From Destination + Concept Changed by uuid From Destination + Y + + + Concept class uuid From Destination + Concept datatype uuid From Destination + Y + + + Concept datatype uuid From Destination + Daemon user/person + Y + + + Set creator to daemon user + Changed by on Destination + Y + + + Creator on Destination + Changed by on Destination + Y + + + Set retired by to daemon id + Class on Destination + Y + + + Retired By on Destination + Class on Destination + Y + + + Daemon user/person + Misc Concept class + Y + + + Misc Concept class + Misc Concept datatype + Y + + + Misc Concept datatype + insert merge mapping for concept + Y + + + Class on Destination + insert merge mapping for missing class + Y + + + insert merge mapping for missing class + Set class to misc class + Y + + + Set class to misc class + Datatype on Destination + Y + + + Class on Destination + Datatype on Destination + Y + + + Datatype on Destination + insert merge mapping for missing datatype + Y + + + insert merge mapping for missing datatype + Set class to na datatype + Y + + + Set class to na datatype + Save Missing concept into Destination + Y + + + Datatype on Destination + Save Missing concept into Destination + Y + + + Concept retired by uuid From Destination + Concept class uuid From Destination + Y + + + + Changed by on Destination + FilterRows + + Y + + 1 + + none + + + insert merge mapping for missing changed by + Retired By on Destination + + + N + + + N + destination_changed_by + IS NULL + + + + N + AND + changed_by + IS NOT NULL + + + + + + + + + + + + + + + 624 + 96 + Y + + + + Class on Destination + FilterRows + + Y + + 1 + + none + + + insert merge mapping for missing class + Datatype on Destination + + + N + + + N + destination_concept_class_uuid + IS NULL + + + + N + AND + class_id + IS NOT NULL + + + + + + + + + + + + + + + 832 + 128 + Y + + + + Concept Changed by uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT DISTINCT uuid as destination_changed_by from users where user_id = ? + Y + N + + + changed_by + Integer + + + + + + + + + + + + 240 + 288 + Y + + + + Concept Creator uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT DISTINCT uuid as destination_creator_uuid from users where user_id = ? + Y + N + + + creator + Integer + + + + + + + + + + + + 240 + 352 + Y + + + + Concept class uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT DISTINCT uuid as destination_concept_class_uuid from concept_class where concept_class_id = ? + Y + N + + + class_id + Integer + + + + + + + + + + + + 256 + 160 + Y + + + + Concept datatype uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_datatype_uuid from concept_datatype where concept_datatype_id = ? + Y + N + + + datatype_id + Integer + + + + + + + + + + + + 208 + 96 + Y + + + + Concept retired by uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT DISTINCT uuid as destination_retired_by from users where retired_by = ? + Y + N + + + retired_by + Integer + + + + + + + + + + + + 320 + 224 + Y + + + + Concepts In Both Source & Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select concept_id as destination_concept_id from concept where uuid = ? + Y + N + + + uuid + String + + + + + + + + + + + + 128 + 544 + Y + + + + Concepts in Source Not Destination + FilterRows + + Y + + 1 + + none + + + Concept Creator uuid From Destination + + + + N + destination_concept_id + IS NULL + + + + + + + + + + + + + 208 + 432 + Y + + + + Creator on Destination + FilterRows + + Y + + 1 + + none + + + insert merge mapping for missing creator + Changed by on Destination + + + N + + + N + destination_creator_uuid + IS NULL + + + + N + AND + creator + IS NOT NULL + + + + + + + + + + + + + + + 320 + 480 + Y + + + + Daemon user/person + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select user_id AS daemon_user_id, person_id AS daemon_person_id from users where system_id = 'daemon'; + Y + N + + + + + + + + + + + + 224 + 32 + Y + + + + Datatype on Destination + FilterRows + + Y + + 1 + + none + + + insert merge mapping for missing datatype + Save Missing concept into Destination + + + N + + + N + destination_datatype_uuid + IS NULL + + + + N + AND + datatype_id + IS NOT NULL + + + + + + + + + + + + + + + 816 + 320 + Y + + + + Load Concepts from Source + TableInput + + N + + 1 + + none + + + OpenMRS source + SELECT * FROM concept + 0 + + N + N + N + + + + + + + + + + 96 + 16 + Y + + + + Misc Concept class + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select concept_class_id AS misc_concept_class_id from concept_class where `name` = 'Misc'; + Y + N + + + + + + + + + + + + 304 + 0 + Y + + + + Misc Concept datatype + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select concept_datatype_id AS na_concept_datatype_id from concept_datatype where `name` = 'N/A'; + Y + N + + + + + + + + + + + + 368 + 80 + Y + + + + Retired By on Destination + FilterRows + + Y + + 1 + + none + + + insert merge mapping for missing retired by + Class on Destination + + + N + + + N + destination_retired_by + IS NULL + + + + N + AND + retired_by + IS NOT NULL + + + + + + + + + + + + + + + 688 + 192 + Y + + + + Save Missing concept into Destination + TableOutput + + Y + + 1 + + none + + + OpenMRS destination + +
concept
+ 1000 + N + N + Y + Y + N + + N + Y + N + + Y + N + + + + retired + retired + + + short_name + short_name + + + description + description + + + form_text + form_text + + + datatype_id + datatype_id + + + class_id + class_id + + + is_set + is_set + + + creator + creator + + + date_created + date_created + + + version + version + + + changed_by + changed_by + + + date_changed + date_changed + + + retired_by + retired_by + + + date_retired + date_retired + + + retire_reason + retire_reason + + + uuid + uuid + + + + + + + + + + + + 960 + 384 + Y + + + + Set changed by to daemon id + SetValueField + + Y + + 1 + + none + + + + + changed_by + daemon_user_id + + + + + + + + + + + + 560 + 208 + Y + + + + Set class to misc class + SetValueField + + Y + + 1 + + none + + + + + class_id + misc_concept_class_id + + + + + + + + + + + + 544 + 384 + Y + + + + Set class to na datatype + SetValueField + + Y + + 1 + + none + + + + + datatype_id + na_concept_datatype_id + + + + + + + + + + + + 562 + 510 + Y + + + + Set creator to daemon user + SetValueField + + Y + + 1 + + none + + + + + creator + daemon_user_id + + + + + + + + + + + + 624 + 16 + Y + + + + Set retired by to daemon id + SetValueField + + Y + + 1 + + none + + + + + retired_by + daemon_user_id + + + + + + + + + + + + 432 + 416 + Y + + + + insert merge mapping for concept + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('concept', 'concept_id', ?, ?); + Y + + + + + + + concept_id + + + uuid + + + + + + + + + + + + 496 + 32 + Y + + + + insert merge mapping for missing changed by + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('users', 'changed_by', ?, ?); + Y + + + + + + + changed_by + + + uuid + + + + + + + + + + + + 800 + 48 + Y + + + + insert merge mapping for missing class + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('concept', 'class_id', ?, ?); + Y + + + + + + + class_id + + + uuid + + + + + + + + + + + + 752 + 256 + Y + + + + insert merge mapping for missing creator + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('concept', 'creator', ?, ?); + Y + + + + + + + creator + + + uuid + + + + + + + + + + + + 496 + 128 + Y + + + + insert merge mapping for missing datatype + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('concept', 'datatype_id', ?, ?); + Y + + + + + + + datatype_id + + + uuid + + + + + + + + + + + + 688 + 416 + Y + + + + insert merge mapping for missing retired by + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('users', 'retired_by', ?, ?); + Y + + + + + + + retired_by + + + uuid + + + + + + + + + + + + 512 + 320 + Y + + + + + + + N + + diff --git a/src/main/resources/pdiresources/transformations/merge-persons.ktr b/src/main/resources/pdiresources/transformations/merge-persons.ktr index c21c1a9..82a1754 100644 --- a/src/main/resources/pdiresources/transformations/merge-persons.ktr +++ b/src/main/resources/pdiresources/transformations/merge-persons.ktr @@ -557,11 +557,6 @@ Persons in Source Not Destination Y - - Creator Exists on Destination - Save Missing persons into Destination - Y - insert merge mapping for persons Creator Exists on Destination @@ -572,32 +567,211 @@ insert merge mapping for missing creator Y + + insert merge mapping for missing creator + Set creator to daemon user + Y + + + Persons in Source Not Destination + Person Creator uuid From Destination + Y + Person Creator uuid From Destination + Person voided by uuid From Destination + Y + + + Person Changed by uuid From Destination Daemon user Y - Daemon user - insert merge mapping for persons + Changed by Exists on Destination + Voided By Exists on Destination Y - insert merge mapping for missing creator - Set creator to daemon user + Changed by Exists on Destination + insert merge mapping for missing changed by + Y + + + Voided By Exists on Destination + insert merge mapping for missing voided by + Y + + + Set changed by to daemon id + Voided By Exists on Destination + Y + + + insert merge mapping for missing changed by + Set changed by to daemon id + Y + + + insert merge mapping for missing voided by + Set voided by to daemon id Y Set creator to daemon user + Changed by Exists on Destination + Y + + + Creator Exists on Destination + Changed by Exists on Destination + Y + + + Person voided by uuid From Destination + Person Cause of death uuid From Destination + Y + + + Person Cause of death uuid From Destination + Person Changed by uuid From Destination + Y + + + Voided By Exists on Destination + Cause of death Exists on Destination + Y + + + Set voided by to daemon id + Cause of death Exists on Destination + Y + + + Cause of death Exists on Destination Save Missing persons into Destination Y - Persons in Source Not Destination - Person Creator uuid From Destination + Cause of death Exists on Destination + insert merge mapping for missing cause of death + Y + + + insert merge mapping for missing cause of death + Set cause of death to none concept id + Y + + + Set cause of death to none concept id + Save Missing persons into Destination + Y + + + Daemon user + Daemon concept + Y + + + Daemon concept + insert merge mapping for persons Y + + Cause of death Exists on Destination + FilterRows + + Y + + 1 + + none + + + insert merge mapping for missing cause of death + Save Missing persons into Destination + + + N + + + N + destination_cause_of_death_uuid + IS NULL + + + + N + AND + cause_of_death + IS NOT NULL + + + + + + + + + + + + + + + 944 + 128 + Y + + + + Changed by Exists on Destination + FilterRows + + Y + + 1 + + none + + + insert merge mapping for missing changed by + Voided By Exists on Destination + + + N + + + N + destination_changed_by + IS NULL + + + + N + AND + changed_by + IS NOT NULL + + + + + + + + + + + + + + + 576 + 80 + Y + + Creator Exists on Destination FilterRows @@ -609,14 +783,26 @@ none - Save Missing persons into Destination - insert merge mapping for missing creator + insert merge mapping for missing creator + Changed by Exists on Destination N - destination_creator_uuid - IS NOT NULL - + + + N + destination_creator_uuid + IS NULL + + + + N + AND + creator + IS NOT NULL + + + @@ -628,8 +814,8 @@ - 336 - 416 + 400 + 96 Y @@ -660,8 +846,8 @@ - 208 - 32 + 304 + 16 Y @@ -692,11 +878,83 @@ - 80 + 64 16 Y + + Person Cause of death uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_cause_of_death_uuid from concept where concept_id = ? + Y + N + + + cause_of_death + Integer + + + + + + + + + + + + 208 + 48 + Y + + + + Person Changed by uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_changed_by from users where user_id = ? + Y + N + + + changed_by + Integer + + + + + + + + + + + + 192 + 0 + Y + + Person Creator uuid From Destination DBJoin @@ -716,7 +974,7 @@ creator - String + Integer @@ -729,7 +987,43 @@ 240 - 160 + 176 + Y + + + + Person voided by uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_voided_by from users where user_id = ? + Y + N + + + voided_by + Integer + + + + + + + + + + + + 240 + 112 Y @@ -765,7 +1059,7 @@ 112 - 544 + 304 Y @@ -780,7 +1074,7 @@ none - Person Creator uuid From Source + @@ -799,8 +1093,8 @@ - 224 - 288 + 144 + 240 Y @@ -911,8 +1205,70 @@ - 896 - 16 + 720 + 320 + Y + + + + Set cause of death to none concept id + SetValueField + + Y + + 1 + + none + + + + + cause_of_death + daemon_concept_id + + + + + + + + + + + + 935 + 314 + Y + + + + Set changed by to daemon id + SetValueField + + Y + + 1 + + none + + + + + changed_by + daemon_user_id + + + + + + + + + + + + 832 + 64 Y @@ -942,8 +1298,174 @@ - 544 - 16 + 528 + 160 + Y + + + + Set voided by to daemon id + SetValueField + + Y + + 1 + + none + + + + + voided_by + daemon_user_id + + + + + + + + + + + + 800 + 192 + Y + + + + Voided By Exists on Destination + FilterRows + + Y + + 1 + + none + + + insert merge mapping for missing voided by + Cause of death Exists on Destination + + + N + + + N + destination_voided_by + IS NULL + + + + N + AND + voided_by + IS NOT NULL + + + + + + + + + + + + + + + 704 + 112 + Y + + + + insert merge mapping for missing cause of death + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('person', 'cause_of_death', ?, ?); + Y + + + + + + + cause_of_death + + + uuid + + + + + + + + + + + + 944 + 224 + Y + + + + insert merge mapping for missing changed by + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('person', 'changed_by', ?, ?); + Y + + + + + + + changed_by + + + uuid + + + + + + + + + + + + 912 + 0 Y @@ -986,8 +1508,52 @@ - 880 - 416 + 384 + 224 + Y + + + + insert merge mapping for missing voided by + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('person', 'voided_by', ?, ?); + Y + + + + + + + voided_by + + + uuid + + + + + + + + + + + + 656 + 208 Y @@ -1030,8 +1596,40 @@ - 416 - 48 + 480 + 16 + Y + + + + Daemon concept + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select concept_id as daemon_concept_id from concept limit 1; + Y + N + + + + + + + + + + + + 384 + 0 Y diff --git a/src/main/resources/pdiresources/transformations/merge-users.ktr b/src/main/resources/pdiresources/transformations/merge-users.ktr index bc2dd7d..383ef4f 100644 --- a/src/main/resources/pdiresources/transformations/merge-users.ktr +++ b/src/main/resources/pdiresources/transformations/merge-users.ktr @@ -559,11 +559,11 @@ Users In Both Source & Destination - Persons in Source Not Destination + Users in Source Not Destination Y - Persons in Source Not Destination + Users in Source Not Destination User Creator uuid From Destination Y @@ -593,36 +593,133 @@ Y - Set person_id to daemon person id - Save Missing users into Destination + User Creator uuid From Destination + User Person uuid From Destination Y Person Exists on Destination - Save Missing users into Destination + insert merge mapping for missing person Y - User Creator uuid From Destination - User Person uuid From Destination + insert merge mapping for missing person + Set person_id to daemon person id Y User Person uuid From Destination + User Changed by uuid From Destination + Y + + + User Changed by uuid From Destination + User retired by uuid From Destination + Y + + + User retired by uuid From Destination Daemon user/person Y + + Set person_id to daemon person id + Changed by Exists on Destination + Y + Person Exists on Destination - insert merge mapping for missing person + Changed by Exists on Destination Y - insert merge mapping for missing person - Set person_id to daemon person id + Changed by Exists on Destination + Retired By Exists on Destination + Y + + + Changed by Exists on Destination + insert merge mapping for missing changed by + Y + + + insert merge mapping for missing changed by + Set changed by to daemon id + Y + + + Set changed by to daemon id + Retired By Exists on Destination + Y + + + Retired By Exists on Destination + insert merge mapping for missing retired by + Y + + + Retired By Exists on Destination + Save Missing users into Destination + Y + + + insert merge mapping for missing retired by + Set retired by to daemon id + Y + + + Set retired by to daemon id + Save Missing users into Destination Y + + Changed by Exists on Destination + FilterRows + + Y + + 1 + + none + + + insert merge mapping for missing changed by + Retired By Exists on Destination + + + N + + + N + destination_changed_by + IS NULL + + + + N + AND + changed_by + IS NOT NULL + + + + + + + + + + + + + + + 960 + 256 + Y + + Creator Exists on Destination FilterRows @@ -634,14 +731,26 @@ none - Person Exists on Destination - insert merge mapping for missing creator + insert merge mapping for missing creator + Person Exists on Destination N - destination_creator_uuid - IS NOT NULL - + + + N + destination_creator_uuid + IS NULL + + + + N + AND + creator + IS NOT NULL + + + @@ -653,8 +762,8 @@ - 352 - 416 + 320 + 480 Y @@ -723,44 +832,8 @@ - User Creator uuid From Destination - DBJoin - - Y - - 1 - - none - - - OpenMRS destination - 0 - SELECT uuid as destination_creator_uuid from users where user_id = ? - Y - N - - - creator - String - - - - - - - - - - - - 256 - 192 - Y - - - - Users In Both Source & Destination - DBJoin + Person Exists on Destination + FilterRows Y @@ -769,17 +842,28 @@ none - OpenMRS destination - 0 - select user_id as destination_user_id from users where uuid = ? - Y - N - - - uuid - String - - + insert merge mapping for missing person + Changed by Exists on Destination + + + N + + + N + destination_person_uuid + IS NULL + + + + N + AND + person_id + IS NOT NULL + + + + + @@ -789,13 +873,13 @@ - 128 - 544 + 880 + 48 Y - Persons in Source Not Destination + Retired By Exists on Destination FilterRows Y @@ -805,14 +889,26 @@ none - User Creator uuid From Destination - + insert merge mapping for missing retired by + Save Missing users into Destination N - destination_user_id - IS NULL - + + + N + destination_retired_by + IS NULL + + + + N + AND + retired_by + IS NOT NULL + + + @@ -824,8 +920,8 @@ - 240 - 336 + 720 + 384 Y @@ -932,13 +1028,13 @@ - 944 + 960 384 Y - Set creator to daemon user + Set changed by to daemon id SetValueField Y @@ -950,7 +1046,7 @@ - creator + changed_by daemon_user_id @@ -963,14 +1059,14 @@ - 704 - 0 + 608 + 336 Y - insert merge mapping for missing creator - ExecSQL + Set creator to daemon user + SetValueField Y @@ -979,25 +1075,12 @@ none - OpenMRS destination - Y - N - Y - N - INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('users', 'creator', ?, ?); - Y - - - - - - + + creator - - - uuid - - + daemon_user_id + + @@ -1007,14 +1090,14 @@ - 592 - 128 + 704 + 0 Y - insert merge mapping for users - ExecSQL + Set person_id to daemon person id + SetValueField Y @@ -1023,25 +1106,12 @@ none - OpenMRS destination - Y - N - Y - N - INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('users', 'user_id', ?, ?); - Y - - - - - - - user_id - - - uuid - - + + + person_id + daemon_person_id + + @@ -1051,14 +1121,14 @@ - 496 - 32 + 816 + 208 Y - Person Exists on Destination - FilterRows + Set retired by to daemon id + SetValueField Y @@ -1067,16 +1137,12 @@ none - Save Missing users into Destination - insert merge mapping for missing person - - - N - destination_person_uuid - IS NOT NULL - - - + + + retired_by + daemon_user_id + + @@ -1086,14 +1152,14 @@ - 672 - 240 + 823 + 455 Y - Set person_id to daemon person id - SetValueField + User Changed by uuid From Destination + DBJoin Y @@ -1102,12 +1168,17 @@ none - + OpenMRS destination + 0 + SELECT uuid as destination_changed_by from users where user_id = ? + Y + N + - person_id - daemon_person_id + changed_by + Integer - + @@ -1117,13 +1188,13 @@ - 672 - 448 + 240 + 192 Y - User Person uuid From Destination + User Creator uuid From Destination DBJoin Y @@ -1135,13 +1206,13 @@ OpenMRS destination 0 - SELECT uuid as destination_person_uuid from person where person_id = ? + SELECT uuid as destination_creator_uuid from users where user_id = ? Y N - person_id - String + creator + Integer @@ -1154,13 +1225,13 @@ 240 - 112 + 336 Y - insert merge mapping for missing person - ExecSQL + User Person uuid From Destination + DBJoin Y @@ -1170,12 +1241,243 @@ OpenMRS destination - Y - N - Y - N - INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('users', 'person_id', ?, ?); - Y + 0 + SELECT uuid as destination_person_uuid from person where person_id = ? + Y + N + + + person_id + Integer + + + + + + + + + + + + 240 + 272 + Y + + + + User retired by uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_retired_by from users where user_id = ? + Y + N + + + retired_by + Integer + + + + + + + + + + + + 224 + 112 + Y + + + + Users In Both Source & Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select user_id as destination_user_id from users where uuid = ? + Y + N + + + uuid + String + + + + + + + + + + + + 128 + 544 + Y + + + + Users in Source Not Destination + FilterRows + + Y + + 1 + + none + + + User Creator uuid From Destination + + + + N + destination_user_id + IS NULL + + + + + + + + + + + + + 208 + 432 + Y + + + + insert merge mapping for missing changed by + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('users', 'changed_by', ?, ?); + Y + + + + + + + changed_by + + + uuid + + + + + + + + + + + + 784 + 288 + Y + + + + insert merge mapping for missing creator + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('users', 'creator', ?, ?); + Y + + + + + + + creator + + + uuid + + + + + + + + + + + + 592 + 128 + Y + + + + insert merge mapping for missing person + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('users', 'person_id', ?, ?); + Y @@ -1197,8 +1499,96 @@ - 624 - 336 + 816 + 144 + Y + + + + insert merge mapping for missing retired by + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('users', 'retired_by', ?, ?); + Y + + + + + + + retired_by + + + uuid + + + + + + + + + + + + 672 + 485 + Y + + + + insert merge mapping for users + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('users', 'user_id', ?, ?); + Y + + + + + + + user_id + + + uuid + + + + + + + + + + + + 496 + 32 Y diff --git a/src/main/resources/pdiresources/transformations/update-mapped-foreign-cause-of-death.ktr b/src/main/resources/pdiresources/transformations/update-mapped-foreign-cause-of-death.ktr new file mode 100644 index 0000000..c20ecbe --- /dev/null +++ b/src/main/resources/pdiresources/transformations/update-mapped-foreign-cause-of-death.ktr @@ -0,0 +1,718 @@ + + + + update-mapped-foreign-cause-of-death + + + + Normal + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/29 02:16:38.102 + - + 2019/08/29 02:16:38.102 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + destination/merge + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + source + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_SOURCE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + Read cause of death from mappings + Get entity's cause of death Source uuid + Y + + + Get entity's cause of death Source uuid + Get cause of death Destination entity id + Y + + + Get cause of death Destination entity id + Update entity's cause of death to mapped in destination + Y + + + + Get cause of death Destination entity id + DBJoin + + Y + + 1 + + none + + + destination/merge + 0 + SELECT concept_id as mapping_destination_cause_of_death_entity_id from concept where uuid = ? + Y + Y + + + source_concept_uuid + String + + + + + + + + + + + + 784 + 0 + Y + + + + Get entity's cause of death Source uuid + DBJoin + + Y + + 1 + + none + + + source + 0 + SELECT uuid as source_concept_uuid from concept where concept_id = ? + Y + Y + + + mapping_source_cause_of_death_entity_id + Integer + + + + + + + + + + + + 320 + 16 + Y + + + + Read cause of death from mappings + TableInput + + Y + + 1 + + none + + + destination/merge + SELECT uuid AS mapping_source_entity_uuid, answer AS mapping_source_cause_of_death_entity_id FROM migrator_merge_mappings WHERE entity = '${ENTITY}' AND attribute = 'cause_of_death'; + 0 + + N + Y + N + + + + + + + + + + 768 + 256 + Y + + + + Update entity's cause of death to mapped in destination + InsertUpdate + + Y + + 1 + + none + + + destination/merge + 100 + N + + +
${ENTITY}
+ + mapping_source_entity_uuid + uuid + = + + + + cause_of_death + mapping_destination_cause_of_death_entity_id + Y + + + + + + + + + + + + 272 + 240 + Y + + + + + + + N + + diff --git a/src/main/resources/pdiresources/transformations/update-mapped-foreign-changed-by.ktr b/src/main/resources/pdiresources/transformations/update-mapped-foreign-changed-by.ktr new file mode 100644 index 0000000..0304136 --- /dev/null +++ b/src/main/resources/pdiresources/transformations/update-mapped-foreign-changed-by.ktr @@ -0,0 +1,718 @@ + + + + update-mapped-foreign-changed-by + + + + Normal + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/29 02:16:38.102 + - + 2019/08/29 02:16:38.102 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + destination/merge + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + source + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_SOURCE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + Read changed by from mappings + Get entity's changed by Source uuid + Y + + + Get entity's changed by Source uuid + Get changed by Destination entity id + Y + + + Get changed by Destination entity id + Update entity's changed by to mapped in destination + Y + + + + Get changed by Destination entity id + DBJoin + + Y + + 1 + + none + + + destination/merge + 0 + SELECT user_id as mapping_destination_changed_by_entity_id from users where uuid = ? + Y + Y + + + source_creator_uuid + String + + + + + + + + + + + + 784 + 0 + Y + + + + Get entity's changed by Source uuid + DBJoin + + Y + + 1 + + none + + + source + 0 + SELECT uuid as source_creator_uuid from users where user_id = ? + Y + Y + + + mapping_source_changed_by_entity_id + Integer + + + + + + + + + + + + 320 + 16 + Y + + + + Read changed by from mappings + TableInput + + Y + + 1 + + none + + + destination/merge + SELECT uuid AS mapping_source_entity_uuid, answer AS mapping_source_changed_by_entity_id FROM migrator_merge_mappings WHERE entity = '${ENTITY}' AND attribute = 'changed_by'; + 0 + + N + Y + N + + + + + + + + + + 768 + 256 + Y + + + + Update entity's changed by to mapped in destination + InsertUpdate + + Y + + 1 + + none + + + destination/merge + 100 + N + + +
${ENTITY}
+ + mapping_source_entity_uuid + uuid + = + + + + changed_by + mapping_destination_changed_by_entity_id + Y + + + + + + + + + + + + 272 + 240 + Y + + + + + + + N + + diff --git a/src/main/resources/pdiresources/transformations/update-mapped-foreign-concept-class.ktr b/src/main/resources/pdiresources/transformations/update-mapped-foreign-concept-class.ktr new file mode 100644 index 0000000..381b8f2 --- /dev/null +++ b/src/main/resources/pdiresources/transformations/update-mapped-foreign-concept-class.ktr @@ -0,0 +1,718 @@ + + + + update-mapped-foreign-concept-class + + + + Normal + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/29 02:16:38.102 + - + 2019/08/29 02:16:38.102 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + destination/merge + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + source + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_SOURCE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + Read concept class from mappings + Get entity's concept class Source uuid + Y + + + Get entity's concept class Source uuid + Get concept class Destination entity id + Y + + + Get concept class Destination entity id + Update entity's concept class to mapped in destination + Y + + + + Get concept class Destination entity id + DBJoin + + Y + + 1 + + none + + + destination/merge + 0 + SELECT concept_class_id as mapping_destination_concept_class_entity_id from concept_class where uuid = ? + Y + Y + + + source_concept_class_uuid + String + + + + + + + + + + + + 784 + 0 + Y + + + + Get entity's concept class Source uuid + DBJoin + + Y + + 1 + + none + + + source + 0 + SELECT uuid as source_concept_class_uuid from concept_class where concept_class_id = ? + Y + Y + + + mapping_source_concept_class_entity_id + Integer + + + + + + + + + + + + 320 + 16 + Y + + + + Read concept class from mappings + TableInput + + Y + + 1 + + none + + + destination/merge + SELECT uuid AS mapping_source_entity_uuid, answer AS mapping_source_concept_class_entity_id FROM migrator_merge_mappings WHERE entity = '${ENTITY}' AND attribute = 'class_id'; + 0 + + N + Y + N + + + + + + + + + + 768 + 256 + Y + + + + Update entity's concept class to mapped in destination + InsertUpdate + + Y + + 1 + + none + + + destination/merge + 100 + N + + +
${ENTITY}
+ + mapping_source_entity_uuid + uuid + = + + + + class_id + mapping_destination_concept_class_entity_id + Y + + + + + + + + + + + + 272 + 240 + Y + + + + + + + N + + diff --git a/src/main/resources/pdiresources/transformations/update-mapped-foreign-concept-datatype.ktr b/src/main/resources/pdiresources/transformations/update-mapped-foreign-concept-datatype.ktr new file mode 100644 index 0000000..688e080 --- /dev/null +++ b/src/main/resources/pdiresources/transformations/update-mapped-foreign-concept-datatype.ktr @@ -0,0 +1,718 @@ + + + + update-mapped-foreign-concept-datatype + + + + Normal + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/29 02:16:38.102 + - + 2019/08/29 02:16:38.102 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + destination/merge + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + source + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_SOURCE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + Read cconcept datatype from mappings + Get entity's concept datatype Source uuid + Y + + + Get entity's concept datatype Source uuid + Get concept datatype Destination entity id + Y + + + Get concept datatype Destination entity id + Update entity's concept datatype to mapped in destination + Y + + + + Get concept datatype Destination entity id + DBJoin + + Y + + 1 + + none + + + destination/merge + 0 + SELECT concept_datatype_id as mapping_destination_concept_datatype_entity_id from concept_datatype where uuid = ? + Y + Y + + + source_concept_datatype_uuid + String + + + + + + + + + + + + 784 + 0 + Y + + + + Get entity's concept datatype Source uuid + DBJoin + + Y + + 1 + + none + + + source + 0 + SELECT uuid as source_concept_datatype_uuid from concept_datatype where concept_datatype_id = ? + Y + Y + + + mapping_source_concept_datatype_entity_id + Integer + + + + + + + + + + + + 320 + 16 + Y + + + + Read cconcept datatype from mappings + TableInput + + Y + + 1 + + none + + + destination/merge + SELECT uuid AS mapping_source_entity_uuid, answer AS mapping_source_concept_datatype_entity_id FROM migrator_merge_mappings WHERE entity = '${ENTITY}' AND attribute = 'datatype_id'; + 0 + + N + Y + N + + + + + + + + + + 768 + 256 + Y + + + + Update entity's concept datatype to mapped in destination + InsertUpdate + + Y + + 1 + + none + + + destination/merge + 100 + N + + +
${ENTITY}
+ + mapping_source_entity_uuid + uuid + = + + + + datatype_id + mapping_destination_concept_datatype_entity_id + Y + + + + + + + + + + + + 272 + 240 + Y + + + + + + + N + + diff --git a/src/main/resources/pdiresources/transformations/update-mapped-foreign-creators.ktr b/src/main/resources/pdiresources/transformations/update-mapped-foreign-creators.ktr new file mode 100644 index 0000000..d75a223 --- /dev/null +++ b/src/main/resources/pdiresources/transformations/update-mapped-foreign-creators.ktr @@ -0,0 +1,718 @@ + + + + update-mapped-foreign-creators + + + + Normal + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/29 02:16:38.102 + - + 2019/08/29 02:16:38.102 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + destination/merge + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + source + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_SOURCE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + Read creators from mappings + Get entity's creator Source uuid + Y + + + Get entity's creator Source uuid + Get creator Destination entity id + Y + + + Get creator Destination entity id + Update entity's creator to mapped in destination + Y + + + + Get creator Destination entity id + DBJoin + + Y + + 1 + + none + + + destination/merge + 0 + SELECT user_id as mapping_destination_creator_entity_id from users where uuid = ? + Y + Y + + + source_creator_uuid + String + + + + + + + + + + + + 784 + 16 + Y + + + + Get entity's creator Source uuid + DBJoin + + Y + + 1 + + none + + + source + 0 + SELECT uuid as source_creator_uuid from users where user_id = ? + Y + Y + + + mapping_source_creator_entity_id + Integer + + + + + + + + + + + + 320 + 16 + Y + + + + Read creators from mappings + TableInput + + Y + + 1 + + none + + + destination/merge + SELECT uuid AS mapping_source_entity_uuid, answer AS mapping_source_creator_entity_id FROM migrator_merge_mappings WHERE entity = '${ENTITY}' AND attribute = 'creator'; + 0 + + N + Y + N + + + + + + + + + + 112 + 192 + Y + + + + Update entity's creator to mapped in destination + InsertUpdate + + Y + + 1 + + none + + + destination/merge + 100 + N + + +
${ENTITY}
+ + mapping_source_entity_uuid + uuid + = + + + + creator + mapping_destination_creator_entity_id + Y + + + + + + + + + + + + 848 + 176 + Y + + + + + + + N + + diff --git a/src/main/resources/pdiresources/transformations/update-mapped-foreign-person-id.ktr b/src/main/resources/pdiresources/transformations/update-mapped-foreign-person-id.ktr new file mode 100644 index 0000000..b9b78a6 --- /dev/null +++ b/src/main/resources/pdiresources/transformations/update-mapped-foreign-person-id.ktr @@ -0,0 +1,718 @@ + + + + update-mapped-foreign-person-id + + + + Normal + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/29 02:16:38.102 + - + 2019/08/29 02:16:38.102 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + destination/merge + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + source + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_SOURCE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + Read person id from mappings + Get entity's person_id Source uuid + Y + + + Get entity's person_id Source uuid + Get person id Destination entity id + Y + + + Get person id Destination entity id + Update entity's person id to mapped in destination + Y + + + + Get person id Destination entity id + DBJoin + + Y + + 1 + + none + + + destination/merge + 0 + SELECT person_id as mapping_destination_person_id_entity_id from person where uuid = ? + Y + Y + + + source_creator_uuid + String + + + + + + + + + + + + 784 + 16 + Y + + + + Get entity's person_id Source uuid + DBJoin + + Y + + 1 + + none + + + source + 0 + SELECT uuid as source_creator_uuid from person where person_id = ? + Y + Y + + + mapping_source_person_id_entity_id + Integer + + + + + + + + + + + + 320 + 16 + Y + + + + Read person id from mappings + TableInput + + Y + + 1 + + none + + + destination/merge + SELECT uuid AS mapping_source_entity_uuid, answer AS mapping_source_person_id_entity_id FROM migrator_merge_mappings WHERE entity = '${ENTITY}' AND attribute = 'person_id'; + 0 + + N + Y + N + + + + + + + + + + 800 + 240 + Y + + + + Update entity's person id to mapped in destination + InsertUpdate + + Y + + 1 + + none + + + destination/merge + 100 + N + + +
${ENTITY}
+ + mapping_source_entity_uuid + uuid + = + + + + person_id + mapping_destination_person_id_entity_id + Y + + + + + + + + + + + + 320 + 256 + Y + + + + + + + N + + diff --git a/src/main/resources/pdiresources/transformations/update-mapped-foreign-retired-by.ktr b/src/main/resources/pdiresources/transformations/update-mapped-foreign-retired-by.ktr new file mode 100644 index 0000000..8f53d68 --- /dev/null +++ b/src/main/resources/pdiresources/transformations/update-mapped-foreign-retired-by.ktr @@ -0,0 +1,718 @@ + + + + update-mapped-foreign-retired-by + + + + Normal + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/29 02:16:38.102 + - + 2019/08/29 02:16:38.102 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + destination/merge + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + source + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_SOURCE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + Read retired by from mappings + Get entity's retired by Source uuid + Y + + + Get entity's retired by Source uuid + Get retired by Destination entity id + Y + + + Get retired by Destination entity id + Update entity's retired by to mapped in destination + Y + + + + Get retired by Destination entity id + DBJoin + + Y + + 1 + + none + + + destination/merge + 0 + SELECT user_id as mapping_destination_retired_entity_id from users where uuid = ? + Y + Y + + + source_creator_uuid + String + + + + + + + + + + + + 784 + 16 + Y + + + + Get entity's retired by Source uuid + DBJoin + + Y + + 1 + + none + + + source + 0 + SELECT uuid as source_creator_uuid from users where user_id = ? + Y + Y + + + mapping_source_retired_by_entity_id + Integer + + + + + + + + + + + + 320 + 16 + Y + + + + Read retired by from mappings + TableInput + + Y + + 1 + + none + + + destination/merge + SELECT uuid AS mapping_source_entity_uuid, answer AS mapping_source_retired_by_entity_id FROM migrator_merge_mappings WHERE entity = '${ENTITY}' AND attribute = 'retired_by'; + 0 + + N + Y + N + + + + + + + + + + 800 + 240 + Y + + + + Update entity's retired by to mapped in destination + InsertUpdate + + Y + + 1 + + none + + + destination/merge + 100 + N + + +
${ENTITY}
+ + mapping_source_entity_uuid + uuid + = + + + + retired_by + mapping_destination_retired_entity_id + Y + + + + + + + + + + + + 320 + 256 + Y + + + + + + + N + + diff --git a/src/main/resources/pdiresources/transformations/update-mapped-foreign-voided-by.ktr b/src/main/resources/pdiresources/transformations/update-mapped-foreign-voided-by.ktr new file mode 100644 index 0000000..a264d9a --- /dev/null +++ b/src/main/resources/pdiresources/transformations/update-mapped-foreign-voided-by.ktr @@ -0,0 +1,718 @@ + + + + update-mapped-foreign-voided-by + + + + Normal + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/29 02:16:38.102 + - + 2019/08/29 02:16:38.102 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + destination/merge + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + source + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_SOURCE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + Read voided by from mappings + Get entity's voided by Source uuid + Y + + + Get entity's voided by Source uuid + Get voided by Destination entity id + Y + + + Get voided by Destination entity id + Update entity's voided by to mapped in destination + Y + + + + Get entity's voided by Source uuid + DBJoin + + Y + + 1 + + none + + + source + 0 + SELECT uuid as source_creator_uuid from users where user_id = ? + Y + Y + + + mapping_source_voided_by_entity_id + Integer + + + + + + + + + + + + 320 + 16 + Y + + + + Get voided by Destination entity id + DBJoin + + Y + + 1 + + none + + + destination/merge + 0 + SELECT user_id as mapping_destination_voided_entity_id from users where uuid = ? + Y + Y + + + source_creator_uuid + String + + + + + + + + + + + + 784 + 16 + Y + + + + Read voided by from mappings + TableInput + + Y + + 1 + + none + + + destination/merge + SELECT uuid AS mapping_source_entity_uuid, answer AS mapping_source_voided_by_entity_id FROM migrator_merge_mappings WHERE entity = '${ENTITY}' AND attribute = 'voided_by'; + 0 + + N + Y + N + + + + + + + + + + 800 + 240 + Y + + + + Update entity's voided by to mapped in destination + InsertUpdate + + Y + + 1 + + none + + + destination/merge + 100 + N + + +
${ENTITY}
+ + mapping_source_entity_uuid + uuid + = + + + + voided_by + mapping_destination_voided_entity_id + Y + + + + + + + + + + + + 320 + 256 + Y + + + + + + + N + + diff --git a/src/main/resources/pdiresources/transformations/update-mapped-user-creators.ktr b/src/main/resources/pdiresources/transformations/update-mapped-user-creators.ktr index 9afec82..16e1639 100644 --- a/src/main/resources/pdiresources/transformations/update-mapped-user-creators.ktr +++ b/src/main/resources/pdiresources/transformations/update-mapped-user-creators.ktr @@ -704,8 +704,8 @@ - 832 - 96 + 848 + 64 Y From ea658e8253c6b4f6746203802142c83450e08165 Mon Sep 17 00:00:00 2001 From: k-joseph Date: Tue, 3 Sep 2019 11:32:55 +0300 Subject: [PATCH 7/9] MEPTS-224: auto formatted code with spotess --- src/main/java/com/openmrs/migrator/Migrator.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/openmrs/migrator/Migrator.java b/src/main/java/com/openmrs/migrator/Migrator.java index 13f3b30..461cd8c 100644 --- a/src/main/java/com/openmrs/migrator/Migrator.java +++ b/src/main/java/com/openmrs/migrator/Migrator.java @@ -8,13 +8,6 @@ import com.openmrs.migrator.core.services.impl.MySQLProps; import com.openmrs.migrator.core.utilities.ConsoleUtils; import com.openmrs.migrator.core.utilities.FileIOUtilities; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import picocli.CommandLine; -import picocli.CommandLine.Command; -import picocli.CommandLine.Option; - import java.io.Console; import java.io.File; import java.io.IOException; @@ -29,6 +22,12 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.Callable; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; @Command(name = "migrator", mixinStandardHelpOptions = true, helpCommand = true) public class Migrator implements Callable> { @@ -148,7 +147,8 @@ private void executeSetupCommand() throws IOException, SQLException, SettingsExc pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/validate-visit-types.ktr"); pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-persons.ktr"); pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-users.ktr"); - pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-concept-datatypes.ktr"); + pdiFiles.add( + SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-concept-datatypes.ktr"); pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-concept-classes.ktr"); pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-concepts.ktr"); pdiFiles.add(SettingsService.SETTINGS_PROPERTIES); From f7cfe4c4e02444d253d54044acf7112ca930688a Mon Sep 17 00:00:00 2001 From: k-joseph Date: Tue, 3 Sep 2019 18:45:39 +0300 Subject: [PATCH 8/9] MEPTS-243: added merge transformations for privileges, person-attribute-types and person-attributes and did some refactorings --- = | 0 .../java/com/openmrs/migrator/Migrator.java | 5 + .../resources/pdiresources/jobs/merge.kjb | 1113 +++++++--- .../transformations/create-mappings.ktr | 8 +- .../transformations/drop-mappings.ktr | 532 ----- .../transformations/merge-concept-classes.ktr | 2 +- .../merge-concept-datatypes.ktr | 296 +-- .../merge-person-attribute-types.ktr | 1638 +++++++++++++++ .../merge-person-attributes.ktr | 1805 +++++++++++++++++ .../transformations/merge-persons.ktr | 66 +- ...-user-persons.ktr => merge-privileges.ktr} | 186 +- ...update-mapped-foreign-edit-privileges.ktr} | 60 +- ...-mapped-foreign-person-attribute-type.ktr} | 62 +- 13 files changed, 4656 insertions(+), 1117 deletions(-) create mode 100644 = delete mode 100644 src/main/resources/pdiresources/transformations/drop-mappings.ktr create mode 100644 src/main/resources/pdiresources/transformations/merge-person-attribute-types.ktr create mode 100644 src/main/resources/pdiresources/transformations/merge-person-attributes.ktr rename src/main/resources/pdiresources/transformations/{update-mapped-user-persons.ktr => merge-privileges.ktr} (81%) rename src/main/resources/pdiresources/transformations/{update-mapped-person-creators.ktr => update-mapped-foreign-edit-privileges.ktr} (92%) rename src/main/resources/pdiresources/transformations/{update-mapped-user-creators.ktr => update-mapped-foreign-person-attribute-type.ktr} (90%) diff --git a/= b/= new file mode 100644 index 0000000..e69de29 diff --git a/src/main/java/com/openmrs/migrator/Migrator.java b/src/main/java/com/openmrs/migrator/Migrator.java index 461cd8c..3cfef88 100644 --- a/src/main/java/com/openmrs/migrator/Migrator.java +++ b/src/main/java/com/openmrs/migrator/Migrator.java @@ -151,6 +151,11 @@ private void executeSetupCommand() throws IOException, SQLException, SettingsExc SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-concept-datatypes.ktr"); pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-concept-classes.ktr"); pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-concepts.ktr"); + pdiFiles.add( + SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-person-attribute-types.ktr"); + pdiFiles.add( + SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-person-attributes.ktr"); + pdiFiles.add(SettingsService.PDI_RESOURCES_DIR + "/transformations/merge-privileges.ktr"); pdiFiles.add(SettingsService.SETTINGS_PROPERTIES); pdiFiles.add(SettingsService.PDI_PLUGINS_DIR + "/pdi-core-plugins-impl-8.2.0.7-719.jar"); diff --git a/src/main/resources/pdiresources/jobs/merge.kjb b/src/main/resources/pdiresources/jobs/merge.kjb index 3606d68..c910757 100644 --- a/src/main/resources/pdiresources/jobs/merge.kjb +++ b/src/main/resources/pdiresources/jobs/merge.kjb @@ -309,7 +309,7 @@ - create mappings table if missing + create or replace mappings table TRANS @@ -342,48 +342,10 @@ N Y 0 - 64 + 80 32 - - drop-mappings - - TRANS - - filename - - ${Internal.Entry.Current.Directory}/../transformations/drop-mappings.ktr - - N - N - N - N - N - N - - - N - N - Basic - N - - N - Y - N - N - N - Pentaho local - - Y - - N - Y - 0 - 560 - 304 - - merge persons @@ -418,8 +380,8 @@ N Y 0 - 336 - 96 + 352 + 240 @@ -456,8 +418,8 @@ N Y 1 - 784 - 16 + 816 + 64 @@ -494,8 +456,8 @@ N Y 2 - 144 - 16 + 416 + 224 @@ -532,8 +494,8 @@ N Y 2 - 928 - 304 + 896 + 160 @@ -570,8 +532,8 @@ N Y 0 - 32 - 304 + 48 + 384 @@ -588,17 +550,12 @@ person JVM - - ENTITY_ID - person_id - JVM - N Y 0 - 416 - 144 + 640 + 0 @@ -615,17 +572,12 @@ users JVM - - ENTITY_ID - user_id - JVM - N Y 0 - 928 - 112 + 944 + 304 @@ -662,8 +614,8 @@ N Y 2 - 688 - 48 + 752 + 80 @@ -700,8 +652,8 @@ N Y 3 - 560 - 80 + 688 + 64 @@ -739,7 +691,7 @@ Y 2 816 - 288 + 144 @@ -776,8 +728,8 @@ N Y 1 - 688 - 256 + 752 + 160 @@ -814,8 +766,8 @@ N Y 1 - 576 - 208 + 672 + 144 @@ -852,8 +804,8 @@ N Y 2 - 192 - 96 + 272 + 224 @@ -891,7 +843,7 @@ Y 2 192 - 176 + 160 @@ -928,8 +880,8 @@ N Y 2 - 64 - 144 + 128 + 96 @@ -946,16 +898,11 @@ concept_class JVM - - ENTITY_ID - concept_class_id - JVM - N Y 0 - 320 + 240 0 @@ -993,8 +940,8 @@ N Y 2 - 608 - 0 + 624 + 80 @@ -1031,8 +978,8 @@ N Y 1 - 400 - 80 + 368 + 96 @@ -1049,16 +996,11 @@ concept_datatype JVM - - ENTITY_ID - concept_datatype_id - JVM - N Y 0 - 512 + 496 16 @@ -1097,191 +1039,664 @@ Y 1 944 - 16 + 80 - - - - Start - create mappings table if missing - 0 - 0 - Y - Y - Y - - - drop-mappings - merge encounter - 0 - 0 - Y - Y - N - - - Set users entities update variables - user creators - 0 - 2 - Y - Y - Y - - - person voided by - person changed by - 2 - 1 - Y - Y - Y - - - person creators - person voided by - 3 - 2 - Y - Y - Y - - - user creators - user retired by - 2 - 2 - Y - Y - N - - - user retired by - user changed by - 2 - 1 - Y - Y - N - - - user changed by - user person id - 1 - 1 - Y - Y - N - - - user person id - drop-mappings - 1 - 0 - Y - Y - N - - - merge concept classes - merge concept datatypes - 2 - 2 - Y - Y - Y - - - merge concept datatypes - merge concepts - 2 - 2 - Y - Y - Y - - - create mappings table if missing - merge concept classes - 0 - 2 - Y - Y - N - - - merge concepts - merge persons - 2 - 0 - Y - Y - Y - - - merge persons - merge users - 0 - 2 - Y - Y - Y - - - merge users - Set concept class entities update variables - 2 - 0 - Y - Y - Y - - - Set concept class entities update variables - concept class - 0 - 1 - Y - Y - Y - - - concept datatype - Set person entities update variables - 2 - 0 - Y - Y - Y - - - concept class - Set concept datatype entities update variables - 1 - 0 - Y - Y - Y - - - Set concept datatype entities update variables - concept datatype - 0 - 2 - Y - Y - Y - - - Set person entities update variables - person creators - 0 - 3 - Y - Y - N - + + merge person attribute types + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/merge-person-attribute-types.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 528 + 128 + + + + merge person attributes + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/merge-person-attributes.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 384 + 144 + + + + Set person attribute types entities update variables + + SET_VARIABLES + + Y + + CURRENT_JOB + + + ENTITY + person_attribute_type + JVM + + + N + Y + 0 + 896 + 352 + + + + Set person attribute entities update variables + + SET_VARIABLES + + Y + + CURRENT_JOB + + + ENTITY + person_attribute + JVM + + + N + Y + 0 + 816 + 448 + + + + privileges + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-edit-privileges.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 704 + 224 + + + + pa type + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-person-attribute-type.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 416 + 384 + + + + merge privilege + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/merge-privileges.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 512 + 224 + + + + pat changedBy + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-changed-by.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 768 + 304 + + + + pat creator + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-creators.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 688 + 288 + + + + pat retired by + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-retired-by.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 672 + 352 + + + + pa changedBy + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-changed-by.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 624 + 432 + + + + pa creator + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-creators.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 688 + 416 + + + + pa voided by + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-voided-by.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 528 + 400 + + + + pa person + + TRANS + + filename + + ${Internal.Entry.Current.Directory}/../transformations/update-mapped-foreign-person-id.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 2 + 304 + 384 + + + + + + Start + create or replace mappings table + 0 + 0 + Y + Y + Y + + + Set users entities update variables + user creators + 0 + 2 + Y + Y + Y + + + person voided by + person changed by + 2 + 1 + Y + Y + Y + + + person creators + person voided by + 3 + 2 + Y + Y + Y + + + user creators + user retired by + 2 + 2 + Y + Y + N + + + user retired by + user changed by + 2 + 1 + Y + Y + N + + + user changed by + user person id + 1 + 1 + Y + Y + N + + + merge concept classes + merge concept datatypes + 2 + 2 + Y + Y + Y + + + merge concept datatypes + merge concepts + 2 + 2 + Y + Y + Y + + + create or replace mappings table + merge concept classes + 0 + 2 + Y + Y + N + + + merge concepts + merge persons + 2 + 0 + Y + Y + Y + + + merge persons + merge users + 0 + 2 + Y + Y + Y + + + Set concept class entities update variables + concept class + 0 + 1 + Y + Y + Y + + + concept datatype + Set person entities update variables + 2 + 0 + Y + Y + Y + + + concept class + Set concept datatype entities update variables + 1 + 0 + Y + Y + Y + + + Set concept datatype entities update variables + concept datatype + 0 + 2 + Y + Y + Y + + + Set person entities update variables + person creators + 0 + 3 + Y + Y + N + person changed by person cause_of_death @@ -1300,6 +1715,150 @@ Y Y + + merge person attribute types + merge person attributes + 2 + 2 + Y + Y + Y + + + merge person attributes + Set concept class entities update variables + 2 + 0 + Y + Y + Y + + + user person id + Set person attribute types entities update variables + 1 + 0 + Y + Y + N + + + Set person attribute types entities update variables + privileges + 0 + 2 + Y + Y + N + + + merge users + merge privilege + 2 + 2 + Y + Y + Y + + + merge privilege + merge person attribute types + 2 + 2 + Y + Y + Y + + + privileges + pat changedBy + 2 + 2 + Y + Y + Y + + + pat changedBy + pat creator + 2 + 2 + Y + Y + Y + + + pat creator + pat retired by + 2 + 2 + Y + Y + Y + + + pat retired by + Set person attribute entities update variables + 2 + 0 + Y + Y + Y + + + pa voided by + pa type + 2 + 2 + Y + Y + Y + + + Set person attribute entities update variables + pa creator + 0 + 2 + Y + Y + Y + + + pa creator + pa changedBy + 2 + 2 + Y + Y + Y + + + pa changedBy + pa voided by + 2 + 2 + Y + Y + Y + + + pa type + pa person + 2 + 2 + Y + Y + Y + + + pa person + merge encounter + 2 + 0 + Y + Y + Y + diff --git a/src/main/resources/pdiresources/transformations/create-mappings.ktr b/src/main/resources/pdiresources/transformations/create-mappings.ktr index 335ff14..e148e7c 100644 --- a/src/main/resources/pdiresources/transformations/create-mappings.ktr +++ b/src/main/resources/pdiresources/transformations/create-mappings.ktr @@ -493,7 +493,7 @@ - create mappings table if missing + create or replace mappings table ExecSQL Y @@ -508,11 +508,13 @@ N N N - CREATE TABLE IF NOT EXISTS migrator_merge_mappings( + DROP TABLE IF EXISTS migrator_merge_mappings; +CREATE TABLE migrator_merge_mappings( mapping_id int(11) NOT NULL AUTO_INCREMENT, entity VARCHAR(38) NOT NULL, attribute VARCHAR(38) NOT NULL, - answer int(11) NOT NULL, + answer int(11), + text_answer VARCHAR(255), uuid VARCHAR(38), PRIMARY KEY (mapping_id) ) ENGINE=INNODB; diff --git a/src/main/resources/pdiresources/transformations/drop-mappings.ktr b/src/main/resources/pdiresources/transformations/drop-mappings.ktr deleted file mode 100644 index 3cf0de2..0000000 --- a/src/main/resources/pdiresources/transformations/drop-mappings.ktr +++ /dev/null @@ -1,532 +0,0 @@ - - - - drop-mappings - - - - Normal - / - - - - - - - - - - - - ID_BATCH - Y - ID_BATCH - - - CHANNEL_ID - Y - CHANNEL_ID - - - TRANSNAME - Y - TRANSNAME - - - STATUS - Y - STATUS - - - LINES_READ - Y - LINES_READ - - - - LINES_WRITTEN - Y - LINES_WRITTEN - - - - LINES_UPDATED - Y - LINES_UPDATED - - - - LINES_INPUT - Y - LINES_INPUT - - - - LINES_OUTPUT - Y - LINES_OUTPUT - - - - LINES_REJECTED - Y - LINES_REJECTED - - - - ERRORS - Y - ERRORS - - - STARTDATE - Y - STARTDATE - - - ENDDATE - Y - ENDDATE - - - LOGDATE - Y - LOGDATE - - - DEPDATE - Y - DEPDATE - - - REPLAYDATE - Y - REPLAYDATE - - - LOG_FIELD - Y - LOG_FIELD - - - EXECUTING_SERVER - N - EXECUTING_SERVER - - - EXECUTING_USER - N - EXECUTING_USER - - - CLIENT - N - CLIENT - - - - - -
- - - - ID_BATCH - Y - ID_BATCH - - - SEQ_NR - Y - SEQ_NR - - - LOGDATE - Y - LOGDATE - - - TRANSNAME - Y - TRANSNAME - - - STEPNAME - Y - STEPNAME - - - STEP_COPY - Y - STEP_COPY - - - LINES_READ - Y - LINES_READ - - - LINES_WRITTEN - Y - LINES_WRITTEN - - - LINES_UPDATED - Y - LINES_UPDATED - - - LINES_INPUT - Y - LINES_INPUT - - - LINES_OUTPUT - Y - LINES_OUTPUT - - - LINES_REJECTED - Y - LINES_REJECTED - - - ERRORS - Y - ERRORS - - - INPUT_BUFFER_ROWS - Y - INPUT_BUFFER_ROWS - - - OUTPUT_BUFFER_ROWS - Y - OUTPUT_BUFFER_ROWS - - - - - -
- - - ID_BATCH - Y - ID_BATCH - - - CHANNEL_ID - Y - CHANNEL_ID - - - LOG_DATE - Y - LOG_DATE - - - LOGGING_OBJECT_TYPE - Y - LOGGING_OBJECT_TYPE - - - OBJECT_NAME - Y - OBJECT_NAME - - - OBJECT_COPY - Y - OBJECT_COPY - - - REPOSITORY_DIRECTORY - Y - REPOSITORY_DIRECTORY - - - FILENAME - Y - FILENAME - - - OBJECT_ID - Y - OBJECT_ID - - - OBJECT_REVISION - Y - OBJECT_REVISION - - - PARENT_CHANNEL_ID - Y - PARENT_CHANNEL_ID - - - ROOT_CHANNEL_ID - Y - ROOT_CHANNEL_ID - - - - - -
- - - ID_BATCH - Y - ID_BATCH - - - CHANNEL_ID - Y - CHANNEL_ID - - - LOG_DATE - Y - LOG_DATE - - - TRANSNAME - Y - TRANSNAME - - - STEPNAME - Y - STEPNAME - - - STEP_COPY - Y - STEP_COPY - - - LINES_READ - Y - LINES_READ - - - LINES_WRITTEN - Y - LINES_WRITTEN - - - LINES_UPDATED - Y - LINES_UPDATED - - - LINES_INPUT - Y - LINES_INPUT - - - LINES_OUTPUT - Y - LINES_OUTPUT - - - LINES_REJECTED - Y - LINES_REJECTED - - - ERRORS - Y - ERRORS - - - LOG_FIELD - N - LOG_FIELD - - - - - -
- - - ID_BATCH - Y - ID_BATCH - - - CHANNEL_ID - Y - CHANNEL_ID - - - LOG_DATE - Y - LOG_DATE - - - METRICS_DATE - Y - METRICS_DATE - - - METRICS_CODE - Y - METRICS_CODE - - - METRICS_DESCRIPTION - Y - METRICS_DESCRIPTION - - - METRICS_SUBJECT - Y - METRICS_SUBJECT - - - METRICS_TYPE - Y - METRICS_TYPE - - - METRICS_VALUE - Y - METRICS_VALUE - - - - - -
- - 0.0 - 0.0 - - 10000 - 50 - 50 - N - Y - 50000 - Y - - N - 1000 - 100 - - - - - - - - - - - 2019/08/28 23:33:23.668 - - - 2019/08/28 23:33:23.668 - - N - - - - - destination/merge - ${ETL_DATABASE_HOST} - MYSQL - Native - ${ETL_MERGE_DATABASE} - ${ETL_DATABASE_PORT} - ${ETL_DATABASE_USER} - ${ETL_DATABASE_PASSWORD} - - - - - - FORCE_IDENTIFIERS_TO_LOWERCASE - N - - - FORCE_IDENTIFIERS_TO_UPPERCASE - N - - - IS_CLUSTERED - N - - - PORT_NUMBER - ${ETL_DATABASE_PORT} - - - PRESERVE_RESERVED_WORD_CASE - Y - - - QUOTE_ALL_FIELDS - N - - - STREAM_RESULTS - Y - - - SUPPORTS_BOOLEAN_DATA_TYPE - Y - - - SUPPORTS_TIMESTAMP_DATA_TYPE - Y - - - USE_POOLING - N - - - - - - - Delete mappings table - Delete - - Y - - 1 - - none - - - destination/merge - 100 - - -
migrator_merge_mappings
- - - - - - - - - - - 144 - 80 - Y - - - - - - - N - - diff --git a/src/main/resources/pdiresources/transformations/merge-concept-classes.ktr b/src/main/resources/pdiresources/transformations/merge-concept-classes.ktr index b5ad914..6b17070 100644 --- a/src/main/resources/pdiresources/transformations/merge-concept-classes.ktr +++ b/src/main/resources/pdiresources/transformations/merge-concept-classes.ktr @@ -670,7 +670,7 @@ none - + Concept class Creator uuid From Destination diff --git a/src/main/resources/pdiresources/transformations/merge-concept-datatypes.ktr b/src/main/resources/pdiresources/transformations/merge-concept-datatypes.ktr index 3c2a890..57c853e 100644 --- a/src/main/resources/pdiresources/transformations/merge-concept-datatypes.ktr +++ b/src/main/resources/pdiresources/transformations/merge-concept-datatypes.ktr @@ -624,7 +624,43 @@ - Creator Exists on Destination + Concept datatype Creator uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_creator_uuid from users where user_id = ? + Y + N + + + creator + Integer + + + + + + + + + + + + 208 + 240 + Y + + + + Concept datatype in Source Not Destination FilterRows Y @@ -634,13 +670,13 @@ none - Retired by Exists on Destination - insert merge mapping for missing creator + Concept datatype Creator uuid From Destination + N - destination_creator_uuid - IS NOT NULL + destination_concept_datatype_id + IS NULL @@ -653,13 +689,13 @@ - 352 - 96 + 336 + 368 Y - Daemon user + Concept datatypes In Both Source & Destination DBJoin Y @@ -671,10 +707,14 @@ OpenMRS destination 0 - select user_id as daemon_user_id from users where system_id = 'daemon'; + select concept_datatype_id as destination_concept_datatype_id from concept_datatype where uuid = ? Y N + + uuid + String + @@ -685,29 +725,32 @@ - 256 - 16 + 144 + 336 Y - Load concept datatypes from Source - TableInput + Creator Exists on Destination + FilterRows - N + Y 1 none - OpenMRS source - SELECT * FROM concept_datatype - 0 - - N - N - N + Retired by Exists on Destination + insert merge mapping for missing creator + + + N + destination_creator_uuid + IS NOT NULL + + + @@ -717,13 +760,13 @@ - 112 - 32 + 352 + 96 Y - Concept datatype Creator uuid From Destination + Daemon user DBJoin Y @@ -735,14 +778,10 @@ OpenMRS destination 0 - SELECT uuid as destination_creator_uuid from users where user_id = ? + select user_id as daemon_user_id from users where system_id = 'daemon'; Y N - - creator - Integer - @@ -753,33 +792,29 @@ - 208 - 240 + 256 + 16 Y - Concept datatypes In Both Source & Destination - DBJoin + Load concept datatypes from Source + TableInput - Y + N 1 none - OpenMRS destination - 0 - select concept_datatype_id as destination_concept_datatype_id from concept_datatype where uuid = ? - Y - N - - - uuid - String - - + OpenMRS source + SELECT * FROM concept_datatype + 0 + + N + N + N @@ -789,13 +824,13 @@ - 144 - 336 + 112 + 32 Y - Concept datatype in Source Not Destination + Retired by Exists on Destination FilterRows Y @@ -805,13 +840,13 @@ none - - + Save Missing concept datatype into Destination + insert merge mapping for missing retired by N - destination_concept_datatype_id - IS NULL + destination_retired_by_uuid + IS NOT NULL @@ -824,8 +859,8 @@ - 336 - 368 + 720 + 16 Y @@ -945,52 +980,8 @@ - insert merge mapping for missing creator - ExecSQL - - Y - - 1 - - none - - - OpenMRS destination - Y - N - Y - N - INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('concept_datatype', 'creator', ?, ?); - Y - - - - - - - creator - - - uuid - - - - - - - - - - - - 416 - 288 - Y - - - - insert merge mapping for concept datatype - ExecSQL + Set retired by to daemon user + SetValueField Y @@ -999,25 +990,12 @@ none - OpenMRS destination - Y - N - Y - N - INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('concept_datatype', 'concept_datatype_id', ?, ?); - Y - - - - - - - concept_datatype_id - - - uuid - - + + + retired_by + daemon_user_id + + @@ -1027,8 +1005,8 @@ - 464 - 16 + 809 + 174 Y @@ -1069,8 +1047,8 @@ - Retired by Exists on Destination - FilterRows + insert merge mapping for concept datatype + ExecSQL Y @@ -1079,16 +1057,25 @@ none - Save Missing concept datatype into Destination - insert merge mapping for missing retired by - - - N - destination_retired_by_uuid - IS NOT NULL - - - + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('concept_datatype', 'concept_datatype_id', ?, ?); + Y + + + + + + + concept_datatype_id + + + uuid + + @@ -1098,13 +1085,13 @@ - 720 + 464 16 Y - insert merge mapping for missing retired by + insert merge mapping for missing creator ExecSQL Y @@ -1119,7 +1106,7 @@ N Y N - INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('concept_datatype', 'retired_by', ?, ?); + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('concept_datatype', 'creator', ?, ?); Y @@ -1127,7 +1114,7 @@ - retired_by + creator uuid @@ -1142,14 +1129,14 @@ - 656 - 240 + 416 + 288 Y - Set retired by to daemon user - SetValueField + insert merge mapping for missing retired by + ExecSQL Y @@ -1158,12 +1145,25 @@ none - - + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('concept_datatype', 'retired_by', ?, ?); + Y + + + + + + retired_by - daemon_user_id - - + + + uuid + + @@ -1173,8 +1173,8 @@ - 809 - 174 + 656 + 240 Y diff --git a/src/main/resources/pdiresources/transformations/merge-person-attribute-types.ktr b/src/main/resources/pdiresources/transformations/merge-person-attribute-types.ktr new file mode 100644 index 0000000..0c6f75c --- /dev/null +++ b/src/main/resources/pdiresources/transformations/merge-person-attribute-types.ktr @@ -0,0 +1,1638 @@ + + + + merge-person-attribute-types + + + + Normal + 0 + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/06 09:23:50.203 + - + 2019/08/06 09:23:50.203 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + OpenMRS destination + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + OpenMRS source + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_SOURCE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + Load Person Attibute Types from Source + Person Attibute Types In Both Source & Destination + Y + + + Person Attibute Types In Both Source & Destination + Person Attibute Types in Source Not Destination + Y + + + insert merge mapping for person Attibute Types + Creator Exists on Destination + Y + + + Creator Exists on Destination + insert merge mapping for missing creator + Y + + + insert merge mapping for missing creator + Set creator to daemon user + Y + + + Person Attibute Types in Source Not Destination + Person Attibute Type Creator uuid From Destination + Y + + + Person Attibute Type Creator uuid From Destination + Person Attibute Type retired by uuid From Destination + Y + + + Changed by Exists on Destination + Retired By Exists on Destination + Y + + + Changed by Exists on Destination + insert merge mapping for missing changed by + Y + + + Retired By Exists on Destination + insert merge mapping for missing retired by + Y + + + Set changed by to daemon id + Retired By Exists on Destination + Y + + + insert merge mapping for missing changed by + Set changed by to daemon id + Y + + + insert merge mapping for missing retired by + Set retired by to daemon id + Y + + + Set creator to daemon user + Changed by Exists on Destination + Y + + + Creator Exists on Destination + Changed by Exists on Destination + Y + + + Person Attibute Type retired by uuid From Destination + Person Attibute Type Changed by uuid From Destination + Y + + + Daemon user + insert merge mapping for person Attibute Types + Y + + + Person Attibute Type Changed by uuid From Destination + Person Attibute Type Privilege uuid From Destination + Y + + + Retired By Exists on Destination + Privilege Exists on Destination + Y + + + Set retired by to daemon id + Privilege Exists on Destination + Y + + + Privilege Exists on Destination + Save Missing person Attibute Types into Destination + Y + + + Privilege Exists on Destination + insert merge mapping for missing privilege + Y + + + insert merge mapping for missing privilege + Set edit_privilege to daemon privilege + Y + + + Set edit_privilege to daemon privilege + Save Missing person Attibute Types into Destination + Y + + + Person Attibute Type Privilege uuid From Destination + Daemon privilege + Y + + + Daemon privilege + Daemon user + Y + + + + Changed by Exists on Destination + FilterRows + + Y + + 1 + + none + + + insert merge mapping for missing changed by + Retired By Exists on Destination + + + N + + + N + destination_changed_by + IS NULL + + + + N + AND + changed_by + IS NOT NULL + + + + + + + + + + + + + + + 576 + 80 + Y + + + + Creator Exists on Destination + FilterRows + + Y + + 1 + + none + + + insert merge mapping for missing creator + Changed by Exists on Destination + + + N + + + N + destination_creator_uuid + IS NULL + + + + N + AND + creator + IS NOT NULL + + + + + + + + + + + + + + + 400 + 96 + Y + + + + Daemon user + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select user_id as daemon_user_id from users where system_id = 'daemon'; + Y + N + + + + + + + + + + + + 416 + 0 + Y + + + + Load Person Attibute Types from Source + TableInput + + N + + 1 + + none + + + OpenMRS source + SELECT * FROM person_attribute_type + 0 + + N + N + N + + + + + + + + + + 0 + 192 + Y + + + + Person Attibute Type Creator uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_creator_uuid from users where user_id = ? + Y + N + + + creator + Integer + + + + + + + + + + + + 224 + 176 + Y + + + + Person Attibute Type retired by uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_retired_by from users where user_id = ? + Y + N + + + retired_by + Integer + + + + + + + + + + + + 176 + 144 + Y + + + + Person Attibute Types In Both Source & Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select person_attribute_type_id as destination_person_attribute_type_id from person_attribute_type where uuid = ? + Y + N + + + uuid + String + + + + + + + + + + + + 112 + 304 + Y + + + + Person Attibute Types in Source Not Destination + FilterRows + + Y + + 1 + + none + + + Person Attibute Type Creator uuid From Destination + + + + N + destination_person_attribute_type_id + IS NULL + + + + + + + + + + + + + 144 + 240 + Y + + + + Person Attibute Type Changed by uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_changed_by from users where user_id = ? + Y + N + + + changed_by + Integer + + + + + + + + + + + + 176 + 80 + Y + + + + Save Missing person Attibute Types into Destination + TableOutput + + Y + + 1 + + none + + + OpenMRS destination + +
person_attribute_type
+ 1000 + N + N + Y + Y + N + + N + Y + N + + Y + N + + + + name + name + + + description + description + + + format + format + + + foreign_key + foreign_key + + + searchable + searchable + + + creator + creator + + + date_created + date_created + + + changed_by + changed_by + + + date_changed + date_changed + + + retired + retired + + + retired_by + retired_by + + + date_retired + date_retired + + + retire_reason + retire_reason + + + edit_privilege + edit_privilege + + + uuid + uuid + + + sort_weight + sort_weight + + + + + + + + + + + + 1008 + 288 + Y + + + + Set changed by to daemon id + SetValueField + + Y + + 1 + + none + + + + + changed_by + daemon_user_id + + + + + + + + + + + + 832 + 64 + Y + + + + Set creator to daemon user + SetValueField + + Y + + 1 + + none + + + + + creator + daemon_user_id + + + + + + + + + + + + 528 + 160 + Y + + + + Set retired by to daemon id + SetValueField + + Y + + 1 + + none + + + + + retired_by + daemon_user_id + + + + + + + + + + + + 800 + 192 + Y + + + + Retired By Exists on Destination + FilterRows + + Y + + 1 + + none + + + insert merge mapping for missing retired by + Privilege Exists on Destination + + + N + + + N + destination_retired_by + IS NULL + + + + N + AND + retired_by + IS NOT NULL + + + + + + + + + + + + + + + 704 + 112 + Y + + + + insert merge mapping for missing changed by + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('person_attribute_type', 'changed_by', ?, ?); + Y + + + + + + + changed_by + + + uuid + + + + + + + + + + + + 912 + 0 + Y + + + + insert merge mapping for missing creator + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('person_attribute_type', 'creator', ?, ?); + Y + + + + + + + creator + + + uuid + + + + + + + + + + + + 384 + 224 + Y + + + + insert merge mapping for missing retired by + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('person_attribute_type', 'retired_by', ?, ?); + Y + + + + + + + retired_by + + + uuid + + + + + + + + + + + + 656 + 208 + Y + + + + insert merge mapping for person Attibute Types + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('person_attribute_type', 'person_attribute_type_id', ?, ?); + Y + + + + + + + person_attribute_type_id + + + uuid + + + + + + + + + + + + 592 + 16 + Y + + + + Person Attibute Type Privilege uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_privilege from privilege where privilege = ? + Y + N + + + edit_privilege + String + + + + + + + + + + + + 208 + 16 + Y + + + + Privilege Exists on Destination + FilterRows + + Y + + 1 + + none + + + insert merge mapping for missing privilege + Save Missing person Attibute Types into Destination + + + N + + + N + destination_privilege + IS NULL + + + + N + AND + edit_privilege + IS NOT NULL + + + + + + + + + + + + + + + 928 + 112 + Y + + + + Set edit_privilege to daemon privilege + SetValueField + + Y + + 1 + + none + + + + + edit_privilege + daemon_privilege + + + + + + + + + + + + 624 + 272 + Y + + + + insert merge mapping for missing privilege + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, text_answer, uuid) VALUES('person_attribute_type', 'edit_privilege', ?, ?); + Y + + + + + + + edit_privilege + + + uuid + + + + + + + + + + + + 832 + 240 + Y + + + + Daemon privilege + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select privilege as daemon_privilege from privilege limit 1; + Y + N + + + + + + + + + + + + 304 + 0 + Y + + + + + + + N + + diff --git a/src/main/resources/pdiresources/transformations/merge-person-attributes.ktr b/src/main/resources/pdiresources/transformations/merge-person-attributes.ktr new file mode 100644 index 0000000..c600ac2 --- /dev/null +++ b/src/main/resources/pdiresources/transformations/merge-person-attributes.ktr @@ -0,0 +1,1805 @@ + + + + merge-person-attributes + + + + Normal + 0 + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2019/08/06 09:23:50.203 + - + 2019/08/06 09:23:50.203 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + OpenMRS destination + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_MERGE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + OpenMRS source + ${ETL_DATABASE_HOST} + MYSQL + Native + ${ETL_SOURCE_DATABASE} + ${ETL_DATABASE_PORT} + ${ETL_DATABASE_USER} + ${ETL_DATABASE_PASSWORD} + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + ${ETL_DATABASE_PORT} + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + STREAM_RESULTS + Y + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + Load Person Attibutes from Source + Person Attibutes In Both Source & Destination + Y + + + Person Attibutes In Both Source & Destination + Person Attibutes in Source Not Destination + Y + + + insert merge mapping for person attribute + Creator Exists on Destination + Y + + + Creator Exists on Destination + insert merge mapping for missing creator + Y + + + insert merge mapping for missing creator + Set creator to daemon user + Y + + + Person Attibutes in Source Not Destination + Person Attibute Creator uuid From Destination + Y + + + Person Attibute Creator uuid From Destination + Person Attibute voided by uuid From Destination + Y + + + Changed by Exists on Destination + Voided By Exists on Destination + Y + + + Changed by Exists on Destination + insert merge mapping for missing changed by + Y + + + Voided By Exists on Destination + insert merge mapping for missing voided by + Y + + + Set changed by to daemon id + Voided By Exists on Destination + Y + + + insert merge mapping for missing changed by + Set changed by to daemon id + Y + + + insert merge mapping for missing voided by + Set voided by to daemon id + Y + + + Set creator to daemon user + Changed by Exists on Destination + Y + + + Creator Exists on Destination + Changed by Exists on Destination + Y + + + Person Attibute voided by uuid From Destination + Person Attibute Type uuid From Destination + Y + + + Person Attibute Type uuid From Destination + Person Changed by uuid From Destination + Y + + + Voided By Exists on Destination + Person Attribute Type Exists on Destination + Y + + + Set voided by to daemon id + Person Attribute Type Exists on Destination + Y + + + Person Attribute Type Exists on Destination + insert merge mapping for missing person attribute type + Y + + + insert merge mapping for missing person attribute type + Set cause of death to none concept id + Y + + + Daemon user + Daemon person attribute type + Y + + + Daemon person attribute type + insert merge mapping for person attribute + Y + + + Person Changed by uuid From Destination + Person Person uuid From Destination + Y + + + Person Person uuid From Destination + Daemon user + Y + + + Person Attribute Type Exists on Destination + Person id on Destination + Y + + + Set cause of death to none concept id + Person id on Destination + Y + + + Person id on Destination + Save Missing persons into Destination + Y + + + Person id on Destination + insert merge mapping for missing person + Y + + + insert merge mapping for missing person + Set person id to daemon person + Y + + + Set person id to daemon person + Save Missing persons into Destination + Y + + + + Person Attribute Type Exists on Destination + FilterRows + + Y + + 1 + + none + + + insert merge mapping for missing person attribute type + Person id on Destination + + + N + + + N + destination_person_attribute_type_uuid + IS NULL + + + + N + AND + person_attribute_type_id + IS NOT NULL + + + + + + + + + + + + + + + 944 + 112 + Y + + + + Changed by Exists on Destination + FilterRows + + Y + + 1 + + none + + + insert merge mapping for missing changed by + Voided By Exists on Destination + + + N + + + N + destination_changed_by + IS NULL + + + + N + AND + changed_by + IS NOT NULL + + + + + + + + + + + + + + + 576 + 80 + Y + + + + Creator Exists on Destination + FilterRows + + Y + + 1 + + none + + + insert merge mapping for missing creator + Changed by Exists on Destination + + + N + + + N + destination_creator_uuid + IS NULL + + + + N + AND + creator + IS NOT NULL + + + + + + + + + + + + + + + 400 + 96 + Y + + + + Daemon person attribute type + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select person_attribute_type_id as daemon_person_attribute_type from person_attribute_type limit 1; + Y + N + + + + + + + + + + + + 432 + 0 + Y + + + + Daemon user + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select user_id as daemon_user_id, person_id as daemon_person_id from users where system_id = 'daemon'; + Y + N + + + + + + + + + + + + 352 + 16 + Y + + + + Load Person Attibutes from Source + TableInput + + N + + 1 + + none + + + OpenMRS source + SELECT * FROM person_attribute + 0 + + N + N + N + + + + + + + + + + 80 + 144 + Y + + + + Person Attibute Type uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_person_attribute_type_uuid from person_attribute_type where person_attribute_type_id = ? + Y + N + + + person_attribute_type_id + Integer + + + + + + + + + + + + 208 + 48 + Y + + + + Person Changed by uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_changed_by from users where user_id = ? + Y + N + + + changed_by + Integer + + + + + + + + + + + + 80 + 16 + Y + + + + Person Attibute Creator uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_creator_uuid from users where user_id = ? + Y + N + + + creator + Integer + + + + + + + + + + + + 240 + 176 + Y + + + + Person Attibute voided by uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_voided_by from users where user_id = ? + Y + N + + + voided_by + Integer + + + + + + + + + + + + 240 + 112 + Y + + + + Person Attibutes In Both Source & Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select person_attribute_id as destination_person_attribute_id from person_attribute where uuid = ? + Y + N + + + uuid + String + + + + + + + + + + + + 112 + 304 + Y + + + + Person Attibutes in Source Not Destination + FilterRows + + Y + + 1 + + none + + + + + + + N + destination_person_attribute_id + IS NULL + + + + + + + + + + + + + 144 + 224 + Y + + + + Save Missing persons into Destination + TableOutput + + Y + + 1 + + none + + + OpenMRS destination + +
person_attribute
+ 1000 + N + N + Y + Y + N + + N + Y + N + + Y + N + + + + person_id + person_id + + + value + value + + + person_attribute_type_id + person_attribute_type_id + + + creator + creator + + + date_created + date_created + + + changed_by + changed_by + + + date_changed + date_changed + + + voided + voided + + + voided_by + voided_by + + + date_voided + date_voided + + + void_reason + void_reason + + + uuid + uuid + + + + + + + + + + + + 464 + 336 + Y + + + + Set cause of death to none concept id + SetValueField + + N + + 1 + + none + + + + + person_attribute_type_id + daemon_person_attribute_type + + + + + + + + + + + + 935 + 314 + Y + + + + Set changed by to daemon id + SetValueField + + Y + + 1 + + none + + + + + changed_by + daemon_user_id + + + + + + + + + + + + 832 + 64 + Y + + + + Set creator to daemon user + SetValueField + + Y + + 1 + + none + + + + + creator + daemon_user_id + + + + + + + + + + + + 528 + 160 + Y + + + + Set voided by to daemon id + SetValueField + + Y + + 1 + + none + + + + + voided_by + daemon_user_id + + + + + + + + + + + + 736 + 176 + Y + + + + Voided By Exists on Destination + FilterRows + + Y + + 1 + + none + + + insert merge mapping for missing voided by + Person Attribute Type Exists on Destination + + + N + + + N + destination_voided_by + IS NULL + + + + N + AND + voided_by + IS NOT NULL + + + + + + + + + + + + + + + 704 + 112 + Y + + + + insert merge mapping for missing person attribute type + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('person_attribute', 'person_attribute_type_id', ?, ?); + Y + + + + + + + person_attribute_type_id + + + uuid + + + + + + + + + + + + 944 + 224 + Y + + + + insert merge mapping for missing changed by + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('person_attribute', 'changed_by', ?, ?); + Y + + + + + + + changed_by + + + uuid + + + + + + + + + + + + 912 + 0 + Y + + + + insert merge mapping for missing creator + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('person_attribute', 'creator', ?, ?); + Y + + + + + + + creator + + + uuid + + + + + + + + + + + + 384 + 256 + Y + + + + insert merge mapping for missing voided by + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('person_attribute', 'voided_by', ?, ?); + Y + + + + + + + voided_by + + + uuid + + + + + + + + + + + + 640 + 208 + Y + + + + insert merge mapping for person attribute + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('person_attribute', 'person_attribute_id', ?, ?); + Y + + + + + + + person_attribute_id + + + uuid + + + + + + + + + + + + 544 + 16 + Y + + + + Person id on Destination + FilterRows + + Y + + 1 + + none + + + insert merge mapping for missing person + Save Missing persons into Destination + + + N + + + N + destination_person_uuid + IS NULL + + + + N + AND + person_id + IS NOT NULL + + + + + + + + + + + + + + + 688 + 288 + Y + + + + Person Person uuid From Destination + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + SELECT uuid as destination_person_uuid from person where person_id = ? + Y + N + + + person_id + Integer + + + + + + + + + + + + 176 + 0 + Y + + + + insert merge mapping for missing person + ExecSQL + + Y + + 1 + + none + + + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, answer, uuid) VALUES('person_attribute', 'person_id', ?, ?); + Y + + + + + + + person_id + + + uuid + + + + + + + + + + + + 864 + 368 + Y + + + + Set person id to daemon person + SetValueField + + N + + 1 + + none + + + + + person_id + daemon_person_id + + + + + + + + + + + + 672 + 400 + Y + + + + + + + N + + diff --git a/src/main/resources/pdiresources/transformations/merge-persons.ktr b/src/main/resources/pdiresources/transformations/merge-persons.ktr index 82a1754..9952967 100644 --- a/src/main/resources/pdiresources/transformations/merge-persons.ktr +++ b/src/main/resources/pdiresources/transformations/merge-persons.ktr @@ -819,6 +819,38 @@ Y + + Daemon concept + DBJoin + + Y + + 1 + + none + + + OpenMRS destination + 0 + select concept_id as daemon_concept_id from concept limit 1; + Y + N + + + + + + + + + + + + 384 + 0 + Y + + Daemon user DBJoin @@ -1074,7 +1106,7 @@ none - + Person Creator uuid From Destination @@ -1601,38 +1633,6 @@ Y - - Daemon concept - DBJoin - - Y - - 1 - - none - - - OpenMRS destination - 0 - select concept_id as daemon_concept_id from concept limit 1; - Y - N - - - - - - - - - - - - 384 - 0 - Y - - diff --git a/src/main/resources/pdiresources/transformations/update-mapped-user-persons.ktr b/src/main/resources/pdiresources/transformations/merge-privileges.ktr similarity index 81% rename from src/main/resources/pdiresources/transformations/update-mapped-user-persons.ktr rename to src/main/resources/pdiresources/transformations/merge-privileges.ktr index b762f3e..290090c 100644 --- a/src/main/resources/pdiresources/transformations/update-mapped-user-persons.ktr +++ b/src/main/resources/pdiresources/transformations/merge-privileges.ktr @@ -1,11 +1,12 @@ - update-mapped-user-persons + merge-privileges Normal + 0 / @@ -427,16 +428,16 @@ - - 2019/08/29 02:16:38.102 + 2019/08/06 09:23:50.203 - - 2019/08/29 02:16:38.102 + 2019/08/06 09:23:50.203 H4sIAAAAAAAAAAMAAAAAAAAAAAA= N - destination/merge + OpenMRS destination ${ETL_DATABASE_HOST} MYSQL Native @@ -491,7 +492,7 @@ - source + OpenMRS source ${ETL_DATABASE_HOST} MYSQL Native @@ -547,23 +548,28 @@ - Read persons from mappings - Get Source User Person uuid + Load Privileges from Source + Privileges In Both Source & Destination Y - Get Source User Person uuid - Get user person Destination person id + insert merge mapping for privileges + Save Missing privileges into Destination Y - Get user person Destination person id - Update user person to mapped in destination + Privileges In Both Source & Destination + Privilege in Source Not Destination + Y + + + Privilege in Source Not Destination + insert merge mapping for privileges Y - Get user person Destination person id + Privileges In Both Source & Destination DBJoin Y @@ -573,14 +579,14 @@ none - destination/merge + OpenMRS destination 0 - SELECT person_id as mapping_destination_user_person_id from person where uuid = ? + select privilege as destination_privilege from privilege where uuid = ? Y N - source_person_uuid + uuid String @@ -593,14 +599,46 @@ - 448 - 144 + 144 + 336 Y - Get Source User Person uuid - DBJoin + Load Privileges from Source + TableInput + + N + + 1 + + none + + + OpenMRS source + SELECT * FROM privilege; + 0 + + N + N + N + + + + + + + + + + 112 + 32 + Y + + + + Save Missing privileges into Destination + TableOutput Y @@ -609,17 +647,37 @@ none - source - 0 - SELECT uuid as source_person_uuid from person where person_id = ? - Y - N - + OpenMRS destination + + privilege
+ 1000 + N + N + Y + Y + N + + N + Y + N + + Y + N + + - mapping_source_user_person_id - Integer + privilege + privilege - + + description + description + + + uuid + uuid + + @@ -629,14 +687,14 @@ - 336 - 32 + 816 + 48 Y - Read persons from mappings - TableInput + insert merge mapping for privileges + ExecSQL Y @@ -645,13 +703,25 @@ none - destination/merge - SELECT uuid AS mapping_source_user_uuid, answer AS mapping_source_user_person_id FROM migrator_merge_mappings WHERE entity = 'users' AND attribute = 'person_id'; - 0 - - N - N - N + OpenMRS destination + Y + N + Y + N + INSERT INTO migrator_merge_mappings(entity, attribute, text_answer, uuid) VALUES('privilege', 'privilege', ?, ?); + Y + + + + + + + privilege + + + uuid + + @@ -661,14 +731,14 @@ - 144 + 448 128 Y - Update user person to mapped in destination - InsertUpdate + Privilege in Source Not Destination + FilterRows Y @@ -677,24 +747,16 @@ none - destination/merge - 100 - N - - - users
- - mapping_source_user_uuid - uuid - = - - - - person_id - mapping_destination_user_person_id - Y - - + insert merge mapping for privileges + + + + N + destination_privilege + IS NULL + + + @@ -704,8 +766,8 @@ - 832 - 112 + 288 + 208 Y diff --git a/src/main/resources/pdiresources/transformations/update-mapped-person-creators.ktr b/src/main/resources/pdiresources/transformations/update-mapped-foreign-edit-privileges.ktr similarity index 92% rename from src/main/resources/pdiresources/transformations/update-mapped-person-creators.ktr rename to src/main/resources/pdiresources/transformations/update-mapped-foreign-edit-privileges.ktr index 39e82d8..7c4b5f8 100644 --- a/src/main/resources/pdiresources/transformations/update-mapped-person-creators.ktr +++ b/src/main/resources/pdiresources/transformations/update-mapped-foreign-edit-privileges.ktr @@ -1,7 +1,7 @@ - update-mapped-person-creators + update-mapped-foreign-edit-privileges @@ -547,23 +547,23 @@ - Read creators from mappings - Get creator Source user uuid + Read privileges from mappings + Get entity's privilege Source uuid Y - Get creator Source user uuid - Get creator Destination user id + Get entity's privilege Source uuid + Get privilege Destination entity id Y - Get creator Destination user id - Update person creator to mapped in destination + Get privilege Destination entity id + Update entity's privilege to mapped in destination Y - Get creator Destination user id + Get privilege Destination entity id DBJoin Y @@ -575,12 +575,12 @@ destination/merge 0 - SELECT user_id as mapping_destination_creator_user_id from users where uuid = ? + SELECT privilege as mapping_destination_privilege from privilege where uuid = ? Y - N + Y - source_creator_uuid + source_privilege_uuid String @@ -593,13 +593,13 @@ - 544 - 176 + 784 + 16 Y - Get creator Source user uuid + Get entity's privilege Source uuid DBJoin Y @@ -611,12 +611,12 @@ source 0 - SELECT uuid as source_creator_uuid from users where user_id = ? + SELECT uuid as source_privilege_uuid from privilege where privilege = ? Y - N + Y - mapping_source_creator_user_id + mapping_source_privilege Integer @@ -629,13 +629,13 @@ - 336 - 32 + 320 + 16 Y - Read creators from mappings + Read privileges from mappings TableInput Y @@ -646,11 +646,11 @@ destination/merge - SELECT uuid AS mapping_source_person_uuid, answer AS mapping_source_creator_user_id FROM migrator_merge_mappings WHERE entity = 'person' AND attribute = 'creator'; + SELECT uuid AS mapping_source_entity_uuid, answer AS mapping_source_privilege FROM migrator_merge_mappings WHERE entity = '${ENTITY}' AND attribute = 'privilege'; 0 N - N + Y N @@ -661,13 +661,13 @@ - 144 - 128 + 112 + 192 Y - Update person creator to mapped in destination + Update entity's privilege to mapped in destination InsertUpdate Y @@ -682,16 +682,16 @@ N - person
+ ${ENTITY}
- mapping_source_person_uuid + mapping_source_entity_uuid uuid = - creator - mapping_destination_creator_user_id + edit_privilege + mapping_destination_privilege Y @@ -704,7 +704,7 @@ - 816 + 848 176 Y diff --git a/src/main/resources/pdiresources/transformations/update-mapped-user-creators.ktr b/src/main/resources/pdiresources/transformations/update-mapped-foreign-person-attribute-type.ktr similarity index 90% rename from src/main/resources/pdiresources/transformations/update-mapped-user-creators.ktr rename to src/main/resources/pdiresources/transformations/update-mapped-foreign-person-attribute-type.ktr index 16e1639..a7509ff 100644 --- a/src/main/resources/pdiresources/transformations/update-mapped-user-creators.ktr +++ b/src/main/resources/pdiresources/transformations/update-mapped-foreign-person-attribute-type.ktr @@ -1,7 +1,7 @@ - update-mapped-user-creators + update-mapped-foreign-person-attribute-type @@ -547,23 +547,23 @@ - Read creators from mappings - Get creator Source user uuid + Read person attribute type from mappings + Get entity's person attribute type Source uuid Y - Get creator Source user uuid - Get creator Destination user id + Get entity's person attribute type Source uuid + Get person attribute Destination entity id Y - Get creator Destination user id - Update user creator to mapped in destination + Get person attribute Destination entity id + Update entity's person attribute type to mapped in destination Y - Get creator Destination user id + Get person attribute Destination entity id DBJoin Y @@ -575,12 +575,12 @@ destination/merge 0 - SELECT user_id as mapping_destination_creator_user_id from users where uuid = ? + SELECT person_attribute_type_id as mapping_destination_person_attribute_type_id from person_attribute_type where uuid = ? Y - N + Y - source_creator_uuid + person_attribute_type_uuid String @@ -593,13 +593,13 @@ - 448 - 144 + 784 + 16 Y - Get creator Source user uuid + Get entity's person attribute type Source uuid DBJoin Y @@ -611,12 +611,12 @@ source 0 - SELECT uuid as source_creator_uuid from users where user_id = ? + SELECT uuid as person_attribute_type_uuid from person_attribute_type where person_attribute_type_id = ? Y - N + Y - mapping_source_creator_user_id + mapping_source_person_attribute_type_id Integer @@ -629,13 +629,13 @@ - 336 - 32 + 320 + 16 Y - Read creators from mappings + Read person attribute type from mappings TableInput Y @@ -646,11 +646,11 @@ destination/merge - SELECT uuid AS mapping_source_user_uuid, answer AS mapping_source_creator_user_id FROM migrator_merge_mappings WHERE entity = 'users' AND attribute = 'creator'; + SELECT uuid AS mapping_source_entity_uuid, answer AS mapping_source_person_attribute_type_id FROM migrator_merge_mappings WHERE entity = '${ENTITY}' AND attribute = 'person_attribute_type_id'; 0 N - N + Y N @@ -661,13 +661,13 @@ - 144 - 128 + 800 + 240 Y - Update user creator to mapped in destination + Update entity's person attribute type to mapped in destination InsertUpdate Y @@ -682,16 +682,16 @@ N - users
+ ${ENTITY}
- mapping_source_user_uuid + mapping_source_entity_uuid uuid = - creator - mapping_destination_creator_user_id + person_attribute_type_id + mapping_destination_person_attribute_type_id Y @@ -704,8 +704,8 @@ - 848 - 64 + 320 + 256 Y From 546378ddabc9cb27f6a10417b02461d6f1187ef9 Mon Sep 17 00:00:00 2001 From: k-joseph Date: Tue, 3 Sep 2019 18:48:07 +0300 Subject: [PATCH 9/9] MEPTS-243: disable merge transformation in control center job --- src/main/resources/pdiresources/jobs/control-center.kjb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/pdiresources/jobs/control-center.kjb b/src/main/resources/pdiresources/jobs/control-center.kjb index 4cabad1..28f0a60 100644 --- a/src/main/resources/pdiresources/jobs/control-center.kjb +++ b/src/main/resources/pdiresources/jobs/control-center.kjb @@ -395,7 +395,7 @@ 64 - + merge JOB @@ -466,7 +466,7 @@ merge 0 0 - Y + N Y N