test: added snapshot-testing based validation of generated resources (#8

)

* test: remove vendored profile resources

* test: added first snapshots tests for the procedure processor

* test: added fhir validation infrastructure

* test: finalized snapshot testing implementation

* chore: ignore snapshots by prettier

* test: don't use full input resource name to avoid "filename too long" errors

* test: added patient mapper tests

* test: use a fixed `app.version` to avoid snapshot invalidations on new releases

.github/workflows/validate-fhir-resources.yaml

@@ -0,0 +1,33 @@
+name: validate-fhir-resources
+
+on:
+  pull_request:
+    branches: [master]
+  push:
+    branches: [master]
+
+permissions: read-all
+
+jobs:
+  validate-fhir-resource:
+    name: Validate FHIR resources
+    runs-on: ubuntu-22.04
+    container: ghcr.io/miracum/ig-build-tools:v2.0.12@sha256:f2cca9f26f7bdb9fe238798b6f60b7798553c7d488633ded6f5a90ce6f083deb
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Install Firely.Terminal
+        run: |
+          dotnet tool install --global Firely.Terminal --version 3.1.0
+          fhir --help
+
+      - name: Restore FHIR package dependencies
+        run: |
+          fhir restore
+
+      - name: Validate generated FHIR resources
+        # TODO: switch to 'xargs' or for loop so if any validation fails, the entire command fails.
+        # for fhir_file in src/test/java/snapshots/**/*.fhir.json; do fhir validate --verbose --fail "$fhir_file"; done
+        run: |
+          find src/test/java/snapshots -name "*.fhir.json" -exec fhir validate --verbose --fail "{}" \;

.gitignore

@@ -53,3 +53,9 @@ deploy/connectors/*
 !deploy/connectors/README.md
 
 bin
+
+# ignored since it isn't automatically refreshed by renovate
+# (we're also pinning the dependency versions anyways)
+fhirpkg.lock.json
+
+*.received.fhir.json

.npmrc

@@ -0,0 +1 @@
+registry=https://packages.simplifier.net

.prettierignore

@@ -0,0 +1 @@
+src/test/java/snapshots/**

.releaserc.json

@@ -92,6 +92,17 @@
                 "hasChanged": true
               }
             ]
+          },
+          {
+            "files": ["package.json"],
+            "from": "\"version\": \".*\"",
+            "to": "\"version\": \"${nextRelease.version}\"",
+            "results": [
+              {
+                "file": "package.json",
+                "hasChanged": true
+              }
+            ]
           }
         ]
       }

README.md

@@ -4,6 +4,8 @@ This project contains a Kafka Stream processor that creates FHIR resources from
 
 ## Used FHIR profiles
 
+See [package.json](package.json) for a list of used packages and their versions.
+
 ## Observations
 
 - Histologie (<https://simplifier.net/oncology/histologie>)
@@ -28,7 +30,7 @@ This project contains a Kafka Stream processor that creates FHIR resources from
 
 ## Patient
 
-- Patient (<https://simplifier.net/packages/de.medizininformatikinitiative.kerndatensatz.person/2.0.0-ballot2/files/533910>)
+- Patient (<https://simplifier.net/medizininformatikinitiative-modulperson>)
 
 ## Dev
 

build.gradle

@@ -46,6 +46,7 @@ dependencies {
 	testImplementation "ca.uhn.hapi.fhir:hapi-fhir-validation-resources-r4:${hapiVersion}"
 	testImplementation "org.springframework.boot:spring-boot-starter-test"
 	testImplementation "org.springframework.cloud:spring-cloud-stream-test-support"
+	testImplementation "com.approvaltests:approvaltests:22.2.0"
 }
 
 dependencyManagement {

package.json

@@ -0,0 +1,13 @@
+{
+  "name": "obds-to-fhir",
+  "version": "0.1.0",
+  "description": "Creates FHIR resources from oBDS-XML data",
+  "dependencies": {
+    "hl7.fhir.r4.core": "4.0.1",
+    "de.dktk.oncology": "1.3.0",
+    "de.medizininformatikinitiative.kerndatensatz.person": "2024.0.0-alpha1"
+  },
+  "fhirVersions": [
+    "4.0.1"
+  ]
+}

src/main/java/org/miracum/streams/ume/obdstofhir/mapper/ObdsPatientMapper.java

@@ -23,7 +23,7 @@ public class ObdsPatientMapper extends ObdsToFhirMapper {
   @Value("${app.enableCheckDigitConv}")
   private boolean checkDigitConversion;
 
-  protected ObdsPatientMapper(FhirProperties fhirProperties) {
+  public ObdsPatientMapper(FhirProperties fhirProperties) {
     super(fhirProperties);
   }
 

src/main/resources/application.yml

@@ -18,7 +18,7 @@ fhir:
     observationId: "https://fhir.diz.uk-erlangen.de/identifiers/onkostar-xml-observation-id"
     procedureId: "https://fhir.diz.uk-erlangen.de/identifiers/onkostar-xml-procedure-id"
     medicationStatementId: "https://fhir.diz.uk-erlangen.de/identifiers/onkostar-xml-medication-statement-id"
-    observationCategorySystem: "http://terminology.hl7.org/CodeSystem/observation-category"
+    observationCategorySystem: "http://hl7.org/fhir/observation-category"
     loinc: "http://loinc.org"
     idco3Morphologie: "urn:oid:2.16.840.1.113883.6.43.1"
     gradingDktk: "http://dktk.dkfz.de/fhir/onco/core/CodeSystem/GradingCS"

src/test/java/org/miracum/streams/ume/obdstofhir/PackageSettings.java

@@ -0,0 +1,5 @@
+package org.miracum.streams.ume.obdstofhir;
+
+public class PackageSettings {
+  public static String ApprovalBaseDirectory = "./snapshots";
+}

src/test/java/org/miracum/streams/ume/obdstofhir/processor/ObdsConditionProcessorTest.java

@@ -2,43 +2,24 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.util.BundleUtil;
-import java.io.File;
 import java.io.IOException;
-import java.nio.file.Files;
 import java.util.*;
 import java.util.stream.Stream;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
+import org.approvaltests.Approvals;
 import org.hl7.fhir.r4.model.Condition;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.miracum.streams.ume.obdstofhir.FhirProperties;
 import org.miracum.streams.ume.obdstofhir.mapper.*;
-import org.miracum.streams.ume.obdstofhir.model.MeldungExport;
-import org.miracum.streams.ume.obdstofhir.model.MeldungExportList;
 import org.miracum.streams.ume.obdstofhir.model.Tupel;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.util.ResourceUtils;
-
-@SpringBootTest(
-    classes = {
-      FhirProperties.class,
-      ObdsConditionMapper.class,
-      ObdsMedicationStatementMapper.class,
-      ObdsObservationMapper.class,
-      ObdsProcedureMapper.class,
-      ObdsPatientMapper.class,
-      ObdsConditionMapper.class
-    })
-@EnableConfigurationProperties()
-public class ObdsConditionProcessorTest extends ObdsProcessorTest {
+
+class ObdsConditionProcessorTest extends ObdsProcessorTest {
 
   private static final Logger log = LoggerFactory.getLogger(ObdsConditionProcessorTest.class);
 
@@ -48,7 +29,6 @@ public class ObdsConditionProcessorTest extends ObdsProcessorTest {
   private final ObdsProcedureMapper onkoProcedureMapper;
   private final ObdsPatientMapper onkoPatientMapper;
   private final ObdsConditionMapper onkoConditionMapper;
-  private final FhirContext ctx = FhirContext.forR4();
 
   @Autowired
   public ObdsConditionProcessorTest(
@@ -123,31 +103,7 @@ void mapCondition_withGivenAdtXml(
       String expectedOnsetDate)
       throws IOException {
 
-    MeldungExportList meldungExportList = new MeldungExportList();
-
-    int payloadId = 1;
-
-    for (var xmlTupel : xmlFileNames) {
-      File xmlFile = ResourceUtils.getFile("classpath:" + xmlTupel.getFirst());
-      String xmlContent = new String(Files.readAllBytes(xmlFile.toPath()));
-
-      var meldungsId = StringUtils.substringBetween(xmlContent, "Meldung_ID=\"", "\" Melder_ID");
-      var melderId = StringUtils.substringBetween(xmlContent, "Melder_ID=\"", "\">");
-      var patId = StringUtils.substringBetween(xmlContent, "Patient_ID=\"", "\">");
-
-      Map<String, Object> payloadOnkoRessource = new HashMap<>();
-      payloadOnkoRessource.put("ID", payloadId);
-      payloadOnkoRessource.put("REFERENZ_NUMMER", patId);
-      payloadOnkoRessource.put("LKR_MELDUNG", Integer.parseInt(meldungsId.replace(melderId, "")));
-      payloadOnkoRessource.put("VERSIONSNUMMER", xmlTupel.getSecond());
-      payloadOnkoRessource.put("XML_DATEN", xmlContent);
-
-      MeldungExport meldungExport = new MeldungExport();
-      meldungExport.getPayload(payloadOnkoRessource);
-      meldungExportList.addElement(meldungExport);
-
-      payloadId++;
-    }
+    var meldungExportList = buildMeldungExportList(xmlFileNames);
 
     ObdsProcessor onkoProcessor =
         new ObdsProcessor(
@@ -192,7 +148,16 @@ void mapCondition_withGivenAdtXml(
       assertThat(conditionList.get(0).getOnsetDateTimeType().getValueAsString())
           .isEqualTo(expectedOnsetDate);
 
-      assertThat(isValid(resultBundle)).isTrue();
+      var fhirJson = fhirParser.encodeResourceToString(resultBundle);
+      Approvals.verify(
+          fhirJson,
+          Approvals.NAMES
+              .withParameters(
+                  xmlFileNames.stream()
+                      .map(t -> t.getFirst().substring(0, 5))
+                      .toArray(String[]::new))
+              .forFile()
+              .withExtension(".fhir.json"));
     }
   }
 }

src/test/java/org/miracum/streams/ume/obdstofhir/processor/ObdsMedicationStatementProcessorTest.java

@@ -4,41 +4,23 @@
 
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.util.BundleUtil;
-import java.io.File;
 import java.io.IOException;
-import java.nio.file.Files;
 import java.util.*;
 import java.util.stream.Stream;
-import org.apache.commons.lang3.StringUtils;
+import org.approvaltests.Approvals;
 import org.hl7.fhir.r4.model.CodeableConcept;
 import org.hl7.fhir.r4.model.MedicationStatement;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.miracum.streams.ume.obdstofhir.FhirProperties;
 import org.miracum.streams.ume.obdstofhir.mapper.*;
-import org.miracum.streams.ume.obdstofhir.model.MeldungExport;
-import org.miracum.streams.ume.obdstofhir.model.MeldungExportList;
 import org.miracum.streams.ume.obdstofhir.model.Tupel;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.util.ResourceUtils;
-
-@SpringBootTest(
-    classes = {
-      FhirProperties.class,
-      ObdsConditionMapper.class,
-      ObdsMedicationStatementMapper.class,
-      ObdsObservationMapper.class,
-      ObdsProcedureMapper.class,
-      ObdsPatientMapper.class,
-      ObdsConditionMapper.class
-    })
-@EnableConfigurationProperties()
-public class ObdsMedicationStatementProcessorTest extends ObdsProcessorTest {
+
+class ObdsMedicationStatementProcessorTest extends ObdsProcessorTest {
 
   private static final Logger log =
       LoggerFactory.getLogger(ObdsMedicationStatementProcessorTest.class);
@@ -101,31 +83,7 @@ void mapMedicationStatement_withGivenAdtXml(
       String expectedIntention)
       throws IOException {
 
-    MeldungExportList meldungExportList = new MeldungExportList();
-
-    int payloadId = 1;
-
-    for (var xmlTupel : xmlFileNames) {
-      File xmlFile = ResourceUtils.getFile("classpath:" + xmlTupel.getFirst());
-      String xmlContent = new String(Files.readAllBytes(xmlFile.toPath()));
-
-      var meldungsId = StringUtils.substringBetween(xmlContent, "Meldung_ID=\"", "\" Melder_ID");
-      var melderId = StringUtils.substringBetween(xmlContent, "Melder_ID=\"", "\">");
-      var patId = StringUtils.substringBetween(xmlContent, "Patient_ID=\"", "\">");
-
-      Map<String, Object> payloadOnkoRessource = new HashMap<>();
-      payloadOnkoRessource.put("ID", payloadId);
-      payloadOnkoRessource.put("REFERENZ_NUMMER", patId);
-      payloadOnkoRessource.put("LKR_MELDUNG", Integer.parseInt(meldungsId.replace(melderId, "")));
-      payloadOnkoRessource.put("VERSIONSNUMMER", xmlTupel.getSecond());
-      payloadOnkoRessource.put("XML_DATEN", xmlContent);
-
-      MeldungExport meldungExport = new MeldungExport();
-      meldungExport.getPayload(payloadOnkoRessource);
-      meldungExportList.addElement(meldungExport);
-
-      payloadId++;
-    }
+    var meldungExportList = buildMeldungExportList(xmlFileNames);
 
     ObdsProcessor medicationStatementProcessor =
         new ObdsProcessor(
@@ -186,7 +144,16 @@ void mapMedicationStatement_withGivenAdtXml(
       String finalPartOfId = partOfId;
       assertThat(partOfReferences).allSatisfy(ref -> ref.equals(finalPartOfId));
 
-      assertThat(isValid(resultBundle)).isTrue();
+      var fhirJson = fhirParser.encodeResourceToString(resultBundle);
+      Approvals.verify(
+          fhirJson,
+          Approvals.NAMES
+              .withParameters(
+                  xmlFileNames.stream()
+                      .map(t -> t.getFirst().substring(0, 5))
+                      .toArray(String[]::new))
+              .forFile()
+              .withExtension(".fhir.json"));
     }
   }
 }