Skip to content
This repository has been archived by the owner on Jul 11, 2024. It is now read-only.

Implement disk cache #34

Open
wants to merge 3 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions executor/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,24 @@
<version>4.5.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<version>1.17.4</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>1.17.4</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.4.5</version>
<scope>test</scope>
</dependency>
</dependencies>

</project>
173 changes: 173 additions & 0 deletions executor/src/test/java/CacheIT.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
import de.rwth.imi.flare.api.model.*;
import de.rwth.imi.flare.api.model.mapping.AttributeSearchParameter;
import de.rwth.imi.flare.api.model.mapping.MappingEntry;
import de.rwth.imi.flare.executor.AuthlessRequestorConfig;
import de.rwth.imi.flare.executor.FlareExecutor;
import de.rwth.imi.flare.requestor.*;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;

import org.testcontainers.containers.FixedHostPortGenericContainer;
import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.images.PullPolicy;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;

import java.io.File;
import java.io.IOException;

import java.net.URI;
import java.net.URISyntaxException;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors;


@Slf4j
@Testcontainers
public class CacheIT {

FlareExecutor executor;
FhirRequestorConfig config;
private final int malesToGenerate =15;
private final int femalesToGenerate = 1000;
private String baseFhirUri;
private int idCounter = 0;
private String singlePatientTemplate;

@Container
private final FixedHostPortGenericContainer<?> fhirContainer = new FixedHostPortGenericContainer<>("samply/blaze:0.18")
.withImagePullPolicy(PullPolicy.alwaysPull())
.withFixedExposedPort(8080, 8080)
.waitingFor(Wait.forHttp("/health").forStatusCode(200))
.withLogConsumer(new Slf4jLogConsumer(log))
.withEnv("LOG_LEVEL", "debug")
.withStartupAttempts(5);

@Test
public void mainCacheIntegrationTest() throws Exception {
baseFhirUri = "http://localhost:" + fhirContainer.getMappedPort(8080) + "/fhir" ;
//baseFhirUri = "http://localhost:8082/fhir"; //overwrite baseFhirUri like this if you want to use your own fhir server
singlePatientTemplate = loadSinglePatientTemplate();
createExecutor();

try{
uploadTestData();
}catch(IOException e){
e.printStackTrace();
}

QueryExpanded query = buildGenderQuery("female");

long startTime1 = System.nanoTime();
CompletableFuture<Integer> compFuture1 = executor.calculatePatientCount(query);
int patientCount1 = compFuture1.get();
float duration1 = (System.nanoTime() - startTime1) / 1000000.f;
assertEquals(femalesToGenerate, patientCount1);

long startTime2 = System.nanoTime();
CompletableFuture<Integer> compFuture2 = executor.calculatePatientCount(query);
int patientCount2 = compFuture2.get();
float duration2 = (System.nanoTime() - startTime2) / 1000000.f;
assertEquals(femalesToGenerate, patientCount2);

System.out.println("time not cached: " + duration1 + "ms");
System.out.println("time cached: " + duration2 + "ms");


//TODO upload lots of patients to test how long it takes to retrieve lots of patient ids from disk
}


private void uploadTestData() throws IOException, InterruptedException {

for(int i = 0; i < malesToGenerate; i++){
String newPatient = generateSinglePatient("male");
postPatient(newPatient);
}

for(int i = 0; i < femalesToGenerate; i++){
String newPatient = generateSinglePatient("female");
postPatient(newPatient);
}
}

private void postPatient(String patient) throws IOException, InterruptedException {
HttpClient client = HttpClient.newHttpClient();
patient = patient.replace("\n", "").replace("\r", "");
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(baseFhirUri))
.POST(HttpRequest.BodyPublishers.ofString(patient))
.header("Content-Type", "application/fhir+json")
.build();

client.send(request,
HttpResponse.BodyHandlers.ofString());
}

private String loadSinglePatientTemplate() throws IOException {
String baseFilePath = new File("").getAbsolutePath();
Path filePath = Path.of(baseFilePath + "/src/test/java/single-patient-template.json");
return Files.readString(filePath);
}

private String generateSinglePatient(String gender) {
String newPatient = singlePatientTemplate;
newPatient = newPatient.replace("pat-generated-id", String.valueOf(idCounter));
newPatient = newPatient.replace("con-generated-id", String.valueOf(idCounter));
newPatient = newPatient.replace("obs-generated-id", String.valueOf(idCounter));
newPatient = newPatient.replace("enc-generated-id", String.valueOf(idCounter));
newPatient = newPatient.replace("<gender>", gender);
newPatient = newPatient.replace("<birthdate>", "1990-01-01");
idCounter++;
return newPatient;
}

public void createExecutor() throws URISyntaxException {
config = new AuthlessRequestorConfig(new URI(baseFhirUri + "/"), "50", new FlareThreadPoolConfig(4,16,10));
executor = new FlareExecutor(new FhirRequestor(config, Executors.newFixedThreadPool(16)));
}

private QueryExpanded buildGenderQuery(String gender) {

Criterion criterion1 = new Criterion();
List<TerminologyCode> femaleTerminology = Arrays.asList(new TerminologyCode("gender", "mii.abide", "Geschlecht"));
criterion1.setTermCodes(femaleTerminology);

MappingEntry mapping = new MappingEntry();
AttributeSearchParameter attributeSearchParameter = new AttributeSearchParameter();
attributeSearchParameter.setAttributeKey(new TerminologyCode("gender", "mii.abide", "Geschlecht"));
attributeSearchParameter.setAttributeFhirPath("gender");
attributeSearchParameter.setAttributeType("code");
attributeSearchParameter.setAttributeSearchParameter("gender");
mapping.setAttributeSearchParameters(List.of(attributeSearchParameter));
mapping.setFhirResourceType("Patient");
criterion1.setMapping(mapping);

AttributeFilter femaleAttributeFilter = new AttributeFilter();
femaleAttributeFilter.setType(FilterType.CONCEPT);
TerminologyCode attributeConcept = new TerminologyCode(gender, "http://hl7.org/fhir/administrative-gender", "someDisplay");
femaleAttributeFilter.setSelectedConcepts(List.of(attributeConcept));
TerminologyCode femaleAttributeCode = new TerminologyCode("gender", "mii.abide", "Geschlecht");
femaleAttributeFilter.setAttributeCode(femaleAttributeCode);
List<AttributeFilter> attributeFilters = List.of(femaleAttributeFilter);
criterion1.setAttributeFilters(attributeFilters);

CriteriaGroup criteriaGroup1 = new CriteriaGroup(List.of(criterion1));
Query expectedResult = new Query();
expectedResult.setInclusionCriteria(List.of(criteriaGroup1));
QueryExpanded parsedQuery = new QueryExpanded();
parsedQuery.setInclusionCriteria(expectedResult.getInclusionCriteria());

return parsedQuery;
}

}
14 changes: 1 addition & 13 deletions executor/src/test/java/ExecutorTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
import de.rwth.imi.flare.api.model.mapping.MappingEntry;
import de.rwth.imi.flare.executor.AuthlessRequestorConfig;
import de.rwth.imi.flare.executor.FlareExecutor;
import de.rwth.imi.flare.requestor.CacheConfig;
import de.rwth.imi.flare.requestor.FhirRequestor;
import de.rwth.imi.flare.requestor.FhirRequestorConfig;
import de.rwth.imi.flare.requestor.FlareThreadPoolConfig;
Expand All @@ -26,18 +25,7 @@ public class ExecutorTest
public ExecutorTest() throws URISyntaxException {
config = new AuthlessRequestorConfig(new URI("http://localhost:8080/fhir/"), "50", new FlareThreadPoolConfig(4,16,10));

CacheConfig cacheConfig = new CacheConfig() {
@Override
public int getCacheSizeInMb() {
return 100;
}

@Override
public int getEntryRefreshTimeHours() {
return 1;
}
};
executor = new FlareExecutor(new FhirRequestor(config, cacheConfig, Executors.newFixedThreadPool(16)));
executor = new FlareExecutor(new FhirRequestor(config, Executors.newFixedThreadPool(16)));
}

@Test
Expand Down
15 changes: 2 additions & 13 deletions executor/src/test/java/ExecutorTests.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import de.rwth.imi.flare.executor.AuthlessRequestorConfig;
import de.rwth.imi.flare.executor.FhirIdRequestor;
import de.rwth.imi.flare.executor.FlareExecutor;
import de.rwth.imi.flare.requestor.CacheConfig;
import de.rwth.imi.flare.requestor.FhirRequestor;
import de.rwth.imi.flare.requestor.FlareThreadPoolConfig;
import java.util.concurrent.Executors;
Expand Down Expand Up @@ -59,18 +58,8 @@ void setUp() throws URISyntaxException {
AuthlessRequestorConfig config = new AuthlessRequestorConfig(
new URI("http://localhost:8080/fhir/"),
"50", new FlareThreadPoolConfig(4, 16, 10));
CacheConfig cacheConfig = new CacheConfig() {
@Override
public int getCacheSizeInMb() {
return 100;
}

@Override
public int getEntryRefreshTimeHours() {
return 1;
}
};
flareExecutor = new FlareExecutor(new FhirRequestor(config, cacheConfig, Executors.newFixedThreadPool(16)));

flareExecutor = new FlareExecutor(new FhirRequestor(config, Executors.newFixedThreadPool(16)));
queryExpanded = getQueryExpanded();
}

Expand Down
1 change: 1 addition & 0 deletions executor/src/test/java/single-patient-template.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"resourceType":"Bundle","type":"transaction","entry":[{"fullUrl":"Patient/pat-generated-id","resource":{"resourceType":"Patient","id":"pat-generated-id","meta":{"profile":["https://www.medizininformatik-initiative.de/fhir/core/modul-person/StructureDefinition/Patient"]},"identifier":[{"use":"usual","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v2-0203","code":"MR"}]},"system":"https://UKFAU.de/pid","value":"pat-generated-id"}],"name":[{"use":"official","family":"y","given":["x"]}],"gender":"<gender>","birthDate":"1990-01-01"},"request":{"method":"PUT","url":"Patient/pat-generated-id"}},{"fullUrl":"Encounter/enc-generated-id","resource":{"resourceType":"Encounter","id":"enc-generated-id","meta":{"profile":["https://www.medizininformatik-initiative.de/fhir/core/modul-fall/StructureDefinition/KontaktGesundheitseinrichtung"]},"identifier":[{"type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v2-0203","code":"VN"}]},"system":"http://dummyurl","value":"enc-generated-id","assigner":{"identifier":{"system":"https://www.medizininformatik-initiative.de/fhir/core/NamingSystem/org-identifier","value":"UKFAU"}}}],"status":"finished","class":{"system":"http://terminology.hl7.org/CodeSystem/v3-ActCode","code":"IMP","display":"inpatient encounter"},"subject":{"reference":"Patient/pat-generated-id","identifier":{"value":"pat-generated-id"}},"period":{"start":"2020-05-13T00:00:00+02:00","end":"2020-05-18T00:00:00+02:00"}},"request":{"method":"PUT","url":"Encounter/enc-generated-id"}},{"fullUrl":"Condition/con-generated-id","resource":{"resourceType":"Condition","id":"con-generated-id","meta":{"profile":["https://www.medizininformatik-initiative.de/fhir/core/modul-diagnose/StructureDefinition/Diagnose"]},"identifier":[{"value":"con-generated-id","system":"https://UKE.de/condition"}],"code":{"coding":[{"system":"http://fhir.de/CodeSystem/bfarm/icd-10-gm","version":"2020","code":"C50.1"}]},"subject":{"reference":"Patient/pat-generated-id","identifier":{"value":"pat-generated-id"}},"recordedDate":"2018-07-09T02:30:51+02:00","encounter":{"reference":"Encounter/enc-generated-id"},"onsetDateTime":"2018-07-09T02:30:51+02:00"},"request":{"method":"PUT","url":"Condition/con-generated-id"}},{"fullUrl":"Observation/obs-generated-id","resource":{"resourceType":"Observation","id":"obs-generated-id","meta":{"profile":["https://www.medizininformatik-initiative.de/fhir/core/modul-labor/StructureDefinition/ObservationLab"]},"identifier":[{"type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v2-0203","code":"OBI"}]},"system":"https://UKE.de/befund","value":"obs-generated-id","assigner":{"identifier":{"system":"https://www.medizininformatik-initiative.de/fhir/core/NamingSystem/org-identifier","value":"UKE"}}}],"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"laboratory","display":"Laboratory"},{"system":"http://loinc.org","code":"26436-6","display":"Laboratory studies (set)"}]}],"code":{"coding":[{"system":"http://loinc.org","code":"55782-7"}]},"subject":{"reference":"Patient/pat-generated-id","identifier":{"value":"pat-generated-id"}},"encounter":{"reference":"Encounter/enc-generated-id","identifier":{"value":"enc-generated-id"}},"effectiveDateTime":"2019-04-12T05:33:27+02:00","valueQuantity":{"value":11.9914,"unit":"gram per deciliter","system":"http://unitsofmeasure.org","code":"g/dL"}},"request":{"method":"PUT","url":"Observation/obs-generated-id"}}]}
15 changes: 15 additions & 0 deletions executor/src/test/resources/logback-test.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n</pattern>
</encoder>
</appender>

<root level="info">
<appender-ref ref="STDOUT"/>
</root>

<logger name="de.rwth.imi.flare" level="DEBUG"/>
<logger name="org.testcontainers" level="INFO"/>
<logger name="com.github.dockerjava" level="WARN"/>
</configuration>
15 changes: 1 addition & 14 deletions flare-cli/src/main/java/de/rwth/imi/flare/cli/CLI.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
import de.rwth.imi.flare.mapping.lookup.NaiveLookupMapping;
import de.rwth.imi.flare.mapping.lookup.SourceMappingEntry;
import de.rwth.imi.flare.parser.i2b2.ParserI2B2;
import de.rwth.imi.flare.requestor.CacheConfig;
import de.rwth.imi.flare.requestor.FhirRequestor;
import de.rwth.imi.flare.requestor.FhirRequestorConfig;
import de.rwth.imi.flare.requestor.FlareThreadPoolConfig;
Expand Down Expand Up @@ -136,19 +135,7 @@ public FlareThreadPoolConfig getThreadPoolConfig() {

};

CacheConfig cacheConfig = new CacheConfig() {
@Override
public int getCacheSizeInMb() {
return 100;
}

@Override
public int getEntryRefreshTimeHours() {
return 1;
}
};

executor = new FlareExecutor(new FhirRequestor(config, cacheConfig, Executors.newFixedThreadPool(16)));
executor = new FlareExecutor(new FhirRequestor(config, Executors.newFixedThreadPool(16)));
}

@Nullable
Expand Down
17 changes: 11 additions & 6 deletions requestor/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,6 @@
<version>5.7.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
<version>3.1.1</version>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
Expand All @@ -75,7 +70,17 @@
<artifactId>testcontainers</artifactId>
<version>1.17.4</version>
</dependency>

<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-jcs3-core</artifactId>
<version>3.0</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
<version>5.3.10</version>
<scope>test</scope>
</dependency>
</dependencies>

</project>
Loading