From cc440d7857dd183a1f1ad509d2fa55f790a85246 Mon Sep 17 00:00:00 2001 From: luca Date: Tue, 19 Dec 2023 15:03:27 +0100 Subject: [PATCH 1/4] task: add gzip/unzip support to tracker/events --- .../events/event/csv/CsvEventService.java | 6 + .../event/csv/DefaultCsvEventService.java | 13 + .../TrackerEventExportTestConfiguration.java | 47 ++ ...TrackerEventsExportControllerByIdTest.java | 480 +++++++++++++++++ .../TrackerEventsExportControllerTest.java | 492 ++++-------------- .../tracker/export/CompressionUtil.java | 77 +++ .../export/TrackerEventsExportController.java | 145 +++++- .../export/csv/TrackerCsvEventService.java | 31 +- .../csv/TrackerCsvTrackedEntityService.java | 33 +- .../hisp/dhis/webapi/utils/ContextUtils.java | 3 +- .../tracker/export/CompressionUtilTest.java | 151 ++++++ 11 files changed, 1054 insertions(+), 424 deletions(-) create mode 100644 dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventExportTestConfiguration.java create mode 100644 dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventsExportControllerByIdTest.java create mode 100644 dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtil.java create mode 100644 dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtilTest.java diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/CsvEventService.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/CsvEventService.java index 25f63c322c80..913f2a99b73b 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/CsvEventService.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/CsvEventService.java @@ -39,6 +39,12 @@ public interface CsvEventService { void writeEvents(OutputStream outputStream, List events, boolean withHeader) throws IOException; + void writeZip(OutputStream outputStream, List toCompress, boolean withHeader, String file) + throws IOException; + + void writeGzip(OutputStream outputStream, List toCompress, boolean withHeader) + throws IOException; + List readEvents(InputStream inputStream, boolean skipFirst) throws IOException, org.locationtech.jts.io.ParseException; } diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java index 6373ef16024b..78903e0252c6 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java @@ -108,6 +108,19 @@ public void writeEvents(OutputStream outputStream, List events, boolean w writer.writeValue(outputStream, dataValues); } + @Override + public void writeZip( + OutputStream outputStream, List toCompress, boolean withHeader, String file) + throws IOException { + // in use only in the new tracker endpoints + } + + @Override + public void writeGzip(OutputStream outputStream, List toCompress, boolean withHeader) + throws IOException { + // in use only in the new tracker endpoints + } + @Override public List readEvents(InputStream inputStream, boolean skipFirst) throws IOException, ParseException { diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventExportTestConfiguration.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventExportTestConfiguration.java new file mode 100644 index 000000000000..f4a567c3974c --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventExportTestConfiguration.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.tracker.export; + +import static org.mockito.Mockito.mock; + +import org.hisp.dhis.dxf2.events.event.EventService; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.context.annotation.Profile; + +@Configuration +class TrackerEventExportTestConfiguration { + + @Primary + @Profile("TrackerEventExport") + @Bean + EventService eventService() { + return mock(EventService.class); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventsExportControllerByIdTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventsExportControllerByIdTest.java new file mode 100644 index 000000000000..c0a41acaf20b --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventsExportControllerByIdTest.java @@ -0,0 +1,480 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.tracker.export; + +import static org.hisp.dhis.webapi.controller.tracker.JsonAssertions.assertFirstRelationship; +import static org.hisp.dhis.webapi.controller.tracker.JsonAssertions.assertHasMember; +import static org.hisp.dhis.webapi.controller.tracker.JsonAssertions.assertHasNoMember; +import static org.hisp.dhis.webapi.controller.tracker.JsonAssertions.assertHasOnlyMembers; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Date; +import java.util.Set; +import org.hisp.dhis.common.CodeGenerator; +import org.hisp.dhis.common.IdentifiableObjectManager; +import org.hisp.dhis.common.ValueType; +import org.hisp.dhis.dataelement.DataElement; +import org.hisp.dhis.eventdatavalue.EventDataValue; +import org.hisp.dhis.jsontree.JsonList; +import org.hisp.dhis.jsontree.JsonObject; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.program.Program; +import org.hisp.dhis.program.ProgramInstance; +import org.hisp.dhis.program.ProgramStage; +import org.hisp.dhis.program.ProgramStageInstance; +import org.hisp.dhis.program.ProgramStatus; +import org.hisp.dhis.program.UserInfoSnapshot; +import org.hisp.dhis.relationship.Relationship; +import org.hisp.dhis.relationship.RelationshipEntity; +import org.hisp.dhis.relationship.RelationshipItem; +import org.hisp.dhis.relationship.RelationshipType; +import org.hisp.dhis.security.acl.AccessStringHelper; +import org.hisp.dhis.trackedentity.TrackedEntityInstance; +import org.hisp.dhis.trackedentity.TrackedEntityType; +import org.hisp.dhis.user.User; +import org.hisp.dhis.user.sharing.UserAccess; +import org.hisp.dhis.web.HttpStatus; +import org.hisp.dhis.webapi.DhisControllerConvenienceTest; +import org.hisp.dhis.webapi.controller.tracker.JsonRelationship; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +class TrackerEventsExportControllerByIdTest extends DhisControllerConvenienceTest { + + @Autowired private IdentifiableObjectManager manager; + + private OrganisationUnit orgUnit; + + private OrganisationUnit anotherOrgUnit; + + private Program program; + + private ProgramStage programStage; + + private User owner; + + private User user; + + private TrackedEntityType trackedEntityType; + + @BeforeEach + void setUp() { + owner = makeUser("owner"); + + orgUnit = createOrganisationUnit('A'); + orgUnit.getSharing().setOwner(owner); + manager.save(orgUnit, false); + + anotherOrgUnit = createOrganisationUnit('B'); + anotherOrgUnit.getSharing().setOwner(owner); + manager.save(anotherOrgUnit, false); + + user = createUserWithId("tester", CodeGenerator.generateUid()); + user.addOrganisationUnit(orgUnit); + user.setTeiSearchOrganisationUnits(Set.of(orgUnit)); + this.userService.updateUser(user); + + program = createProgram('A'); + program.addOrganisationUnit(orgUnit); + program.getSharing().setOwner(owner); + program.getSharing().addUserAccess(userAccess()); + manager.save(program, false); + + programStage = createProgramStage('A', program); + programStage.getSharing().setOwner(owner); + programStage.getSharing().addUserAccess(userAccess()); + manager.save(programStage, false); + + trackedEntityType = trackedEntityTypeAccessible(); + } + + @Test + void getEventById() { + TrackedEntityInstance to = trackedEntityInstance(); + ProgramStageInstance from = programStageInstance(programInstance(to)); + relationship(from, to); + + JsonObject json = GET("/tracker/events/{id}", from.getUid()).content(HttpStatus.OK); + + assertDefaultResponse(json, from); + } + + @Test + void getEventByIdWithFields() { + TrackedEntityInstance tei = trackedEntityInstance(); + ProgramStageInstance event = programStageInstance(programInstance(tei)); + + JsonObject json = + GET("/tracker/events/{id}?fields=orgUnit,status", event.getUid()).content(HttpStatus.OK); + + assertFalse(json.isEmpty()); + assertHasOnlyMembers(json, "orgUnit", "status"); + } + + @Test + void getEventByIdWithFieldsRelationships() { + TrackedEntityInstance to = trackedEntityInstance(); + ProgramStageInstance from = programStageInstance(programInstance(to)); + Relationship r = relationship(from, to); + + JsonList relationships = + GET("/tracker/events/{id}?fields=relationships", from.getUid()) + .content(HttpStatus.OK) + .getList("relationships", JsonRelationship.class); + + assertFalse(relationships.isEmpty()); + assertEquals(1, relationships.size()); + JsonRelationship relationship = assertFirstRelationship(r, relationships); + assertEventWithinRelationship(from, relationship.getFrom()); + assertTrackedEntityWithinRelationship(to, relationship.getTo()); + } + + @Test + void getEventByIdRelationshipsNoAccessToRelationshipType() { + TrackedEntityInstance to = trackedEntityInstance(); + ProgramStageInstance from = programStageInstance(programInstance(to)); + relationship(relationshipTypeNotAccessible(), from, to); + this.switchContextToUser(user); + + JsonList relationships = + GET("/tracker/events/{id}?fields=relationships", from.getUid()) + .content(HttpStatus.OK) + .getList("relationships", JsonRelationship.class); + + assertEquals(0, relationships.size()); + } + + @Test + void getEventByIdRelationshipsNoAccessToRelationshipItemTo() { + TrackedEntityType type = trackedEntityTypeNotAccessible(); + TrackedEntityInstance to = trackedEntityInstance(type); + ProgramStageInstance from = programStageInstance(programInstance(to)); + relationship(from, to); + this.switchContextToUser(user); + + JsonList relationships = + GET("/tracker/events/{id}?fields=relationships", from.getUid()) + .content(HttpStatus.OK) + .getList("relationships", JsonRelationship.class); + + assertEquals(0, relationships.size()); + } + + @Test + void getEventByIdRelationshipsNoAccessToBothRelationshipItems() { + TrackedEntityInstance to = trackedEntityInstanceNotInSearchScope(); + ProgramStageInstance from = programStageInstance(programInstance(to)); + relationship(from, to); + this.switchContextToUser(user); + + assertTrue( + GET("/tracker/events/{id}", from.getUid()) + .error(HttpStatus.CONFLICT) + .getMessage() + .contains("OWNERSHIP_ACCESS_DENIED")); + } + + @Test + void getEventByIdRelationshipsNoAccessToRelationshipItemFrom() { + TrackedEntityType type = trackedEntityTypeNotAccessible(); + TrackedEntityInstance from = trackedEntityInstance(type); + ProgramStageInstance to = programStageInstance(programInstance(from)); + relationship(from, to); + this.switchContextToUser(user); + + JsonList relationships = + GET("/tracker/events/{id}?fields=relationships", to.getUid()) + .content(HttpStatus.OK) + .getList("relationships", JsonRelationship.class); + + assertEquals(0, relationships.size()); + } + + @Test + void getEventByIdContainsCreatedByAndUpdateByAndAssignedUserInDataValues() { + + TrackedEntityInstance tei = trackedEntityInstance(); + ProgramInstance programInstance = programInstance(tei); + ProgramStageInstance programStageInstance = programStageInstance(programInstance); + programStageInstance.setCreatedByUserInfo(UserInfoSnapshot.from(user)); + programStageInstance.setLastUpdatedByUserInfo(UserInfoSnapshot.from(user)); + programStageInstance.setAssignedUser(user); + EventDataValue eventDataValue = new EventDataValue(); + eventDataValue.setValue("6"); + DataElement dataElement = createDataElement('A'); + dataElement.setValueType(ValueType.NUMBER); + manager.save(dataElement); + eventDataValue.setDataElement(dataElement.getUid()); + eventDataValue.setCreatedByUserInfo(UserInfoSnapshot.from(user)); + eventDataValue.setLastUpdatedByUserInfo(UserInfoSnapshot.from(user)); + Set eventDataValues = Set.of(eventDataValue); + programStageInstance.setEventDataValues(eventDataValues); + manager.save(programStageInstance); + + JsonObject event = + GET("/tracker/events/{id}", programStageInstance.getUid()).content(HttpStatus.OK); + + assertTrue(event.isObject()); + assertFalse(event.isEmpty()); + assertEquals(programStageInstance.getUid(), event.getString("event").string()); + assertEquals(programInstance.getUid(), event.getString("enrollment").string()); + assertEquals(orgUnit.getUid(), event.getString("orgUnit").string()); + assertEquals(user.getUsername(), event.getString("createdBy.username").string()); + assertEquals(user.getUsername(), event.getString("updatedBy.username").string()); + assertEquals(user.getDisplayName(), event.getString("assignedUser.displayName").string()); + assertFalse(event.getArray("dataValues").isEmpty()); + assertEquals( + user.getUsername(), + event.getArray("dataValues").getObject(0).getString("createdBy.username").string()); + assertEquals( + user.getUsername(), + event.getArray("dataValues").getObject(0).getString("updatedBy.username").string()); + } + + @Test + void getEventByIdNotFound() { + assertEquals( + "Event with id Hq3Kc6HK4OZ could not be found.", + GET("/tracker/events/Hq3Kc6HK4OZ").error(HttpStatus.NOT_FOUND).getMessage()); + } + + private TrackedEntityType trackedEntityTypeAccessible() { + TrackedEntityType type = trackedEntityType('A'); + type.getSharing().addUserAccess(userAccess()); + manager.save(type, false); + return type; + } + + private TrackedEntityType trackedEntityTypeNotAccessible() { + TrackedEntityType type = trackedEntityType('B'); + manager.save(type, false); + return type; + } + + private TrackedEntityType trackedEntityType(char uniqueChar) { + TrackedEntityType type = createTrackedEntityType(uniqueChar); + type.getSharing().setOwner(owner); + type.getSharing().setPublicAccess(AccessStringHelper.DEFAULT); + return type; + } + + private TrackedEntityInstance trackedEntityInstance() { + TrackedEntityInstance tei = trackedEntityInstance(orgUnit); + manager.save(tei, false); + return tei; + } + + private TrackedEntityInstance trackedEntityInstanceNotInSearchScope() { + TrackedEntityInstance tei = trackedEntityInstance(anotherOrgUnit); + manager.save(tei, false); + return tei; + } + + private TrackedEntityInstance trackedEntityInstance(TrackedEntityType trackedEntityType) { + TrackedEntityInstance tei = trackedEntityInstance(orgUnit, trackedEntityType); + manager.save(tei, false); + return tei; + } + + private TrackedEntityInstance trackedEntityInstance(OrganisationUnit orgUnit) { + return trackedEntityInstance(orgUnit, trackedEntityType); + } + + private TrackedEntityInstance trackedEntityInstance( + OrganisationUnit orgUnit, TrackedEntityType trackedEntityType) { + TrackedEntityInstance tei = createTrackedEntityInstance(orgUnit); + tei.setTrackedEntityType(trackedEntityType); + tei.getSharing().setPublicAccess(AccessStringHelper.DEFAULT); + tei.getSharing().setOwner(owner); + return tei; + } + + private ProgramInstance programInstance(TrackedEntityInstance tei) { + ProgramInstance programInstance = new ProgramInstance(program, tei, tei.getOrganisationUnit()); + programInstance.setAutoFields(); + programInstance.setEnrollmentDate(new Date()); + programInstance.setIncidentDate(new Date()); + programInstance.setStatus(ProgramStatus.COMPLETED); + manager.save(programInstance); + return programInstance; + } + + private ProgramStageInstance programStageInstance(ProgramInstance programInstance) { + ProgramStageInstance programStageInstance = + new ProgramStageInstance( + programInstance, programStage, programInstance.getOrganisationUnit()); + programStageInstance.setAutoFields(); + manager.save(programStageInstance); + return programStageInstance; + } + + private UserAccess userAccess() { + UserAccess a = new UserAccess(); + a.setUser(user); + a.setAccess(AccessStringHelper.FULL); + return a; + } + + private RelationshipType relationshipTypeAccessible( + RelationshipEntity from, RelationshipEntity to) { + RelationshipType type = relationshipType(from, to); + type.getSharing().addUserAccess(userAccess()); + manager.save(type, false); + return type; + } + + private RelationshipType relationshipTypeNotAccessible() { + return relationshipType( + RelationshipEntity.PROGRAM_STAGE_INSTANCE, RelationshipEntity.TRACKED_ENTITY_INSTANCE); + } + + private RelationshipType relationshipType(RelationshipEntity from, RelationshipEntity to) { + RelationshipType type = createRelationshipType('A'); + type.getFromConstraint().setRelationshipEntity(from); + type.getToConstraint().setRelationshipEntity(to); + type.getSharing().setOwner(owner); + type.getSharing().setPublicAccess(AccessStringHelper.DEFAULT); + manager.save(type, false); + return type; + } + + private Relationship relationship(ProgramStageInstance from, TrackedEntityInstance to) { + return relationship( + relationshipTypeAccessible( + RelationshipEntity.PROGRAM_STAGE_INSTANCE, RelationshipEntity.TRACKED_ENTITY_INSTANCE), + from, + to); + } + + private Relationship relationship( + RelationshipType type, ProgramStageInstance from, TrackedEntityInstance to) { + Relationship r = new Relationship(); + + RelationshipItem fromItem = new RelationshipItem(); + fromItem.setProgramStageInstance(from); + from.getRelationshipItems().add(fromItem); + fromItem.setRelationship(r); + r.setFrom(fromItem); + + RelationshipItem toItem = new RelationshipItem(); + toItem.setTrackedEntityInstance(to); + to.getRelationshipItems().add(toItem); + r.setTo(toItem); + toItem.setRelationship(r); + + r.setRelationshipType(type); + r.setKey(type.getUid()); + r.setInvertedKey(type.getUid()); + r.setAutoFields(); + r.getSharing().setOwner(owner); + manager.save(r, false); + return r; + } + + private Relationship relationship(TrackedEntityInstance from, ProgramStageInstance to) { + Relationship r = new Relationship(); + + RelationshipItem fromItem = new RelationshipItem(); + fromItem.setTrackedEntityInstance(from); + from.getRelationshipItems().add(fromItem); + r.setFrom(fromItem); + fromItem.setRelationship(r); + + RelationshipItem toItem = new RelationshipItem(); + toItem.setProgramStageInstance(to); + to.getRelationshipItems().add(toItem); + r.setTo(toItem); + toItem.setRelationship(r); + + RelationshipType type = + relationshipTypeAccessible( + RelationshipEntity.PROGRAM_STAGE_INSTANCE, RelationshipEntity.TRACKED_ENTITY_INSTANCE); + r.setRelationshipType(type); + r.setKey(type.getUid()); + r.setInvertedKey(type.getUid()); + + r.setAutoFields(); + r.getSharing().setOwner(owner); + manager.save(r, false); + return r; + } + + private void assertEventWithinRelationship(ProgramStageInstance expected, JsonObject json) { + JsonObject jsonEvent = json.getObject("event"); + assertFalse(jsonEvent.isEmpty(), "event should not be empty"); + assertEquals(expected.getUid(), jsonEvent.getString("event").string(), "event UID"); + assertFalse(jsonEvent.has("status")); + assertFalse(jsonEvent.has("orgUnit")); + assertFalse(jsonEvent.has("programStage")); + assertFalse( + jsonEvent.has("relationships"), "relationships is not returned within relationship items"); + } + + private void assertTrackedEntityWithinRelationship( + TrackedEntityInstance expected, JsonObject json) { + JsonObject jsonTEI = json.getObject("trackedEntity"); + assertFalse(jsonTEI.isEmpty(), "trackedEntity should not be empty"); + assertEquals( + expected.getUid(), jsonTEI.getString("trackedEntity").string(), "trackedEntity UID"); + assertFalse(jsonTEI.has("trackedEntityType")); + assertFalse(jsonTEI.has("orgUnit")); + assertFalse( + jsonTEI.has("relationships"), "relationships is not returned within relationship items"); + assertTrue(jsonTEI.getArray("attributes").isEmpty()); + } + + private void assertDefaultResponse(JsonObject json, ProgramStageInstance programStageInstance) { + // note that some fields are not included in the response because they + // are not part of the setup + // i.e attributeOptionCombo, ... + assertTrue(json.isObject()); + assertFalse(json.isEmpty()); + assertEquals(programStageInstance.getUid(), json.getString("event").string(), "event UID"); + assertEquals("ACTIVE", json.getString("status").string()); + assertEquals(program.getUid(), json.getString("program").string()); + assertEquals(programStage.getUid(), json.getString("programStage").string()); + assertEquals( + programStageInstance.getProgramInstance().getUid(), json.getString("enrollment").string()); + assertEquals(orgUnit.getUid(), json.getString("orgUnit").string()); + assertEquals(orgUnit.getName(), json.getString("orgUnitName").string()); + assertFalse(json.getBoolean("followup").booleanValue()); + assertFalse(json.getBoolean("deleted").booleanValue()); + assertHasMember(json, "createdAt"); + assertHasMember(json, "createdAtClient"); + assertHasMember(json, "updatedAt"); + assertHasMember(json, "updatedAtClient"); + assertHasMember(json, "dataValues"); + assertHasMember(json, "notes"); + assertHasNoMember(json, "attributeOptionCombo"); + assertHasNoMember(json, "attributeCategoryOptions"); + assertHasNoMember(json, "relationships"); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventsExportControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventsExportControllerTest.java index 3402d0699263..abe3c6253875 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventsExportControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventsExportControllerTest.java @@ -27,73 +27,55 @@ */ package org.hisp.dhis.webapi.controller.tracker.export; -import static org.hisp.dhis.webapi.controller.tracker.JsonAssertions.assertFirstRelationship; -import static org.hisp.dhis.webapi.controller.tracker.JsonAssertions.assertHasMember; -import static org.hisp.dhis.webapi.controller.tracker.JsonAssertions.assertHasNoMember; -import static org.hisp.dhis.webapi.controller.tracker.JsonAssertions.assertHasOnlyMembers; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; -import java.util.Date; import java.util.Set; +import java.util.stream.Stream; import org.hisp.dhis.common.CodeGenerator; import org.hisp.dhis.common.IdentifiableObjectManager; -import org.hisp.dhis.common.ValueType; import org.hisp.dhis.dataelement.DataElement; -import org.hisp.dhis.eventdatavalue.EventDataValue; -import org.hisp.dhis.jsontree.JsonList; -import org.hisp.dhis.jsontree.JsonObject; +import org.hisp.dhis.dxf2.events.event.EventService; +import org.hisp.dhis.dxf2.events.event.Events; +import org.hisp.dhis.feedback.BadRequestException; +import org.hisp.dhis.feedback.ForbiddenException; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.program.Program; -import org.hisp.dhis.program.ProgramInstance; import org.hisp.dhis.program.ProgramStage; -import org.hisp.dhis.program.ProgramStageInstance; -import org.hisp.dhis.program.ProgramStatus; -import org.hisp.dhis.program.UserInfoSnapshot; -import org.hisp.dhis.relationship.Relationship; -import org.hisp.dhis.relationship.RelationshipEntity; -import org.hisp.dhis.relationship.RelationshipItem; -import org.hisp.dhis.relationship.RelationshipType; import org.hisp.dhis.security.acl.AccessStringHelper; -import org.hisp.dhis.trackedentity.TrackedEntityInstance; -import org.hisp.dhis.trackedentity.TrackedEntityType; import org.hisp.dhis.user.User; import org.hisp.dhis.user.sharing.UserAccess; import org.hisp.dhis.web.HttpStatus; import org.hisp.dhis.webapi.DhisControllerConvenienceTest; -import org.hisp.dhis.webapi.controller.tracker.JsonRelationship; +import org.hisp.dhis.webapi.utils.ContextUtils; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +@ContextConfiguration(classes = TrackerEventExportTestConfiguration.class) +@ActiveProfiles("TrackerEventExport") class TrackerEventsExportControllerTest extends DhisControllerConvenienceTest { - @Autowired private IdentifiableObjectManager manager; - private OrganisationUnit orgUnit; - - private OrganisationUnit anotherOrgUnit; - - private Program program; - - private ProgramStage programStage; - - private User owner; + @Autowired private EventService eventService; private User user; - private TrackedEntityType trackedEntityType; - @BeforeEach void setUp() { - owner = makeUser("owner"); + User owner = makeUser("owner"); - orgUnit = createOrganisationUnit('A'); + OrganisationUnit orgUnit = createOrganisationUnit('A'); orgUnit.getSharing().setOwner(owner); manager.save(orgUnit, false); - anotherOrgUnit = createOrganisationUnit('B'); + OrganisationUnit anotherOrgUnit = createOrganisationUnit('B'); anotherOrgUnit.getSharing().setOwner(owner); manager.save(anotherOrgUnit, false); @@ -102,238 +84,91 @@ void setUp() { user.setTeiSearchOrganisationUnits(Set.of(orgUnit)); this.userService.updateUser(user); - program = createProgram('A'); + Program program = createProgram('A'); program.addOrganisationUnit(orgUnit); program.getSharing().setOwner(owner); program.getSharing().addUserAccess(userAccess()); manager.save(program, false); - programStage = createProgramStage('A', program); + ProgramStage programStage = createProgramStage('A', program); programStage.getSharing().setOwner(owner); programStage.getSharing().addUserAccess(userAccess()); manager.save(programStage, false); - trackedEntityType = trackedEntityTypeAccessible(); - } - - @Test - void getEventById() { - TrackedEntityInstance to = trackedEntityInstance(); - ProgramStageInstance from = programStageInstance(programInstance(to)); - relationship(from, to); - - JsonObject json = GET("/tracker/events/{id}", from.getUid()).content(HttpStatus.OK); - - assertDefaultResponse(json, from); - } - - @Test - void getEventByIdWithFields() { - TrackedEntityInstance tei = trackedEntityInstance(); - ProgramStageInstance event = programStageInstance(programInstance(tei)); - - JsonObject json = - GET("/tracker/events/{id}?fields=orgUnit,status", event.getUid()).content(HttpStatus.OK); - - assertFalse(json.isEmpty()); - assertHasOnlyMembers(json, "orgUnit", "status"); - } - - @Test - void getEventByIdWithFieldsRelationships() { - TrackedEntityInstance to = trackedEntityInstance(); - ProgramStageInstance from = programStageInstance(programInstance(to)); - Relationship r = relationship(from, to); - - JsonList relationships = - GET("/tracker/events/{id}?fields=relationships", from.getUid()) - .content(HttpStatus.OK) - .getList("relationships", JsonRelationship.class); - - assertFalse(relationships.isEmpty()); - assertEquals(1, relationships.size()); - JsonRelationship relationship = assertFirstRelationship(r, relationships); - assertEventWithinRelationship(from, relationship.getFrom()); - assertTrackedEntityWithinRelationship(to, relationship.getTo()); - } - - @Test - void getEventByIdRelationshipsNoAccessToRelationshipType() { - TrackedEntityInstance to = trackedEntityInstance(); - ProgramStageInstance from = programStageInstance(programInstance(to)); - relationship(relationshipTypeNotAccessible(), from, to); - this.switchContextToUser(user); - - JsonList relationships = - GET("/tracker/events/{id}?fields=relationships", from.getUid()) - .content(HttpStatus.OK) - .getList("relationships", JsonRelationship.class); - - assertEquals(0, relationships.size()); - } - - @Test - void getEventByIdRelationshipsNoAccessToRelationshipItemTo() { - TrackedEntityType type = trackedEntityTypeNotAccessible(); - TrackedEntityInstance to = trackedEntityInstance(type); - ProgramStageInstance from = programStageInstance(programInstance(to)); - relationship(from, to); - this.switchContextToUser(user); - - JsonList relationships = - GET("/tracker/events/{id}?fields=relationships", from.getUid()) - .content(HttpStatus.OK) - .getList("relationships", JsonRelationship.class); - - assertEquals(0, relationships.size()); - } - - @Test - void getEventByIdRelationshipsNoAccessToBothRelationshipItems() { - TrackedEntityInstance to = trackedEntityInstanceNotInSearchScope(); - ProgramStageInstance from = programStageInstance(programInstance(to)); - relationship(from, to); - this.switchContextToUser(user); - - assertTrue( - GET("/tracker/events/{id}", from.getUid()) - .error(HttpStatus.CONFLICT) - .getMessage() - .contains("OWNERSHIP_ACCESS_DENIED")); - } - - @Test - void getEventByIdRelationshipsNoAccessToRelationshipItemFrom() { - TrackedEntityType type = trackedEntityTypeNotAccessible(); - TrackedEntityInstance from = trackedEntityInstance(type); - ProgramStageInstance to = programStageInstance(programInstance(from)); - relationship(from, to); - this.switchContextToUser(user); - - JsonList relationships = - GET("/tracker/events/{id}?fields=relationships", to.getUid()) - .content(HttpStatus.OK) - .getList("relationships", JsonRelationship.class); - - assertEquals(0, relationships.size()); - } - - @Test - void getEventByIdContainsCreatedByAndUpdateByAndAssignedUserInDataValues() { - - TrackedEntityInstance tei = trackedEntityInstance(); - ProgramInstance programInstance = programInstance(tei); - ProgramStageInstance programStageInstance = programStageInstance(programInstance); - programStageInstance.setCreatedByUserInfo(UserInfoSnapshot.from(user)); - programStageInstance.setLastUpdatedByUserInfo(UserInfoSnapshot.from(user)); - programStageInstance.setAssignedUser(user); - EventDataValue eventDataValue = new EventDataValue(); - eventDataValue.setValue("6"); - DataElement dataElement = createDataElement('A'); - dataElement.setValueType(ValueType.NUMBER); - manager.save(dataElement); - eventDataValue.setDataElement(dataElement.getUid()); - eventDataValue.setCreatedByUserInfo(UserInfoSnapshot.from(user)); - eventDataValue.setLastUpdatedByUserInfo(UserInfoSnapshot.from(user)); - Set eventDataValues = Set.of(eventDataValue); - programStageInstance.setEventDataValues(eventDataValues); - manager.save(programStageInstance); - - JsonObject event = - GET("/tracker/events/{id}", programStageInstance.getUid()).content(HttpStatus.OK); - - assertTrue(event.isObject()); - assertFalse(event.isEmpty()); - assertEquals(programStageInstance.getUid(), event.getString("event").string()); - assertEquals(programInstance.getUid(), event.getString("enrollment").string()); - assertEquals(orgUnit.getUid(), event.getString("orgUnit").string()); - assertEquals(user.getUsername(), event.getString("createdBy.username").string()); - assertEquals(user.getUsername(), event.getString("updatedBy.username").string()); - assertEquals(user.getDisplayName(), event.getString("assignedUser.displayName").string()); - assertFalse(event.getArray("dataValues").isEmpty()); - assertEquals( - user.getUsername(), - event.getArray("dataValues").getObject(0).getString("createdBy.username").string()); - assertEquals( - user.getUsername(), - event.getArray("dataValues").getObject(0).getString("updatedBy.username").string()); - } - - @Test - void getEventByIdNotFound() { - assertEquals( - "Event with id Hq3Kc6HK4OZ could not be found.", - GET("/tracker/events/Hq3Kc6HK4OZ").error(HttpStatus.NOT_FOUND).getMessage()); - } - - private TrackedEntityType trackedEntityTypeAccessible() { - TrackedEntityType type = trackedEntityType('A'); - type.getSharing().addUserAccess(userAccess()); - manager.save(type, false); - return type; - } - - private TrackedEntityType trackedEntityTypeNotAccessible() { - TrackedEntityType type = trackedEntityType('B'); - manager.save(type, false); - return type; - } - - private TrackedEntityType trackedEntityType(char uniqueChar) { - TrackedEntityType type = createTrackedEntityType(uniqueChar); - type.getSharing().setOwner(owner); - type.getSharing().setPublicAccess(AccessStringHelper.DEFAULT); - return type; - } - - private TrackedEntityInstance trackedEntityInstance() { - TrackedEntityInstance tei = trackedEntityInstance(orgUnit); - manager.save(tei, false); - return tei; - } - - private TrackedEntityInstance trackedEntityInstanceNotInSearchScope() { - TrackedEntityInstance tei = trackedEntityInstance(anotherOrgUnit); - manager.save(tei, false); - return tei; - } - - private TrackedEntityInstance trackedEntityInstance(TrackedEntityType trackedEntityType) { - TrackedEntityInstance tei = trackedEntityInstance(orgUnit, trackedEntityType); - manager.save(tei, false); - return tei; - } - - private TrackedEntityInstance trackedEntityInstance(OrganisationUnit orgUnit) { - return trackedEntityInstance(orgUnit, trackedEntityType); - } - - private TrackedEntityInstance trackedEntityInstance( - OrganisationUnit orgUnit, TrackedEntityType trackedEntityType) { - TrackedEntityInstance tei = createTrackedEntityInstance(orgUnit); - tei.setTrackedEntityType(trackedEntityType); - tei.getSharing().setPublicAccess(AccessStringHelper.DEFAULT); - tei.getSharing().setOwner(owner); - return tei; - } - - private ProgramInstance programInstance(TrackedEntityInstance tei) { - ProgramInstance programInstance = new ProgramInstance(program, tei, tei.getOrganisationUnit()); - programInstance.setAutoFields(); - programInstance.setEnrollmentDate(new Date()); - programInstance.setIncidentDate(new Date()); - programInstance.setStatus(ProgramStatus.COMPLETED); - manager.save(programInstance); - return programInstance; - } - - private ProgramStageInstance programStageInstance(ProgramInstance programInstance) { - ProgramStageInstance programStageInstance = - new ProgramStageInstance( - programInstance, programStage, programInstance.getOrganisationUnit()); - programStageInstance.setAutoFields(); - manager.save(programStageInstance); - return programStageInstance; + DataElement de = createDataElement('A'); + de.getSharing().setOwner(owner); + manager.save(de, false); + } + + static Stream + shouldMatchContentTypeAndAttachmentWhenEndpointForCompressedEventJsonIsInvoked() { + return Stream.of( + arguments( + "/tracker/events.json.zip?attachment=file.json.zip", + "application/json+zip", + "attachment; filename=file.json.zip", + "binary"), + arguments( + "/tracker/events.json.zip", + "application/json+zip", + "attachment; filename=events.json.zip", + "binary"), + arguments( + "/tracker/events.json.gz?attachment=file.json.gz", + "application/json+gzip", + "attachment; filename=file.json.gz", + "binary"), + arguments( + "/tracker/events.json.gz", + "application/json+gzip", + "attachment; filename=events.json.gz", + "binary"), + arguments( + "/tracker/events.csv", + "application/csv; charset=UTF-8", + "attachment; filename=events.csv", + null), + arguments( + "/tracker/events.csv?attachment=file.csv", + "application/csv; charset=UTF-8", + "attachment; filename=file.csv", + null), + arguments( + "/tracker/events.csv.gz", + "application/csv+gzip", + "attachment; filename=events.csv.gz", + "binary"), + arguments( + "/tracker/events.csv.gz?attachment=file.csv.gz", + "application/csv+gzip", + "attachment; filename=file.csv.gz", + "binary"), + arguments( + "/tracker/events.csv.zip", + "application/csv+zip", + "attachment; filename=events.csv.zip", + "binary"), + arguments( + "/tracker/events.csv.zip?attachment=file.csv.zip", + "application/csv+zip", + "attachment; filename=file.csv.zip", + "binary")); + } + + @ParameterizedTest + @MethodSource + void shouldMatchContentTypeAndAttachmentWhenEndpointForCompressedEventJsonIsInvoked( + String url, String expectedContentType, String expectedAttachment, String encoding) + throws ForbiddenException, BadRequestException { + + when(eventService.getEvents(any())).thenReturn(new Events()); + injectSecurityContext(user); + + HttpResponse res = GET(url); + assertEquals(HttpStatus.OK, res.status()); + assertEquals(expectedContentType, res.header("Content-Type")); + assertEquals(expectedAttachment, res.header(ContextUtils.HEADER_CONTENT_DISPOSITION)); + assertEquals(encoding, res.header(ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING)); } private UserAccess userAccess() { @@ -342,139 +177,4 @@ private UserAccess userAccess() { a.setAccess(AccessStringHelper.FULL); return a; } - - private RelationshipType relationshipTypeAccessible( - RelationshipEntity from, RelationshipEntity to) { - RelationshipType type = relationshipType(from, to); - type.getSharing().addUserAccess(userAccess()); - manager.save(type, false); - return type; - } - - private RelationshipType relationshipTypeNotAccessible() { - return relationshipType( - RelationshipEntity.PROGRAM_STAGE_INSTANCE, RelationshipEntity.TRACKED_ENTITY_INSTANCE); - } - - private RelationshipType relationshipType(RelationshipEntity from, RelationshipEntity to) { - RelationshipType type = createRelationshipType('A'); - type.getFromConstraint().setRelationshipEntity(from); - type.getToConstraint().setRelationshipEntity(to); - type.getSharing().setOwner(owner); - type.getSharing().setPublicAccess(AccessStringHelper.DEFAULT); - manager.save(type, false); - return type; - } - - private Relationship relationship(ProgramStageInstance from, TrackedEntityInstance to) { - return relationship( - relationshipTypeAccessible( - RelationshipEntity.PROGRAM_STAGE_INSTANCE, RelationshipEntity.TRACKED_ENTITY_INSTANCE), - from, - to); - } - - private Relationship relationship( - RelationshipType type, ProgramStageInstance from, TrackedEntityInstance to) { - Relationship r = new Relationship(); - - RelationshipItem fromItem = new RelationshipItem(); - fromItem.setProgramStageInstance(from); - from.getRelationshipItems().add(fromItem); - fromItem.setRelationship(r); - r.setFrom(fromItem); - - RelationshipItem toItem = new RelationshipItem(); - toItem.setTrackedEntityInstance(to); - to.getRelationshipItems().add(toItem); - r.setTo(toItem); - toItem.setRelationship(r); - - r.setRelationshipType(type); - r.setKey(type.getUid()); - r.setInvertedKey(type.getUid()); - r.setAutoFields(); - r.getSharing().setOwner(owner); - manager.save(r, false); - return r; - } - - private Relationship relationship(TrackedEntityInstance from, ProgramStageInstance to) { - Relationship r = new Relationship(); - - RelationshipItem fromItem = new RelationshipItem(); - fromItem.setTrackedEntityInstance(from); - from.getRelationshipItems().add(fromItem); - r.setFrom(fromItem); - fromItem.setRelationship(r); - - RelationshipItem toItem = new RelationshipItem(); - toItem.setProgramStageInstance(to); - to.getRelationshipItems().add(toItem); - r.setTo(toItem); - toItem.setRelationship(r); - - RelationshipType type = - relationshipTypeAccessible( - RelationshipEntity.PROGRAM_STAGE_INSTANCE, RelationshipEntity.TRACKED_ENTITY_INSTANCE); - r.setRelationshipType(type); - r.setKey(type.getUid()); - r.setInvertedKey(type.getUid()); - - r.setAutoFields(); - r.getSharing().setOwner(owner); - manager.save(r, false); - return r; - } - - private void assertEventWithinRelationship(ProgramStageInstance expected, JsonObject json) { - JsonObject jsonEvent = json.getObject("event"); - assertFalse(jsonEvent.isEmpty(), "event should not be empty"); - assertEquals(expected.getUid(), jsonEvent.getString("event").string(), "event UID"); - assertFalse(jsonEvent.has("status")); - assertFalse(jsonEvent.has("orgUnit")); - assertFalse(jsonEvent.has("programStage")); - assertFalse( - jsonEvent.has("relationships"), "relationships is not returned within relationship items"); - } - - private void assertTrackedEntityWithinRelationship( - TrackedEntityInstance expected, JsonObject json) { - JsonObject jsonTEI = json.getObject("trackedEntity"); - assertFalse(jsonTEI.isEmpty(), "trackedEntity should not be empty"); - assertEquals( - expected.getUid(), jsonTEI.getString("trackedEntity").string(), "trackedEntity UID"); - assertFalse(jsonTEI.has("trackedEntityType")); - assertFalse(jsonTEI.has("orgUnit")); - assertFalse( - jsonTEI.has("relationships"), "relationships is not returned within relationship items"); - assertTrue(jsonTEI.getArray("attributes").isEmpty()); - } - - private void assertDefaultResponse(JsonObject json, ProgramStageInstance programStageInstance) { - // note that some fields are not included in the response because they - // are not part of the setup - // i.e attributeOptionCombo, ... - assertTrue(json.isObject()); - assertFalse(json.isEmpty()); - assertEquals(programStageInstance.getUid(), json.getString("event").string(), "event UID"); - assertEquals("ACTIVE", json.getString("status").string()); - assertEquals(program.getUid(), json.getString("program").string()); - assertEquals(programStage.getUid(), json.getString("programStage").string()); - assertEquals( - programStageInstance.getProgramInstance().getUid(), json.getString("enrollment").string()); - assertEquals(orgUnit.getUid(), json.getString("orgUnit").string()); - assertEquals(orgUnit.getName(), json.getString("orgUnitName").string()); - assertFalse(json.getBoolean("followup").booleanValue()); - assertFalse(json.getBoolean("deleted").booleanValue()); - assertHasMember(json, "createdAt"); - assertHasMember(json, "createdAtClient"); - assertHasMember(json, "updatedAt"); - assertHasMember(json, "updatedAtClient"); - assertHasMember(json, "dataValues"); - assertHasMember(json, "notes"); - assertHasNoMember(json, "attributeOptionCombo"); - assertHasNoMember(json, "attributeCategoryOptions"); - assertHasNoMember(json, "relationships"); - } } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtil.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtil.java new file mode 100644 index 000000000000..f6d66d83e6a2 --- /dev/null +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtil.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.tracker.export; + +import com.fasterxml.jackson.databind.ObjectWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +public class CompressionUtil { + + private CompressionUtil() { + throw new IllegalStateException( + "Utility class to compress exported objects in Zip o GZip format"); + } + + /** + * @param requestOutputStream Output stream from request + * @param toCompress Objects to compress + * @param objectWriter Object writer from a mapper + * @param attachment Attachment file name + * @param + * @throws IOException + */ + public static void writeZip( + OutputStream requestOutputStream, T toCompress, ObjectWriter objectWriter, String attachment) + throws IOException { + ZipOutputStream outputStream = new ZipOutputStream(requestOutputStream); + outputStream.putNextEntry(new ZipEntry(attachment)); + + objectWriter.writeValue(outputStream, toCompress); + outputStream.close(); + } + + /** + * @param requestOutputStream Output stream from request + * @param toCompress Objects to compress + * @param objectWriter Object writer from a mapper + * @param + * @throws IOException + */ + public static void writeGzip( + OutputStream requestOutputStream, T toCompress, ObjectWriter objectWriter) + throws IOException { + GZIPOutputStream outputStream = new GZIPOutputStream(requestOutputStream); + + objectWriter.writeValue(outputStream, toCompress); + outputStream.close(); + } +} diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventsExportController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventsExportController.java index 77ed130e7c9a..87581e8ed2ce 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventsExportController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventsExportController.java @@ -28,19 +28,25 @@ package org.hisp.dhis.webapi.controller.tracker.export; import static org.hisp.dhis.webapi.controller.tracker.TrackerControllerSupport.RESOURCE_PATH; +import static org.hisp.dhis.webapi.controller.tracker.export.CompressionUtil.writeGzip; +import static org.hisp.dhis.webapi.controller.tracker.export.CompressionUtil.writeZip; import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_CSV; import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_CSV_GZIP; +import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_CSV_ZIP; +import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_JSON_GZIP; +import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_JSON_ZIP; import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_TEXT_CSV; +import static org.hisp.dhis.webapi.utils.ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING; import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import java.io.OutputStream; import java.util.Collections; import java.util.List; -import java.util.zip.GZIPOutputStream; +import java.util.Objects; import javax.annotation.Nonnull; -import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import lombok.RequiredArgsConstructor; import org.hisp.dhis.common.DhisApiVersion; @@ -97,6 +103,8 @@ public class TrackerEventsExportController { private final EventFieldsParamMapper eventsMapper; + private final ObjectMapper objectMapper; + @GetMapping(produces = APPLICATION_JSON_VALUE) public PagingWrapper getEvents( TrackerEventCriteria eventCriteria, @@ -126,12 +134,60 @@ public PagingWrapper getEvents( return pagingWrapper.withInstances(objectNodes); } - @GetMapping(produces = {CONTENT_TYPE_CSV, CONTENT_TYPE_CSV_GZIP, CONTENT_TYPE_TEXT_CSV}) - public void getCsvEvents( + @GetMapping(produces = CONTENT_TYPE_JSON_GZIP) + void getEventsAsGzip(TrackerEventCriteria eventCriteria, HttpServletResponse response) + throws BadRequestException, IOException, ForbiddenException { + EventQueryParams eventQueryParams = requestToSearchParams.map(eventCriteria); + + if (areAllEnrollmentsInvalid(eventCriteria, eventQueryParams)) { + return; + } + + Events events = eventService.getEvents(eventQueryParams); + + String attachment = getAttachmentOrDefault(eventCriteria.getAttachment(), "json", "gz"); + + response.addHeader( + ContextUtils.HEADER_CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); + response.addHeader(HEADER_CONTENT_TRANSFER_ENCODING, "binary"); + response.setContentType(CONTENT_TYPE_JSON_GZIP); + + writeGzip( + response.getOutputStream(), + EVENTS_MAPPER.fromCollection(events.getEvents()), + objectMapper.writer()); + } + + @GetMapping(produces = CONTENT_TYPE_JSON_ZIP) + void getEventsAsZip(TrackerEventCriteria eventCriteria, HttpServletResponse response) + throws BadRequestException, ForbiddenException, IOException { + EventQueryParams eventQueryParams = requestToSearchParams.map(eventCriteria); + + if (areAllEnrollmentsInvalid(eventCriteria, eventQueryParams)) { + return; + } + + Events events = eventService.getEvents(eventQueryParams); + + String attachment = getAttachmentOrDefault(eventCriteria.getAttachment(), "json", "zip"); + + response.addHeader( + ContextUtils.HEADER_CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); + response.addHeader(HEADER_CONTENT_TRANSFER_ENCODING, "binary"); + response.setContentType(CONTENT_TYPE_JSON_ZIP); + + writeZip( + response.getOutputStream(), + EVENTS_MAPPER.fromCollection(events.getEvents()), + objectMapper.writer(), + attachment); + } + + @GetMapping(produces = {CONTENT_TYPE_CSV, CONTENT_TYPE_TEXT_CSV}) + void getEventsAsCsv( TrackerEventCriteria eventCriteria, HttpServletResponse response, - @RequestParam(required = false, defaultValue = "false") boolean skipHeader, - HttpServletRequest request) + @RequestParam(required = false, defaultValue = "false") boolean skipHeader) throws IOException, BadRequestException, ForbiddenException { EventQueryParams eventQueryParams = requestToSearchParams.map(eventCriteria); @@ -141,21 +197,82 @@ public void getCsvEvents( Events events = eventService.getEvents(eventQueryParams); + String attachment = getAttachmentOrDefault(eventCriteria.getAttachment(), "csv"); + OutputStream outputStream = response.getOutputStream(); response.setContentType(CONTENT_TYPE_CSV); - response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"events.csv\""); - - if (ContextUtils.isAcceptCsvGzip(request)) { - response.addHeader(ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, "binary"); - outputStream = new GZIPOutputStream(outputStream); - response.setContentType(CONTENT_TYPE_CSV_GZIP); - response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"events.csv.gz\""); - } + response.setHeader( + HttpHeaders.CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); csvEventService.writeEvents( outputStream, EVENTS_MAPPER.fromCollection(events.getEvents()), !skipHeader); } + @GetMapping(produces = {CONTENT_TYPE_CSV_GZIP}) + void getEventsAsCsvGZip( + TrackerEventCriteria eventCriteria, + HttpServletResponse response, + @RequestParam(required = false, defaultValue = "false") boolean skipHeader) + throws IOException, BadRequestException, ForbiddenException { + EventQueryParams eventQueryParams = requestToSearchParams.map(eventCriteria); + + if (areAllEnrollmentsInvalid(eventCriteria, eventQueryParams)) { + return; + } + + Events events = eventService.getEvents(eventQueryParams); + + String attachment = getAttachmentOrDefault(eventCriteria.getAttachment(), "csv", "gz"); + + response.addHeader(ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, "binary"); + response.setContentType(CONTENT_TYPE_CSV_GZIP); + response.addHeader( + ContextUtils.HEADER_CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); + + csvEventService.writeGzip( + response.getOutputStream(), EVENTS_MAPPER.fromCollection(events.getEvents()), !skipHeader); + } + + @GetMapping(produces = {CONTENT_TYPE_CSV_ZIP}) + void getEventsAsCsvZip( + TrackerEventCriteria eventCriteria, + HttpServletResponse response, + @RequestParam(required = false, defaultValue = "false") boolean skipHeader) + throws IOException, BadRequestException, ForbiddenException { + EventQueryParams eventQueryParams = requestToSearchParams.map(eventCriteria); + + if (areAllEnrollmentsInvalid(eventCriteria, eventQueryParams)) { + return; + } + + Events events = eventService.getEvents(eventQueryParams); + + String attachment = getAttachmentOrDefault(eventCriteria.getAttachment(), "csv", "zip"); + + response.addHeader(ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, "binary"); + response.setContentType(CONTENT_TYPE_CSV_ZIP); + response.addHeader( + ContextUtils.HEADER_CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); + + csvEventService.writeZip( + response.getOutputStream(), + EVENTS_MAPPER.fromCollection(events.getEvents()), + !skipHeader, + attachment); + } + + private String getAttachmentOrDefault(String filename, String type, String compression) { + return Objects.toString(filename, String.join(".", EVENTS, type, compression)); + } + + private String getAttachmentOrDefault(String filename, String type) { + return Objects.toString(filename, String.join(".", EVENTS, type)); + } + + public String getContentDispositionHeaderValue(String filename) { + return "attachment; filename=" + filename; + } + private boolean areAllEnrollmentsInvalid( TrackerEventCriteria eventCriteria, EventQueryParams eventQueryParams) { return !CollectionUtils.isEmpty(eventCriteria.getEnrollments()) diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvEventService.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvEventService.java index cef3b65a7e1a..f5ed70afab81 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvEventService.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvEventService.java @@ -46,6 +46,7 @@ import org.hisp.dhis.dxf2.events.event.csv.CsvEventService; import org.hisp.dhis.event.EventStatus; import org.hisp.dhis.util.DateUtils; +import org.hisp.dhis.webapi.controller.tracker.export.CompressionUtil; import org.hisp.dhis.webapi.controller.tracker.view.DataValue; import org.hisp.dhis.webapi.controller.tracker.view.Event; import org.hisp.dhis.webapi.controller.tracker.view.User; @@ -66,13 +67,8 @@ public class TrackerCsvEventService implements CsvEventService { @Override public void writeEvents(OutputStream outputStream, List events, boolean withHeader) throws IOException { - final CsvSchema csvSchema = - CSV_MAPPER - .schemaFor(CsvEventDataValue.class) - .withLineSeparator("\n") - .withUseHeader(withHeader); - ObjectWriter writer = CSV_MAPPER.writer(csvSchema.withUseHeader(withHeader)); + ObjectWriter writer = getObjectWriter(withHeader); List dataValues = new ArrayList<>(); @@ -147,6 +143,29 @@ public void writeEvents(OutputStream outputStream, List events, boolean w writer.writeValue(outputStream, dataValues); } + @Override + public void writeZip( + OutputStream outputStream, List toCompress, boolean withHeader, String file) + throws IOException { + CompressionUtil.writeZip(outputStream, toCompress, getObjectWriter(withHeader), file); + } + + @Override + public void writeGzip(OutputStream outputStream, List toCompress, boolean withHeader) + throws IOException { + CompressionUtil.writeGzip(outputStream, toCompress, getObjectWriter(withHeader)); + } + + private ObjectWriter getObjectWriter(boolean withHeader) { + final CsvSchema csvSchema = + CSV_MAPPER + .schemaFor(CsvEventDataValue.class) + .withLineSeparator("\n") + .withUseHeader(withHeader); + + return CSV_MAPPER.writer(csvSchema.withUseHeader(withHeader)); + } + @Override public List readEvents(InputStream inputStream, boolean skipFirst) throws IOException, ParseException { diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvTrackedEntityService.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvTrackedEntityService.java index 4afcb446e63f..66797845ae5b 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvTrackedEntityService.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvTrackedEntityService.java @@ -39,6 +39,7 @@ import java.util.Collections; import java.util.List; import org.hisp.dhis.dxf2.events.event.csv.CsvEventService; +import org.hisp.dhis.webapi.controller.tracker.export.CompressionUtil; import org.hisp.dhis.webapi.controller.tracker.view.Attribute; import org.hisp.dhis.webapi.controller.tracker.view.TrackedEntity; import org.springframework.stereotype.Service; @@ -52,13 +53,7 @@ public class TrackerCsvTrackedEntityService implements CsvEventService trackedEntities, boolean withHeader) throws IOException { - final CsvSchema csvSchema = - CSV_MAPPER - .schemaFor(CsvTrackedEntity.class) - .withLineSeparator("\n") - .withUseHeader(withHeader); - - ObjectWriter writer = CSV_MAPPER.writer(csvSchema.withUseHeader(withHeader)); + ObjectWriter writer = getObjectWriter(withHeader); List attributes = new ArrayList<>(); @@ -99,6 +94,30 @@ public void writeEvents( writer.writeValue(outputStream, attributes); } + @Override + public void writeZip( + OutputStream outputStream, List toCompress, boolean withHeader, String file) + throws IOException { + CompressionUtil.writeZip(outputStream, toCompress, getObjectWriter(withHeader), file); + } + + @Override + public void writeGzip( + OutputStream outputStream, List toCompress, boolean withHeader) + throws IOException { + CompressionUtil.writeGzip(outputStream, toCompress, getObjectWriter(withHeader)); + } + + private ObjectWriter getObjectWriter(boolean withHeader) { + final CsvSchema csvSchema = + CSV_MAPPER + .schemaFor(CsvTrackedEntity.class) + .withLineSeparator("\n") + .withUseHeader(withHeader); + + return CSV_MAPPER.writer(csvSchema.withUseHeader(withHeader)); + } + private String checkForNull(Instant instant) { if (instant == null) { return null; diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/ContextUtils.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/ContextUtils.java index 5d1cbda31142..8a704d6db5a7 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/ContextUtils.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/ContextUtils.java @@ -59,7 +59,8 @@ public class ContextUtils { public static final String CONTENT_TYPE_PDF = "application/pdf"; - public static final String CONTENT_TYPE_ZIP = "application/zip"; + public static final String CONTENT_TYPE_JSON_ZIP = "application/json+zip"; + public static final String CONTENT_TYPE_JSON_GZIP = "application/json+gzip"; public static final String CONTENT_TYPE_GZIP = "application/gzip"; diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtilTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtilTest.java new file mode 100644 index 000000000000..457b712493c8 --- /dev/null +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtilTest.java @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.tracker.export; + +import static org.hisp.dhis.webapi.controller.tracker.export.CompressionUtil.writeGzip; +import static org.hisp.dhis.webapi.controller.tracker.export.CompressionUtil.writeZip; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.List; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import org.hisp.dhis.common.CodeGenerator; +import org.hisp.dhis.webapi.controller.tracker.view.Event; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class CompressionUtilTest { + + private static final Event FIRST_EVENT = new Event(); + private static final Event SECOND_EVENT = new Event(); + @InjectMocks private ObjectMapper objectMapper; + + @BeforeEach + void setUp() { + FIRST_EVENT.setEvent(CodeGenerator.generateUid()); + SECOND_EVENT.setEvent(CodeGenerator.generateUid()); + } + + @Test + void shouldUnzipFileAndMatchEventsWhenCreateZipFileFromEventList() throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + List eventToZip = getEvents(); + + writeZip(outputStream, eventToZip, objectMapper.writer(), "file.json.zip"); + + ZipInputStream zipInputStream = + new ZipInputStream(new ByteArrayInputStream(outputStream.toByteArray())); + var buff = new byte[1024]; + List eventsFromZip; + + ZipEntry zipEntry = zipInputStream.getNextEntry(); + + assertNotNull(zipEntry, "Events Zip file has no entry"); + assertEquals("file.json.zip", zipEntry.getName(), "Events Zip file has a wrong name"); + + var byteArrayOutputStream = new ByteArrayOutputStream(); + int l; + while ((l = zipInputStream.read(buff)) > 0) { + byteArrayOutputStream.write(buff, 0, l); + } + eventsFromZip = + objectMapper.readValue(byteArrayOutputStream.toString(), new TypeReference<>() {}); + + assertNull(zipInputStream.getNextEntry()); // assert only one file is created + assertEquals(eventToZip.size(), eventsFromZip.size()); + assertEquals( + FIRST_EVENT, + eventsFromZip.stream() + .filter(e -> e.getEvent().equals(FIRST_EVENT.getEvent())) + .findAny() + .orElse(null), + "The event does not match or does not exist in the Zip File."); + assertEquals( + SECOND_EVENT, + eventsFromZip.stream() + .filter(e -> e.getEvent().equals(SECOND_EVENT.getEvent())) + .findAny() + .orElse(null), + "The event does not match or does not exist in the Zip File."); + } + + @Test + void shouldGUnzipFileAndMatchEventsWhenCreateGZipFileFromEventList() throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + List eventToGZip = getEvents(); + + writeGzip(outputStream, eventToGZip, objectMapper.writer()); + + GZIPInputStream gzipInputStream = + new GZIPInputStream(new ByteArrayInputStream(outputStream.toByteArray())); + var buff = new byte[1024]; + List eventsFromGZip; + + var byteArrayOutputStream = new ByteArrayOutputStream(); + int l; + while ((l = gzipInputStream.read(buff)) > 0) { + byteArrayOutputStream.write(buff, 0, l); + } + eventsFromGZip = + objectMapper.readValue(byteArrayOutputStream.toString(), new TypeReference<>() {}); + + assertEquals(eventToGZip.size(), eventsFromGZip.size()); + assertEquals( + FIRST_EVENT, + eventToGZip.stream() + .filter(e -> e.getEvent().equals(FIRST_EVENT.getEvent())) + .findAny() + .orElse(null), + "The event does not match or does not exist in the GZip File."); + assertEquals( + SECOND_EVENT, + eventToGZip.stream() + .filter(e -> e.getEvent().equals(SECOND_EVENT.getEvent())) + .findAny() + .orElse(null), + "The event does not match or does not exist in the GZip File."); + } + + List getEvents() { + return List.of(FIRST_EVENT, SECOND_EVENT); + } +} From f766c390e9c8966b40372e6ed5755342b6974ccb Mon Sep 17 00:00:00 2001 From: luca Date: Tue, 2 Jan 2024 13:48:29 +0100 Subject: [PATCH 2/4] fix: harmonize compressione support events/trackedentities --- ...erTrackedEntitiesExportControllerTest.java | 8 +- .../export/TrackerEventsExportController.java | 11 +- ...rackerTrackedEntitiesExportController.java | 118 ++++++++++++------ .../hisp/dhis/webapi/utils/ContextUtils.java | 2 + 4 files changed, 92 insertions(+), 47 deletions(-) diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerTrackedEntitiesExportControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerTrackedEntitiesExportControllerTest.java index 2b3d8df6eadd..1b3d76d14a4d 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerTrackedEntitiesExportControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerTrackedEntitiesExportControllerTest.java @@ -326,9 +326,7 @@ void getTrackedEntityReturnsCsvFormat() { () -> assertTrue(response.header("content-type").contains(ContextUtils.CONTENT_TYPE_CSV)), () -> assertTrue( - response - .header("content-disposition") - .contains("filename=\"trackedEntities.csv\"")), + response.header("content-disposition").contains("filename=trackedEntities.csv")), () -> assertTrue(response.content().toString().contains("trackedEntity,trackedEntityType"))); } @@ -350,7 +348,7 @@ void getTrackedEntityReturnsCsvZipFormat() { assertTrue( response .header("content-disposition") - .contains("filename=\"trackedEntities.csv.zip\""))); + .contains("filename=trackedEntities.csv.zip"))); } @Test @@ -371,7 +369,7 @@ void getTrackedEntityReturnsCsvGZipFormat() { assertTrue( response .header("content-disposition") - .contains("filename=\"trackedEntities.csv.gz\""))); + .contains("filename=trackedEntities.csv.gz"))); } @Test diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventsExportController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventsExportController.java index 87581e8ed2ce..9e9fb4d882e0 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventsExportController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEventsExportController.java @@ -30,6 +30,7 @@ import static org.hisp.dhis.webapi.controller.tracker.TrackerControllerSupport.RESOURCE_PATH; import static org.hisp.dhis.webapi.controller.tracker.export.CompressionUtil.writeGzip; import static org.hisp.dhis.webapi.controller.tracker.export.CompressionUtil.writeZip; +import static org.hisp.dhis.webapi.utils.ContextUtils.BINARY_HEADER_CONTENT_TRANSFER_ENCODING; import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_CSV; import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_CSV_GZIP; import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_CSV_ZIP; @@ -149,7 +150,7 @@ void getEventsAsGzip(TrackerEventCriteria eventCriteria, HttpServletResponse res response.addHeader( ContextUtils.HEADER_CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); - response.addHeader(HEADER_CONTENT_TRANSFER_ENCODING, "binary"); + response.addHeader(HEADER_CONTENT_TRANSFER_ENCODING, BINARY_HEADER_CONTENT_TRANSFER_ENCODING); response.setContentType(CONTENT_TYPE_JSON_GZIP); writeGzip( @@ -173,7 +174,7 @@ void getEventsAsZip(TrackerEventCriteria eventCriteria, HttpServletResponse resp response.addHeader( ContextUtils.HEADER_CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); - response.addHeader(HEADER_CONTENT_TRANSFER_ENCODING, "binary"); + response.addHeader(HEADER_CONTENT_TRANSFER_ENCODING, BINARY_HEADER_CONTENT_TRANSFER_ENCODING); response.setContentType(CONTENT_TYPE_JSON_ZIP); writeZip( @@ -224,7 +225,8 @@ void getEventsAsCsvGZip( String attachment = getAttachmentOrDefault(eventCriteria.getAttachment(), "csv", "gz"); - response.addHeader(ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, "binary"); + response.addHeader( + ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, BINARY_HEADER_CONTENT_TRANSFER_ENCODING); response.setContentType(CONTENT_TYPE_CSV_GZIP); response.addHeader( ContextUtils.HEADER_CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); @@ -249,7 +251,8 @@ void getEventsAsCsvZip( String attachment = getAttachmentOrDefault(eventCriteria.getAttachment(), "csv", "zip"); - response.addHeader(ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, "binary"); + response.addHeader( + ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, BINARY_HEADER_CONTENT_TRANSFER_ENCODING); response.setContentType(CONTENT_TYPE_CSV_ZIP); response.addHeader( ContextUtils.HEADER_CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerTrackedEntitiesExportController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerTrackedEntitiesExportController.java index 62d53b1a68c9..c9af6785067e 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerTrackedEntitiesExportController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerTrackedEntitiesExportController.java @@ -28,6 +28,7 @@ package org.hisp.dhis.webapi.controller.tracker.export; import static org.hisp.dhis.webapi.controller.tracker.TrackerControllerSupport.RESOURCE_PATH; +import static org.hisp.dhis.webapi.utils.ContextUtils.BINARY_HEADER_CONTENT_TRANSFER_ENCODING; import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_CSV; import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_CSV_GZIP; import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_CSV_ZIP; @@ -38,11 +39,8 @@ import java.io.IOException; import java.io.OutputStream; import java.util.List; -import java.util.zip.GZIPOutputStream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; +import java.util.Objects; import javax.annotation.Nonnull; -import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import lombok.RequiredArgsConstructor; import org.hisp.dhis.common.DhisApiVersion; @@ -145,51 +143,96 @@ PagingWrapper getInstances( return pagingWrapper.withInstances(objectNodes); } - @GetMapping( - produces = { - CONTENT_TYPE_CSV, - CONTENT_TYPE_CSV_GZIP, - CONTENT_TYPE_CSV_ZIP, - CONTENT_TYPE_TEXT_CSV - }) - public void getCsvTrackedEntities( + @GetMapping(produces = {CONTENT_TYPE_CSV, CONTENT_TYPE_TEXT_CSV}) + void getTrackedEntitiesAsCsv( TrackerTrackedEntityCriteria criteria, HttpServletResponse response, - HttpServletRequest request, @RequestParam(required = false, defaultValue = "false") boolean skipHeader) throws IOException, BadRequestException, ForbiddenException { + TrackedEntityInstanceQueryParams queryParams = criteriaMapper.map(criteria); TrackedEntityInstanceParams trackedEntityInstanceParams = fieldsMapper.map(CSV_FIELDS, criteria.isIncludeDeleted()); - List trackedEntityInstances = + String attachment = getAttachmentOrDefault(criteria.getAttachment(), "csv"); + + response.setContentType(CONTENT_TYPE_CSV); + response.setHeader( + HttpHeaders.CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); + + csvEventService.writeEvents( + response.getOutputStream(), TRACKED_ENTITY_MAPPER.fromCollection( trackedEntityInstanceService.getTrackedEntityInstances( - queryParams, trackedEntityInstanceParams, false, false)); + queryParams, trackedEntityInstanceParams, false, false)), + !skipHeader); + } - OutputStream outputStream = response.getOutputStream(); + @GetMapping(produces = {CONTENT_TYPE_CSV_ZIP}) + void getTrackedEntitiesAsCsvZip( + TrackerTrackedEntityCriteria criteria, + HttpServletResponse response, + @RequestParam(required = false, defaultValue = "false") boolean skipHeader) + throws IOException, BadRequestException, ForbiddenException { - if (ContextUtils.isAcceptCsvGzip(request)) { - response.addHeader(ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, "binary"); - outputStream = new GZIPOutputStream(outputStream); - response.setContentType(CONTENT_TYPE_CSV_GZIP); - response.setHeader( - HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"trackedEntities.csv.gz\""); - } else if (ContextUtils.isAcceptCsvZip(request)) { - response.addHeader(ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, "binary"); - response.setContentType(CONTENT_TYPE_CSV_ZIP); - response.setHeader( - HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"trackedEntities.csv.zip\""); - ZipOutputStream zos = new ZipOutputStream(outputStream); - zos.putNextEntry(new ZipEntry("trackedEntities.csv")); - outputStream = zos; - } else { - response.setContentType(CONTENT_TYPE_CSV); - response.setHeader( - HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"trackedEntities.csv\""); - } + TrackedEntityInstanceQueryParams queryParams = criteriaMapper.map(criteria); + TrackedEntityInstanceParams trackedEntityInstanceParams = + fieldsMapper.map(CSV_FIELDS, criteria.isIncludeDeleted()); + + String attachment = getAttachmentOrDefault(criteria.getAttachment(), "csv", "zip"); + + response.addHeader( + ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, BINARY_HEADER_CONTENT_TRANSFER_ENCODING); + response.setContentType(CONTENT_TYPE_CSV_ZIP); + response.setHeader( + HttpHeaders.CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); + + csvEventService.writeZip( + response.getOutputStream(), + TRACKED_ENTITY_MAPPER.fromCollection( + trackedEntityInstanceService.getTrackedEntityInstances( + queryParams, trackedEntityInstanceParams, false, false)), + !skipHeader, + attachment); + } + + @GetMapping(produces = {CONTENT_TYPE_CSV_GZIP}) + void getTrackedEntitiesAsCsvGZip( + TrackerTrackedEntityCriteria criteria, + HttpServletResponse response, + @RequestParam(required = false, defaultValue = "false") boolean skipHeader) + throws IOException, BadRequestException, ForbiddenException { + + TrackedEntityInstanceQueryParams queryParams = criteriaMapper.map(criteria); + TrackedEntityInstanceParams trackedEntityInstanceParams = + fieldsMapper.map(CSV_FIELDS, criteria.isIncludeDeleted()); + + String attachment = getAttachmentOrDefault(criteria.getAttachment(), "csv", "gz"); - csvEventService.writeEvents(outputStream, trackedEntityInstances, !skipHeader); + response.addHeader( + ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, BINARY_HEADER_CONTENT_TRANSFER_ENCODING); + response.setContentType(CONTENT_TYPE_CSV_GZIP); + response.setHeader( + HttpHeaders.CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); + + csvEventService.writeGzip( + response.getOutputStream(), + TRACKED_ENTITY_MAPPER.fromCollection( + trackedEntityInstanceService.getTrackedEntityInstances( + queryParams, trackedEntityInstanceParams, false, false)), + !skipHeader); + } + + private String getAttachmentOrDefault(String filename, String type, String compression) { + return Objects.toString(filename, String.join(".", TRACKED_ENTITIES, type, compression)); + } + + private String getAttachmentOrDefault(String filename, String type) { + return Objects.toString(filename, String.join(".", TRACKED_ENTITIES, type)); + } + + public String getContentDispositionHeaderValue(String filename) { + return "attachment; filename=" + filename; } @GetMapping(value = "{id}") @@ -224,8 +267,7 @@ public void getCsvTrackedEntityInstanceById( OutputStream outputStream = response.getOutputStream(); response.setContentType(CONTENT_TYPE_CSV); - response.setHeader( - HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"trackedEntity.csv\""); + response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=trackedEntity.csv"); csvEventService.writeEvents(outputStream, List.of(trackedEntity), !skipHeader); } } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/ContextUtils.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/ContextUtils.java index 8a704d6db5a7..490f9ca88209 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/ContextUtils.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/ContextUtils.java @@ -106,6 +106,8 @@ public class ContextUtils { public static final String HEADER_CONTENT_TRANSFER_ENCODING = "Content-Transfer-Encoding"; + public static final String BINARY_HEADER_CONTENT_TRANSFER_ENCODING = "binary"; + public static final String HEADER_VALUE_NO_STORE = "no-cache, no-store, max-age=0, must-revalidate"; From 433dffbd0f216ba431f7b6769feef22b96634602 Mon Sep 17 00:00:00 2001 From: luca Date: Thu, 4 Jan 2024 11:33:45 +0100 Subject: [PATCH 3/4] fix: compress csv object, improve testing --- ...erTrackedEntitiesExportControllerTest.java | 37 +++++ ...rackerTrackedEntitiesExportController.java | 2 +- .../export/csv/TrackerCsvEventService.java | 147 ++++++++++-------- .../csv/TrackerCsvTrackedEntityService.java | 57 +++---- .../tracker/export/CompressionUtilTest.java | 8 +- .../csv/TrackerCsvEventServiceTest.java | 65 ++++++++ .../TrackerCsvTrackedEntityServiceTest.java | 79 ++++++++++ 7 files changed, 297 insertions(+), 98 deletions(-) diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerTrackedEntitiesExportControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerTrackedEntitiesExportControllerTest.java index 1b3d76d14a4d..09d195919c6e 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerTrackedEntitiesExportControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerTrackedEntitiesExportControllerTest.java @@ -312,6 +312,43 @@ void getTrackedEntityByIdNotFound() { GET("/tracker/trackedEntities/Hq3Kc6HK4OZ").error(HttpStatus.NOT_FOUND).getMessage()); } + @Test + void getTrackedEntityCsvById() { + TrackedEntityInstance te = trackedEntityInstance(); + + this.switchContextToUser(user); + + WebClient.HttpResponse response = + GET( + "/tracker/trackedEntities/{id}", + te.getUid(), + WebClient.Accept(ContextUtils.CONTENT_TYPE_CSV)); + + String csvResponse = response.content(ContextUtils.CONTENT_TYPE_CSV); + + assertTrue(response.header("content-type").contains(ContextUtils.CONTENT_TYPE_CSV)); + assertTrue(response.header("content-disposition").contains("filename=trackedEntity.csv")); + assertEquals(trackedEntityToCsv(te), csvResponse); + } + + String trackedEntityToCsv(TrackedEntityInstance te) { + return "trackedEntity,trackedEntityType,createdAt,createdAtClient,updatedAt,updatedAtClient,orgUnit,inactive,deleted,potentialDuplicate,geometry,latitude,longitude,storedBy,createdBy,updatedBy,attrCreatedAt,attrUpdatedAt,attribute,displayName,value,valueType\n" + .concat( + String.join( + ",", + te.getUid(), + te.getTrackedEntityType().getUid(), + DateUtils.instantFromDate(te.getCreated()).toString(), + DateUtils.instantFromDate(te.getCreatedAtClient()).toString(), + DateUtils.instantFromDate(te.getLastUpdated()).toString(), + DateUtils.instantFromDate(te.getLastUpdatedAtClient()).toString(), + te.getOrganisationUnit().getUid(), + Boolean.toString(te.isInactive()), + Boolean.toString(te.isDeleted()), + Boolean.toString(te.isPotentialDuplicate()), + ",,,,,,,,,,," + "\n")); + } + @Test void getTrackedEntityReturnsCsvFormat() { WebClient.HttpResponse response = diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerTrackedEntitiesExportController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerTrackedEntitiesExportController.java index c9af6785067e..b02d22257b1b 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerTrackedEntitiesExportController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerTrackedEntitiesExportController.java @@ -251,7 +251,7 @@ public ResponseEntity getTrackedEntityInstanceById( @GetMapping( value = "{id}", - produces = {CONTENT_TYPE_CSV, CONTENT_TYPE_CSV_GZIP, CONTENT_TYPE_TEXT_CSV}) + produces = {CONTENT_TYPE_CSV, CONTENT_TYPE_TEXT_CSV}) public void getCsvTrackedEntityInstanceById( @PathVariable String id, HttpServletResponse response, diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvEventService.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvEventService.java index f5ed70afab81..985ee704e477 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvEventService.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvEventService.java @@ -70,90 +70,85 @@ public void writeEvents(OutputStream outputStream, List events, boolean w ObjectWriter writer = getObjectWriter(withHeader); - List dataValues = new ArrayList<>(); + writer.writeValue(outputStream, getCsvEventDataValues(events)); + } - for (Event event : events) { - CsvEventDataValue templateDataValue = new CsvEventDataValue(); - templateDataValue.setEvent(event.getEvent()); - templateDataValue.setStatus(event.getStatus() != null ? event.getStatus().name() : null); - templateDataValue.setProgram(event.getProgram()); - templateDataValue.setProgramStage(event.getProgramStage()); - templateDataValue.setEnrollment(event.getEnrollment()); - templateDataValue.setOrgUnit(event.getOrgUnit()); - templateDataValue.setOrgUnitName(event.getOrgUnitName()); - templateDataValue.setOccurredAt( - event.getOccurredAt() == null ? null : event.getOccurredAt().toString()); - templateDataValue.setScheduledAt( - event.getScheduledAt() == null ? null : event.getScheduledAt().toString()); - templateDataValue.setFollowup(event.isFollowup()); - templateDataValue.setDeleted(event.isDeleted()); - templateDataValue.setCreatedAt( - event.getCreatedAt() == null ? null : event.getCreatedAt().toString()); - templateDataValue.setCreatedAtClient( - event.getCreatedAtClient() == null ? null : event.getCreatedAtClient().toString()); - templateDataValue.setUpdatedAt( - event.getUpdatedAt() == null ? null : event.getUpdatedAt().toString()); - templateDataValue.setUpdatedAtClient( - event.getUpdatedAtClient() == null ? null : event.getUpdatedAtClient().toString()); - templateDataValue.setCompletedAt( - event.getCompletedAt() == null ? null : event.getCompletedAt().toString()); - templateDataValue.setUpdatedBy( - event.getUpdatedBy() == null ? null : event.getUpdatedBy().getUsername()); - templateDataValue.setStoredBy(event.getStoredBy()); - templateDataValue.setCompletedAt( - event.getCompletedAt() == null ? null : event.getCompletedAt().toString()); - templateDataValue.setCompletedBy(event.getCompletedBy()); - templateDataValue.setAttributeOptionCombo(event.getAttributeOptionCombo()); - templateDataValue.setAttributeCategoryOptions(event.getAttributeCategoryOptions()); - templateDataValue.setAssignedUser( - event.getAssignedUser() == null ? null : event.getAssignedUser().getUsername()); - - if (event.getGeometry() != null) { - templateDataValue.setGeometry(event.getGeometry().toText()); - - if (event.getGeometry().getGeometryType().equals("Point")) { - templateDataValue.setLongitude(event.getGeometry().getCoordinate().x); - templateDataValue.setLatitude(event.getGeometry().getCoordinate().y); - } - } + private static CsvEventDataValue map(DataValue value, CsvEventDataValue templateDataValue) { + CsvEventDataValue dataValue = new CsvEventDataValue(templateDataValue); + dataValue.setDataElement(value.getDataElement()); + dataValue.setValue(value.getValue()); + dataValue.setProvidedElsewhere(value.isProvidedElsewhere()); + dataValue.setCreatedAtDataValue( + value.getCreatedAt() == null ? null : value.getCreatedAt().toString()); + dataValue.setUpdatedAtDataValue( + value.getUpdatedAt() == null ? null : value.getUpdatedAt().toString()); - if (event.getDataValues().isEmpty()) { - dataValues.add(templateDataValue); - continue; - } + if (value.getStoredBy() != null) { + dataValue.setStoredBy(value.getStoredBy()); + } + return dataValue; + } - for (DataValue value : event.getDataValues()) { - CsvEventDataValue dataValue = new CsvEventDataValue(templateDataValue); - dataValue.setDataElement(value.getDataElement()); - dataValue.setValue(value.getValue()); - dataValue.setProvidedElsewhere(value.isProvidedElsewhere()); - dataValue.setCreatedAtDataValue( - value.getCreatedAt() == null ? null : value.getCreatedAt().toString()); - dataValue.setUpdatedAtDataValue( - value.getUpdatedAt() == null ? null : value.getUpdatedAt().toString()); - - if (value.getStoredBy() != null) { - dataValue.setStoredBy(value.getStoredBy()); - } + private static CsvEventDataValue map(Event event) { + CsvEventDataValue templateDataValue = new CsvEventDataValue(); + templateDataValue.setEvent(event.getEvent()); + templateDataValue.setStatus(event.getStatus() != null ? event.getStatus().name() : null); + templateDataValue.setProgram(event.getProgram()); + templateDataValue.setProgramStage(event.getProgramStage()); + templateDataValue.setEnrollment(event.getEnrollment()); + templateDataValue.setOrgUnit(event.getOrgUnit()); + templateDataValue.setOrgUnitName(event.getOrgUnitName()); + templateDataValue.setOccurredAt( + event.getOccurredAt() == null ? null : event.getOccurredAt().toString()); + templateDataValue.setScheduledAt( + event.getScheduledAt() == null ? null : event.getScheduledAt().toString()); + templateDataValue.setFollowup(event.isFollowup()); + templateDataValue.setDeleted(event.isDeleted()); + templateDataValue.setCreatedAt( + event.getCreatedAt() == null ? null : event.getCreatedAt().toString()); + templateDataValue.setCreatedAtClient( + event.getCreatedAtClient() == null ? null : event.getCreatedAtClient().toString()); + templateDataValue.setUpdatedAt( + event.getUpdatedAt() == null ? null : event.getUpdatedAt().toString()); + templateDataValue.setUpdatedAtClient( + event.getUpdatedAtClient() == null ? null : event.getUpdatedAtClient().toString()); + templateDataValue.setCompletedAt( + event.getCompletedAt() == null ? null : event.getCompletedAt().toString()); + templateDataValue.setUpdatedBy( + event.getUpdatedBy() == null ? null : event.getUpdatedBy().getUsername()); + templateDataValue.setStoredBy(event.getStoredBy()); + templateDataValue.setCompletedAt( + event.getCompletedAt() == null ? null : event.getCompletedAt().toString()); + templateDataValue.setCompletedBy(event.getCompletedBy()); + templateDataValue.setAttributeOptionCombo(event.getAttributeOptionCombo()); + templateDataValue.setAttributeCategoryOptions(event.getAttributeCategoryOptions()); + templateDataValue.setAssignedUser( + event.getAssignedUser() == null ? null : event.getAssignedUser().getUsername()); + + if (event.getGeometry() != null) { + templateDataValue.setGeometry(event.getGeometry().toText()); - dataValues.add(dataValue); + if (event.getGeometry().getGeometryType().equals("Point")) { + templateDataValue.setLongitude(event.getGeometry().getCoordinate().x); + templateDataValue.setLatitude(event.getGeometry().getCoordinate().y); } } - - writer.writeValue(outputStream, dataValues); + return templateDataValue; } @Override public void writeZip( OutputStream outputStream, List toCompress, boolean withHeader, String file) throws IOException { - CompressionUtil.writeZip(outputStream, toCompress, getObjectWriter(withHeader), file); + CompressionUtil.writeZip( + outputStream, getCsvEventDataValues(toCompress), getObjectWriter(withHeader), file); } @Override public void writeGzip(OutputStream outputStream, List toCompress, boolean withHeader) throws IOException { - CompressionUtil.writeGzip(outputStream, toCompress, getObjectWriter(withHeader)); + CompressionUtil.writeGzip( + outputStream, getCsvEventDataValues(toCompress), getObjectWriter(withHeader)); } private ObjectWriter getObjectWriter(boolean withHeader) { @@ -166,6 +161,24 @@ private ObjectWriter getObjectWriter(boolean withHeader) { return CSV_MAPPER.writer(csvSchema.withUseHeader(withHeader)); } + private List getCsvEventDataValues(List events) { + List dataValues = new ArrayList<>(); + + for (Event event : events) { + CsvEventDataValue templateDataValue = map(event); + + if (event.getDataValues().isEmpty()) { + dataValues.add(templateDataValue); + continue; + } + + for (DataValue value : event.getDataValues()) { + dataValues.add(map(value, templateDataValue)); + } + } + return dataValues; + } + @Override public List readEvents(InputStream inputStream, boolean skipFirst) throws IOException, ParseException { diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvTrackedEntityService.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvTrackedEntityService.java index 66797845ae5b..7f2435ae6608 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvTrackedEntityService.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvTrackedEntityService.java @@ -55,6 +55,36 @@ public void writeEvents( throws IOException { ObjectWriter writer = getObjectWriter(withHeader); + writer.writeValue(outputStream, getCsvTrackedEntities(trackedEntities)); + } + + @Override + public void writeZip( + OutputStream outputStream, List toCompress, boolean withHeader, String file) + throws IOException { + CompressionUtil.writeZip( + outputStream, getCsvTrackedEntities(toCompress), getObjectWriter(withHeader), file); + } + + @Override + public void writeGzip( + OutputStream outputStream, List toCompress, boolean withHeader) + throws IOException { + CompressionUtil.writeGzip( + outputStream, getCsvTrackedEntities(toCompress), getObjectWriter(withHeader)); + } + + private ObjectWriter getObjectWriter(boolean withHeader) { + final CsvSchema csvSchema = + CSV_MAPPER + .schemaFor(CsvTrackedEntity.class) + .withLineSeparator("\n") + .withUseHeader(withHeader); + + return CSV_MAPPER.writer(csvSchema.withUseHeader(withHeader)); + } + + private List getCsvTrackedEntities(List trackedEntities) { List attributes = new ArrayList<>(); for (TrackedEntity trackedEntity : trackedEntities) { @@ -90,32 +120,7 @@ public void writeEvents( addAttributes(trackedEntity, trackedEntityValue, attributes); } } - - writer.writeValue(outputStream, attributes); - } - - @Override - public void writeZip( - OutputStream outputStream, List toCompress, boolean withHeader, String file) - throws IOException { - CompressionUtil.writeZip(outputStream, toCompress, getObjectWriter(withHeader), file); - } - - @Override - public void writeGzip( - OutputStream outputStream, List toCompress, boolean withHeader) - throws IOException { - CompressionUtil.writeGzip(outputStream, toCompress, getObjectWriter(withHeader)); - } - - private ObjectWriter getObjectWriter(boolean withHeader) { - final CsvSchema csvSchema = - CSV_MAPPER - .schemaFor(CsvTrackedEntity.class) - .withLineSeparator("\n") - .withUseHeader(withHeader); - - return CSV_MAPPER.writer(csvSchema.withUseHeader(withHeader)); + return attributes; } private String checkForNull(Instant instant) { diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtilTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtilTest.java index 457b712493c8..5a55806b7981 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtilTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtilTest.java @@ -97,14 +97,14 @@ void shouldUnzipFileAndMatchEventsWhenCreateZipFileFromEventList() throws IOExce .filter(e -> e.getEvent().equals(FIRST_EVENT.getEvent())) .findAny() .orElse(null), - "The event does not match or does not exist in the Zip File."); + "The event does not match or not exists in the Zip File."); assertEquals( SECOND_EVENT, eventsFromZip.stream() .filter(e -> e.getEvent().equals(SECOND_EVENT.getEvent())) .findAny() .orElse(null), - "The event does not match or does not exist in the Zip File."); + "The event does not match or not exists in the Zip File."); } @Test @@ -135,14 +135,14 @@ void shouldGUnzipFileAndMatchEventsWhenCreateGZipFileFromEventList() throws IOEx .filter(e -> e.getEvent().equals(FIRST_EVENT.getEvent())) .findAny() .orElse(null), - "The event does not match or does not exist in the GZip File."); + "The event does not match or not exists in the GZip File."); assertEquals( SECOND_EVENT, eventToGZip.stream() .filter(e -> e.getEvent().equals(SECOND_EVENT.getEvent())) .findAny() .orElse(null), - "The event does not match or does not exist in the GZip File."); + "The event does not match or not exists in the GZip File."); } List getEvents() { diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvEventServiceTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvEventServiceTest.java index 4ae1459b87b9..a5a3a5488e5a 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvEventServiceTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvEventServiceTest.java @@ -29,6 +29,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -42,6 +43,9 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.hisp.dhis.event.EventStatus; import org.hisp.dhis.webapi.controller.tracker.view.DataValue; import org.hisp.dhis.webapi.controller.tracker.view.Event; @@ -224,4 +228,65 @@ void testReadEventsParsesGeometryEvenIfQuoted(String csv) throws IOException, Pa geometryFactory.createPoint(new Coordinate(-11.4283223849698, 8.06311527044516)); assertEquals(expected, events.get(0).getGeometry()); } + + @Test + void zipFileAndMatchCsvEventDataValuesWhenCreateZipFileFromEventList() + throws IOException, ParseException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + File event = new File("src/test/resources/controller/tracker/csv/completeEvent.csv"); + InputStream inputStream = Files.asByteSource(event).openStream(); + + List events = service.readEvents(inputStream, false); + + service.writeZip(outputStream, events, false, "file.json.zip"); + + ZipInputStream zipInputStream = + new ZipInputStream(new ByteArrayInputStream(outputStream.toByteArray())); + var buff = new byte[1024]; + + ZipEntry zipEntry = zipInputStream.getNextEntry(); + + assertNotNull(zipEntry, "Events Zip file has no entry"); + assertEquals("file.json.zip", zipEntry.getName(), "Events Zip file has a wrong name"); + + var csvStream = new ByteArrayOutputStream(); + int l; + while ((l = zipInputStream.read(buff)) > 0) { + csvStream.write(buff, 0, l); + } + + assertEquals( + "eventId,COMPLETED,programId,programStageId,enrollmentId,orgUnitId,2020-02-26T23:01:00Z,2020-02-26T23:02:00Z,,,,false,false,2020-02-26T23:03:00Z,,2020-02-26T23:05:00Z,,admin,2020-02-26T23:07:00Z,,attributeOptionCombo,,,dataElement,value,admin,false,,2020-02-26T23:08:00Z,2020-02-26T23:09:00Z", + csvStream.toString(), + "The event does not match or not exists in the Zip File."); + } + + @Test + void gzipFileAndMatchCsvEventDataValuesWhenCreateGZipFileFromEventList() + throws IOException, ParseException { + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + File event = new File("src/test/resources/controller/tracker/csv/completeEvent.csv"); + InputStream inputStream = Files.asByteSource(event).openStream(); + + List events = service.readEvents(inputStream, false); + service.writeGzip(outputStream, events, false); + + GZIPInputStream gzipInputStream = + new GZIPInputStream(new ByteArrayInputStream(outputStream.toByteArray())); + var buff = new byte[1024]; + + var csvStream = new ByteArrayOutputStream(); + int l; + while ((l = gzipInputStream.read(buff)) > 0) { + csvStream.write(buff, 0, l); + } + + assertEquals( + "eventId,COMPLETED,programId,programStageId,enrollmentId,orgUnitId,2020-02-26T23:01:00Z,2020-02-26T23:02:00Z,,,,false,false,2020-02-26T23:03:00Z,,2020-02-26T23:05:00Z,,admin,2020-02-26T23:07:00Z,,attributeOptionCombo,,,dataElement,value,admin,false,,2020-02-26T23:08:00Z,2020-02-26T23:09:00Z", + csvStream.toString(), + "The event does not match or not exists in the GZip File."); + } } diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvTrackedEntityServiceTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvTrackedEntityServiceTest.java index c01a6a81be10..524ae4c8deed 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvTrackedEntityServiceTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvTrackedEntityServiceTest.java @@ -28,7 +28,9 @@ package org.hisp.dhis.webapi.controller.tracker.export.csv; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.time.Instant; @@ -36,6 +38,9 @@ import java.util.Arrays; import java.util.List; import java.util.stream.IntStream; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.geotools.geojson.geom.GeometryJSON; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.webapi.controller.tracker.view.Attribute; @@ -146,6 +151,80 @@ void multipleTrackedEntitiesWithAttributesAreWritten() throws IOException { out.toString()); } + @Test + void zipFileAndMatchCsvTeWhenCreateZipFileFromTrackedEntityList() throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + List trackedEntities = new ArrayList<>(); + + trackedEntities.add(getTrackedEntityToCompress()); + + service.writeZip(outputStream, trackedEntities, false, "file.json.zip"); + + ZipInputStream zipInputStream = + new ZipInputStream(new ByteArrayInputStream(outputStream.toByteArray())); + var buff = new byte[1024]; + + ZipEntry zipEntry = zipInputStream.getNextEntry(); + + assertNotNull(zipEntry, "Events Zip file has no entry"); + assertEquals("file.json.zip", zipEntry.getName(), "Events Zip file has a wrong name"); + + var csvStream = new ByteArrayOutputStream(); + int l; + while ((l = zipInputStream.read(buff)) > 0) { + csvStream.write(buff, 0, l); + } + + assertEquals( + "\"Test tracked entity\",,2022-09-29T15:15:30Z,,,,\"Test org unit\",false,false,false,\"POINT (40 5)\",5.0,40.0,,,,,,\"attribute 1\",,\"Age test\",AGE\n" + + "\"Test tracked entity\",,2022-09-29T15:15:30Z,,,,\"Test org unit\",false,false,false,\"POINT (40 5)\",5.0,40.0,,,,,,\"attribute 2\",,\"Text test\",TEXT", + csvStream.toString(), + "The tracked entity does not match or not exists in the Zip File."); + } + + @Test + void gzipFileAndMatchCsvTeWhenCreateGZipFileFromTrackedEntityList() throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + List trackedEntities = new ArrayList<>(); + + trackedEntities.add(getTrackedEntityToCompress()); + + service.writeGzip(outputStream, trackedEntities, false); + + GZIPInputStream gzipInputStream = + new GZIPInputStream(new ByteArrayInputStream(outputStream.toByteArray())); + var buff = new byte[1024]; + + var csvStream = new ByteArrayOutputStream(); + int l; + while ((l = gzipInputStream.read(buff)) > 0) { + csvStream.write(buff, 0, l); + } + + assertEquals( + "\"Test tracked entity\",,2022-09-29T15:15:30Z,,,,\"Test org unit\",false,false,false,\"POINT (40 5)\",5.0,40.0,,,,,,\"attribute 1\",,\"Age test\",AGE\n" + + "\"Test tracked entity\",,2022-09-29T15:15:30Z,,,,\"Test org unit\",false,false,false,\"POINT (40 5)\",5.0,40.0,,,,,,\"attribute 2\",,\"Text test\",TEXT", + csvStream.toString(), + "The tracked entity does not match or not exists in the Zip File."); + } + + private TrackedEntity getTrackedEntityToCompress() throws IOException { + TrackedEntity trackedEntity = createTrackedEntity(); + + GeometryJSON geometryJSON = new GeometryJSON(); + String pointCoordinates = "[40,5]"; + Geometry geometry = + geometryJSON.read("{\"type\":\"Point\", \"coordinates\": " + pointCoordinates + " }"); + trackedEntity.setGeometry(geometry); + + Attribute attribute1 = createAttribute("attribute 1", ValueType.AGE, "Age test"); + Attribute attribute2 = createAttribute("attribute 2", ValueType.TEXT, "Text test"); + trackedEntity.setAttributes(Arrays.asList(attribute1, attribute2)); + return trackedEntity; + } + private Attribute createAttribute(String attr, ValueType valueType, String value) { Attribute attribute = new Attribute(); attribute.setAttribute(attr); From b9f86a2688c1d6ad3600c76f8905a5d3c5fef19f Mon Sep 17 00:00:00 2001 From: luca Date: Thu, 4 Jan 2024 12:34:20 +0100 Subject: [PATCH 4/4] fix: unit tests --- .../tracker/export/csv/TrackerCsvEventServiceTest.java | 4 ++-- .../export/csv/TrackerCsvTrackedEntityServiceTest.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvEventServiceTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvEventServiceTest.java index a5a3a5488e5a..d2e23d87c16a 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvEventServiceTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvEventServiceTest.java @@ -257,7 +257,7 @@ void zipFileAndMatchCsvEventDataValuesWhenCreateZipFileFromEventList() } assertEquals( - "eventId,COMPLETED,programId,programStageId,enrollmentId,orgUnitId,2020-02-26T23:01:00Z,2020-02-26T23:02:00Z,,,,false,false,2020-02-26T23:03:00Z,,2020-02-26T23:05:00Z,,admin,2020-02-26T23:07:00Z,,attributeOptionCombo,,,dataElement,value,admin,false,,2020-02-26T23:08:00Z,2020-02-26T23:09:00Z", + "eventId,COMPLETED,programId,programStageId,enrollmentId,orgUnitId,,2020-02-26T23:01:00Z,2020-02-26T23:02:00Z,,,,false,false,2020-02-26T23:03:00Z,,2020-02-26T23:05:00Z,,admin,2020-02-26T23:07:00Z,,,,,dataElement,value,admin,false,,2020-02-26T23:08:00Z,2020-02-26T23:09:00Z\n", csvStream.toString(), "The event does not match or not exists in the Zip File."); } @@ -285,7 +285,7 @@ void gzipFileAndMatchCsvEventDataValuesWhenCreateGZipFileFromEventList() } assertEquals( - "eventId,COMPLETED,programId,programStageId,enrollmentId,orgUnitId,2020-02-26T23:01:00Z,2020-02-26T23:02:00Z,,,,false,false,2020-02-26T23:03:00Z,,2020-02-26T23:05:00Z,,admin,2020-02-26T23:07:00Z,,attributeOptionCombo,,,dataElement,value,admin,false,,2020-02-26T23:08:00Z,2020-02-26T23:09:00Z", + "eventId,COMPLETED,programId,programStageId,enrollmentId,orgUnitId,,2020-02-26T23:01:00Z,2020-02-26T23:02:00Z,,,,false,false,2020-02-26T23:03:00Z,,2020-02-26T23:05:00Z,,admin,2020-02-26T23:07:00Z,,,,,dataElement,value,admin,false,,2020-02-26T23:08:00Z,2020-02-26T23:09:00Z\n", csvStream.toString(), "The event does not match or not exists in the GZip File."); } diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvTrackedEntityServiceTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvTrackedEntityServiceTest.java index 524ae4c8deed..b79683d418ec 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvTrackedEntityServiceTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/csv/TrackerCsvTrackedEntityServiceTest.java @@ -178,7 +178,7 @@ void zipFileAndMatchCsvTeWhenCreateZipFileFromTrackedEntityList() throws IOExcep assertEquals( "\"Test tracked entity\",,2022-09-29T15:15:30Z,,,,\"Test org unit\",false,false,false,\"POINT (40 5)\",5.0,40.0,,,,,,\"attribute 1\",,\"Age test\",AGE\n" - + "\"Test tracked entity\",,2022-09-29T15:15:30Z,,,,\"Test org unit\",false,false,false,\"POINT (40 5)\",5.0,40.0,,,,,,\"attribute 2\",,\"Text test\",TEXT", + + "\"Test tracked entity\",,2022-09-29T15:15:30Z,,,,\"Test org unit\",false,false,false,\"POINT (40 5)\",5.0,40.0,,,,,,\"attribute 2\",,\"Text test\",TEXT\n", csvStream.toString(), "The tracked entity does not match or not exists in the Zip File."); } @@ -205,7 +205,7 @@ void gzipFileAndMatchCsvTeWhenCreateGZipFileFromTrackedEntityList() throws IOExc assertEquals( "\"Test tracked entity\",,2022-09-29T15:15:30Z,,,,\"Test org unit\",false,false,false,\"POINT (40 5)\",5.0,40.0,,,,,,\"attribute 1\",,\"Age test\",AGE\n" - + "\"Test tracked entity\",,2022-09-29T15:15:30Z,,,,\"Test org unit\",false,false,false,\"POINT (40 5)\",5.0,40.0,,,,,,\"attribute 2\",,\"Text test\",TEXT", + + "\"Test tracked entity\",,2022-09-29T15:15:30Z,,,,\"Test org unit\",false,false,false,\"POINT (40 5)\",5.0,40.0,,,,,,\"attribute 2\",,\"Text test\",TEXT\n", csvStream.toString(), "The tracked entity does not match or not exists in the Zip File."); }