From 0c035b4a8bc3ec883b1cf6fc92e3bc5ffdf72e34 Mon Sep 17 00:00:00 2001 From: luca Date: Tue, 19 Dec 2023 15:03:27 +0100 Subject: [PATCH 1/9] task: add gzip/unzip support to tracker/events --- .../event/EventExportTestConfiguration.java | 51 ++ .../event/EventsExportControllerByIdTest.java | 523 ++++++++++++++++++ .../event/EventsExportControllerTest.java | 472 ++-------------- .../event/EventsExportControllerUnitTest.java | 5 +- .../export/event/CompressedEventService.java | 62 +++ .../export/event/EventRequestParams.java | 2 + .../export/event/EventsExportController.java | 67 ++- .../hisp/dhis/webapi/utils/ContextUtils.java | 3 +- .../event/CompressedEventServiceTest.java | 151 +++++ 9 files changed, 908 insertions(+), 428 deletions(-) create mode 100644 dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventExportTestConfiguration.java create mode 100644 dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerByIdTest.java create mode 100644 dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CompressedEventService.java create mode 100644 dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CompressedEventServiceTest.java diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventExportTestConfiguration.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventExportTestConfiguration.java new file mode 100644 index 000000000000..6853eef5784b --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventExportTestConfiguration.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.tracker.export.event; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.HashSet; +import org.hisp.dhis.tracker.export.event.EventService; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; + +@Configuration +public class EventExportTestConfiguration { + + @Primary + @Bean + public EventService eventService() { + EventService eventService = mock(EventService.class); + // Orderable fields are checked within the controller constructor + when(eventService.getOrderableFields()) + .thenReturn(new HashSet<>(EventMapper.ORDERABLE_FIELDS.values())); + return eventService; + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerByIdTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerByIdTest.java new file mode 100644 index 000000000000..62022869a965 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerByIdTest.java @@ -0,0 +1,523 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.tracker.export.event; + +import static org.hisp.dhis.webapi.controller.tracker.JsonAssertions.assertHasMember; +import static org.hisp.dhis.webapi.controller.tracker.JsonAssertions.assertHasNoMember; +import static org.hisp.dhis.webapi.controller.tracker.JsonAssertions.assertHasOnlyMembers; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Date; +import java.util.List; +import java.util.Set; +import org.hisp.dhis.common.CodeGenerator; +import org.hisp.dhis.common.IdentifiableObjectManager; +import org.hisp.dhis.dataelement.DataElement; +import org.hisp.dhis.eventdatavalue.EventDataValue; +import org.hisp.dhis.jsontree.JsonList; +import org.hisp.dhis.jsontree.JsonObject; +import org.hisp.dhis.note.Note; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.program.Enrollment; +import org.hisp.dhis.program.Event; +import org.hisp.dhis.program.Program; +import org.hisp.dhis.program.ProgramStage; +import org.hisp.dhis.program.ProgramStatus; +import org.hisp.dhis.program.UserInfoSnapshot; +import org.hisp.dhis.relationship.Relationship; +import org.hisp.dhis.relationship.RelationshipEntity; +import org.hisp.dhis.relationship.RelationshipItem; +import org.hisp.dhis.relationship.RelationshipType; +import org.hisp.dhis.security.acl.AccessStringHelper; +import org.hisp.dhis.trackedentity.TrackedEntity; +import org.hisp.dhis.trackedentity.TrackedEntityType; +import org.hisp.dhis.user.User; +import org.hisp.dhis.user.sharing.UserAccess; +import org.hisp.dhis.web.HttpStatus; +import org.hisp.dhis.webapi.DhisControllerConvenienceTest; +import org.hisp.dhis.webapi.controller.tracker.JsonDataValue; +import org.hisp.dhis.webapi.controller.tracker.JsonEvent; +import org.hisp.dhis.webapi.controller.tracker.JsonNote; +import org.hisp.dhis.webapi.controller.tracker.JsonRelationship; +import org.hisp.dhis.webapi.controller.tracker.JsonRelationshipItem; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +class EventsExportControllerByIdTest extends DhisControllerConvenienceTest { + private static final String DATA_ELEMENT_VALUE = "value"; + + @Autowired private IdentifiableObjectManager manager; + + private OrganisationUnit orgUnit; + + private OrganisationUnit anotherOrgUnit; + + private Program program; + + private ProgramStage programStage; + + private User owner; + + private User user; + + private TrackedEntityType trackedEntityType; + + private EventDataValue dv; + + private DataElement de; + + @BeforeEach + void setUp() { + owner = makeUser("owner"); + + orgUnit = createOrganisationUnit('A'); + orgUnit.getSharing().setOwner(owner); + manager.save(orgUnit, false); + + anotherOrgUnit = createOrganisationUnit('B'); + anotherOrgUnit.getSharing().setOwner(owner); + manager.save(anotherOrgUnit, false); + + user = createUserWithId("tester", CodeGenerator.generateUid()); + user.addOrganisationUnit(orgUnit); + user.setTeiSearchOrganisationUnits(Set.of(orgUnit)); + this.userService.updateUser(user); + + program = createProgram('A'); + program.addOrganisationUnit(orgUnit); + program.getSharing().setOwner(owner); + program.getSharing().addUserAccess(userAccess()); + manager.save(program, false); + + programStage = createProgramStage('A', program); + programStage.getSharing().setOwner(owner); + programStage.getSharing().addUserAccess(userAccess()); + manager.save(programStage, false); + + de = createDataElement('A'); + de.getSharing().setOwner(owner); + manager.save(de, false); + + dv = new EventDataValue(); + dv.setDataElement(de.getUid()); + dv.setStoredBy("user"); + dv.setValue(DATA_ELEMENT_VALUE); + + trackedEntityType = trackedEntityTypeAccessible(); + } + + @Test + void getEventById() { + Event event = event(enrollment(trackedEntity())); + + JsonEvent json = + GET("/tracker/events/{id}", event.getUid()).content(HttpStatus.OK).as(JsonEvent.class); + + assertDefaultResponse(json, event); + } + + @Test + void getEventByIdWithFields() { + Event event = event(enrollment(trackedEntity())); + + JsonEvent jsonEvent = + GET("/tracker/events/{id}?fields=orgUnit,status", event.getUid()) + .content(HttpStatus.OK) + .as(JsonEvent.class); + + assertHasOnlyMembers(jsonEvent, "orgUnit", "status"); + assertEquals(event.getOrganisationUnit().getUid(), jsonEvent.getOrgUnit()); + assertEquals(event.getStatus().toString(), jsonEvent.getStatus()); + } + + @Test + void getEventByIdWithNotes() { + Event event = event(enrollment(trackedEntity())); + event.setNotes(List.of(note("oqXG28h988k", "my notes", owner.getUid()))); + manager.update(event); + + JsonEvent jsonEvent = + GET("/tracker/events/{uid}?fields=notes", event.getUid()) + .content(HttpStatus.OK) + .as(JsonEvent.class); + + JsonNote note = jsonEvent.getNotes().get(0); + assertEquals("oqXG28h988k", note.getNote()); + assertEquals("my notes", note.getValue()); + assertEquals(owner.getUid(), note.getStoredBy()); + } + + @Test + void getEventByIdWithDataValues() { + Event event = event(enrollment(trackedEntity())); + event.getEventDataValues().add(dv); + manager.update(event); + + JsonEvent eventJson = + GET("/tracker/events/{id}?fields=dataValues", event.getUid()) + .content(HttpStatus.OK) + .as(JsonEvent.class); + + assertHasOnlyMembers(eventJson, "dataValues"); + JsonDataValue dataValue = eventJson.getDataValues().get(0); + assertEquals(de.getUid(), dataValue.getDataElement()); + assertEquals(dv.getValue(), dataValue.getValue()); + assertHasMember(dataValue, "createdAt"); + assertHasMember(dataValue, "updatedAt"); + assertHasMember(dataValue, "storedBy"); + } + + @Test + void getEventByIdWithFieldsRelationships() { + TrackedEntity to = trackedEntity(); + Event from = event(enrollment(to)); + Relationship relationship = relationship(from, to); + + JsonList relationships = + GET("/tracker/events/{id}?fields=relationships", from.getUid()) + .content(HttpStatus.OK) + .getList("relationships", JsonRelationship.class); + + JsonRelationship jsonRelationship = relationships.get(0); + assertEquals(relationship.getUid(), jsonRelationship.getRelationship()); + + JsonRelationshipItem.JsonEvent event = jsonRelationship.getFrom().getEvent(); + assertEquals(relationship.getFrom().getEvent().getUid(), event.getEvent()); + assertEquals(relationship.getFrom().getEvent().getEnrollment().getUid(), event.getEnrollment()); + + JsonRelationshipItem.JsonTrackedEntity trackedEntity = + jsonRelationship.getTo().getTrackedEntity(); + assertEquals( + relationship.getTo().getTrackedEntity().getUid(), trackedEntity.getTrackedEntity()); + + assertHasMember(jsonRelationship, "relationshipName"); + assertHasMember(jsonRelationship, "relationshipType"); + assertHasMember(jsonRelationship, "createdAt"); + assertHasMember(jsonRelationship, "updatedAt"); + assertHasMember(jsonRelationship, "bidirectional"); + } + + @Test + void getEventByIdRelationshipsNoAccessToRelationshipType() { + TrackedEntity to = trackedEntity(); + Event from = event(enrollment(to)); + relationship(relationshipTypeNotAccessible(), from, to); + this.switchContextToUser(user); + + JsonList relationships = + GET("/tracker/events/{id}?fields=relationships", from.getUid()) + .content(HttpStatus.OK) + .getList("relationships", JsonRelationship.class); + + assertEquals(0, relationships.size()); + } + + @Test + void getEventByIdRelationshipsNoAccessToRelationshipItemTo() { + TrackedEntityType type = trackedEntityTypeNotAccessible(); + TrackedEntity to = trackedEntity(type); + Event from = event(enrollment(to)); + relationship(from, to); + this.switchContextToUser(user); + + JsonList relationships = + GET("/tracker/events/{id}?fields=relationships", from.getUid()) + .content(HttpStatus.OK) + .getList("relationships", JsonRelationship.class); + + assertEquals(0, relationships.size()); + } + + @Test + void getEventByIdRelationshipsNoAccessToBothRelationshipItems() { + TrackedEntity to = trackedEntityNotInSearchScope(); + Event from = event(enrollment(to)); + relationship(from, to); + this.switchContextToUser(user); + + assertTrue( + GET("/tracker/events/{id}", from.getUid()) + .error(HttpStatus.FORBIDDEN) + .getMessage() + .contains("OWNERSHIP_ACCESS_DENIED")); + } + + @Test + void getEventByIdRelationshipsNoAccessToRelationshipItemFrom() { + TrackedEntityType type = trackedEntityTypeNotAccessible(); + TrackedEntity from = trackedEntity(type); + Event to = event(enrollment(from)); + relationship(from, to); + this.switchContextToUser(user); + + JsonList relationships = + GET("/tracker/events/{id}?fields=relationships", to.getUid()) + .content(HttpStatus.OK) + .getList("relationships", JsonRelationship.class); + + assertEquals(0, relationships.size()); + } + + @Test + void getEventByIdContainsCreatedByAndUpdateByAndAssignedUserInDataValues() { + + TrackedEntity te = trackedEntity(); + Enrollment enrollment = enrollment(te); + Event event = event(enrollment); + event.setCreatedByUserInfo(UserInfoSnapshot.from(user)); + event.setLastUpdatedByUserInfo(UserInfoSnapshot.from(user)); + event.setAssignedUser(user); + EventDataValue eventDataValue = new EventDataValue(); + eventDataValue.setValue("6"); + + eventDataValue.setDataElement(de.getUid()); + eventDataValue.setCreatedByUserInfo(UserInfoSnapshot.from(user)); + eventDataValue.setLastUpdatedByUserInfo(UserInfoSnapshot.from(user)); + Set eventDataValues = Set.of(eventDataValue); + event.setEventDataValues(eventDataValues); + manager.save(event); + + JsonObject jsonEvent = GET("/tracker/events/{id}", event.getUid()).content(HttpStatus.OK); + + assertTrue(jsonEvent.isObject()); + assertFalse(jsonEvent.isEmpty()); + assertEquals(event.getUid(), jsonEvent.getString("event").string()); + assertEquals(enrollment.getUid(), jsonEvent.getString("enrollment").string()); + assertEquals(orgUnit.getUid(), jsonEvent.getString("orgUnit").string()); + assertEquals(user.getUsername(), jsonEvent.getString("createdBy.username").string()); + assertEquals(user.getUsername(), jsonEvent.getString("updatedBy.username").string()); + assertEquals(user.getDisplayName(), jsonEvent.getString("assignedUser.displayName").string()); + assertFalse(jsonEvent.getArray("dataValues").isEmpty()); + assertEquals( + user.getUsername(), + jsonEvent.getArray("dataValues").getObject(0).getString("createdBy.username").string()); + assertEquals( + user.getUsername(), + jsonEvent.getArray("dataValues").getObject(0).getString("updatedBy.username").string()); + } + + @Test + void getEventByIdNotFound() { + assertEquals( + "Event with id Hq3Kc6HK4OZ could not be found.", + GET("/tracker/events/Hq3Kc6HK4OZ").error(HttpStatus.NOT_FOUND).getMessage()); + } + + private TrackedEntityType trackedEntityTypeAccessible() { + TrackedEntityType type = trackedEntityType('A'); + type.getSharing().addUserAccess(userAccess()); + manager.save(type, false); + return type; + } + + private TrackedEntityType trackedEntityTypeNotAccessible() { + TrackedEntityType type = trackedEntityType('B'); + manager.save(type, false); + return type; + } + + private TrackedEntityType trackedEntityType(char uniqueChar) { + TrackedEntityType type = createTrackedEntityType(uniqueChar); + type.getSharing().setOwner(owner); + type.getSharing().setPublicAccess(AccessStringHelper.DEFAULT); + return type; + } + + private TrackedEntity trackedEntity() { + TrackedEntity te = trackedEntity(orgUnit); + manager.save(te, false); + return te; + } + + private TrackedEntity trackedEntityNotInSearchScope() { + TrackedEntity te = trackedEntity(anotherOrgUnit); + manager.save(te, false); + return te; + } + + private TrackedEntity trackedEntity(TrackedEntityType trackedEntityType) { + TrackedEntity te = trackedEntity(orgUnit, trackedEntityType); + manager.save(te, false); + return te; + } + + private TrackedEntity trackedEntity(OrganisationUnit orgUnit) { + return trackedEntity(orgUnit, trackedEntityType); + } + + private TrackedEntity trackedEntity( + OrganisationUnit orgUnit, TrackedEntityType trackedEntityType) { + TrackedEntity te = createTrackedEntity(orgUnit); + te.setTrackedEntityType(trackedEntityType); + te.getSharing().setPublicAccess(AccessStringHelper.DEFAULT); + te.getSharing().setOwner(owner); + return te; + } + + private Enrollment enrollment(TrackedEntity te) { + Enrollment enrollment = new Enrollment(program, te, te.getOrganisationUnit()); + enrollment.setAutoFields(); + enrollment.setEnrollmentDate(new Date()); + enrollment.setOccurredDate(new Date()); + enrollment.setStatus(ProgramStatus.COMPLETED); + manager.save(enrollment); + return enrollment; + } + + private Event event(Enrollment enrollment) { + Event event = new Event(enrollment, programStage, enrollment.getOrganisationUnit()); + event.setAutoFields(); + manager.save(event); + return event; + } + + private UserAccess userAccess() { + UserAccess a = new UserAccess(); + a.setUser(user); + a.setAccess(AccessStringHelper.FULL); + return a; + } + + private RelationshipType relationshipTypeAccessible( + RelationshipEntity from, RelationshipEntity to) { + RelationshipType type = relationshipType(from, to); + type.getSharing().addUserAccess(userAccess()); + manager.save(type, false); + return type; + } + + private RelationshipType relationshipTypeNotAccessible() { + return relationshipType( + RelationshipEntity.PROGRAM_STAGE_INSTANCE, RelationshipEntity.TRACKED_ENTITY_INSTANCE); + } + + private RelationshipType relationshipType(RelationshipEntity from, RelationshipEntity to) { + RelationshipType type = createRelationshipType('A'); + type.getFromConstraint().setRelationshipEntity(from); + type.getToConstraint().setRelationshipEntity(to); + type.getSharing().setOwner(owner); + type.getSharing().setPublicAccess(AccessStringHelper.DEFAULT); + manager.save(type, false); + return type; + } + + private Relationship relationship(Event from, TrackedEntity to) { + return relationship( + relationshipTypeAccessible( + RelationshipEntity.PROGRAM_STAGE_INSTANCE, RelationshipEntity.TRACKED_ENTITY_INSTANCE), + from, + to); + } + + private Relationship relationship(RelationshipType type, Event from, TrackedEntity to) { + Relationship r = new Relationship(); + + RelationshipItem fromItem = new RelationshipItem(); + fromItem.setEvent(from); + from.getRelationshipItems().add(fromItem); + fromItem.setRelationship(r); + r.setFrom(fromItem); + + RelationshipItem toItem = new RelationshipItem(); + toItem.setTrackedEntity(to); + to.getRelationshipItems().add(toItem); + r.setTo(toItem); + toItem.setRelationship(r); + + r.setRelationshipType(type); + r.setKey(type.getUid()); + r.setInvertedKey(type.getUid()); + r.setAutoFields(); + r.getSharing().setOwner(owner); + manager.save(r, false); + return r; + } + + private void relationship(TrackedEntity from, Event to) { + Relationship r = new Relationship(); + + RelationshipItem fromItem = new RelationshipItem(); + fromItem.setTrackedEntity(from); + from.getRelationshipItems().add(fromItem); + r.setFrom(fromItem); + fromItem.setRelationship(r); + + RelationshipItem toItem = new RelationshipItem(); + toItem.setEvent(to); + to.getRelationshipItems().add(toItem); + r.setTo(toItem); + toItem.setRelationship(r); + + RelationshipType type = + relationshipTypeAccessible( + RelationshipEntity.PROGRAM_STAGE_INSTANCE, RelationshipEntity.TRACKED_ENTITY_INSTANCE); + r.setRelationshipType(type); + r.setKey(type.getUid()); + r.setInvertedKey(type.getUid()); + + r.setAutoFields(); + r.getSharing().setOwner(owner); + manager.save(r, false); + } + + private Note note(String uid, String value, String storedBy) { + Note note = new Note(value, storedBy); + note.setUid(uid); + manager.save(note, false); + return note; + } + + private void assertDefaultResponse(JsonObject json, Event event) { + // note that some fields are not included in the response because they + // are not part of the setup + // i.e. attributeOptionCombo, ... + assertTrue(json.isObject()); + assertFalse(json.isEmpty()); + assertEquals(event.getUid(), json.getString("event").string(), "event UID"); + assertEquals("ACTIVE", json.getString("status").string()); + assertEquals(program.getUid(), json.getString("program").string()); + assertEquals(programStage.getUid(), json.getString("programStage").string()); + assertEquals(event.getEnrollment().getUid(), json.getString("enrollment").string()); + assertEquals(orgUnit.getUid(), json.getString("orgUnit").string()); + assertFalse(json.getBoolean("followUp").booleanValue()); + assertFalse(json.getBoolean("followup").booleanValue()); + assertFalse(json.getBoolean("deleted").booleanValue()); + assertHasMember(json, "createdAt"); + assertHasMember(json, "createdAtClient"); + assertHasMember(json, "updatedAt"); + assertHasMember(json, "updatedAtClient"); + assertHasMember(json, "dataValues"); + assertHasMember(json, "notes"); + assertHasNoMember(json, "attributeOptionCombo"); + assertHasNoMember(json, "attributeCategoryOptions"); + assertHasNoMember(json, "relationships"); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java index 7bb7122af412..eea603c55573 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java @@ -28,82 +28,49 @@ package org.hisp.dhis.webapi.controller.tracker.export.event; import static org.hisp.dhis.utils.Assertions.assertStartsWith; -import static org.hisp.dhis.webapi.controller.tracker.JsonAssertions.assertHasMember; -import static org.hisp.dhis.webapi.controller.tracker.JsonAssertions.assertHasNoMember; -import static org.hisp.dhis.webapi.controller.tracker.JsonAssertions.assertHasOnlyMembers; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; -import java.util.Date; import java.util.List; import java.util.Set; import org.hisp.dhis.common.CodeGenerator; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.dataelement.DataElement; -import org.hisp.dhis.eventdatavalue.EventDataValue; -import org.hisp.dhis.jsontree.JsonList; -import org.hisp.dhis.jsontree.JsonObject; -import org.hisp.dhis.note.Note; +import org.hisp.dhis.feedback.BadRequestException; +import org.hisp.dhis.feedback.ForbiddenException; import org.hisp.dhis.organisationunit.OrganisationUnit; -import org.hisp.dhis.program.Enrollment; -import org.hisp.dhis.program.Event; import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramStage; -import org.hisp.dhis.program.ProgramStatus; -import org.hisp.dhis.program.UserInfoSnapshot; -import org.hisp.dhis.relationship.Relationship; -import org.hisp.dhis.relationship.RelationshipEntity; -import org.hisp.dhis.relationship.RelationshipItem; -import org.hisp.dhis.relationship.RelationshipType; import org.hisp.dhis.security.acl.AccessStringHelper; -import org.hisp.dhis.trackedentity.TrackedEntity; -import org.hisp.dhis.trackedentity.TrackedEntityType; +import org.hisp.dhis.tracker.export.event.EventService; import org.hisp.dhis.user.User; import org.hisp.dhis.user.sharing.UserAccess; import org.hisp.dhis.web.HttpStatus; import org.hisp.dhis.webapi.DhisControllerConvenienceTest; -import org.hisp.dhis.webapi.controller.tracker.JsonDataValue; -import org.hisp.dhis.webapi.controller.tracker.JsonEvent; -import org.hisp.dhis.webapi.controller.tracker.JsonNote; -import org.hisp.dhis.webapi.controller.tracker.JsonRelationship; -import org.hisp.dhis.webapi.controller.tracker.JsonRelationshipItem; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +@ContextConfiguration(classes = EventExportTestConfiguration.class) class EventsExportControllerTest extends DhisControllerConvenienceTest { - private static final String DATA_ELEMENT_VALUE = "value"; @Autowired private IdentifiableObjectManager manager; - private OrganisationUnit orgUnit; - - private OrganisationUnit anotherOrgUnit; - - private Program program; - - private ProgramStage programStage; - - private User owner; + @Autowired private EventService eventService; private User user; - private TrackedEntityType trackedEntityType; - - private EventDataValue dv; - - private DataElement de; - @BeforeEach void setUp() { - owner = makeUser("owner"); + User owner = makeUser("owner"); - orgUnit = createOrganisationUnit('A'); + OrganisationUnit orgUnit = createOrganisationUnit('A'); orgUnit.getSharing().setOwner(owner); manager.save(orgUnit, false); - anotherOrgUnit = createOrganisationUnit('B'); + OrganisationUnit anotherOrgUnit = createOrganisationUnit('B'); anotherOrgUnit.getSharing().setOwner(owner); manager.save(anotherOrgUnit, false); @@ -112,224 +79,20 @@ void setUp() { user.setTeiSearchOrganisationUnits(Set.of(orgUnit)); this.userService.updateUser(user); - program = createProgram('A'); + Program program = createProgram('A'); program.addOrganisationUnit(orgUnit); program.getSharing().setOwner(owner); program.getSharing().addUserAccess(userAccess()); manager.save(program, false); - programStage = createProgramStage('A', program); + ProgramStage programStage = createProgramStage('A', program); programStage.getSharing().setOwner(owner); programStage.getSharing().addUserAccess(userAccess()); manager.save(programStage, false); - de = createDataElement('A'); + DataElement de = createDataElement('A'); de.getSharing().setOwner(owner); manager.save(de, false); - - dv = new EventDataValue(); - dv.setDataElement(de.getUid()); - dv.setStoredBy("user"); - dv.setValue(DATA_ELEMENT_VALUE); - - trackedEntityType = trackedEntityTypeAccessible(); - } - - @Test - void getEventById() { - Event event = event(enrollment(trackedEntity())); - - JsonEvent json = - GET("/tracker/events/{id}", event.getUid()).content(HttpStatus.OK).as(JsonEvent.class); - - assertDefaultResponse(json, event); - } - - @Test - void getEventByIdWithFields() { - Event event = event(enrollment(trackedEntity())); - - JsonEvent jsonEvent = - GET("/tracker/events/{id}?fields=orgUnit,status", event.getUid()) - .content(HttpStatus.OK) - .as(JsonEvent.class); - - assertHasOnlyMembers(jsonEvent, "orgUnit", "status"); - assertEquals(event.getOrganisationUnit().getUid(), jsonEvent.getOrgUnit()); - assertEquals(event.getStatus().toString(), jsonEvent.getStatus()); - } - - @Test - void getEventByIdWithNotes() { - Event event = event(enrollment(trackedEntity())); - event.setNotes(List.of(note("oqXG28h988k", "my notes", owner.getUid()))); - manager.update(event); - - JsonEvent jsonEvent = - GET("/tracker/events/{uid}?fields=notes", event.getUid()) - .content(HttpStatus.OK) - .as(JsonEvent.class); - - JsonNote note = jsonEvent.getNotes().get(0); - assertEquals("oqXG28h988k", note.getNote()); - assertEquals("my notes", note.getValue()); - assertEquals(owner.getUid(), note.getStoredBy()); - } - - @Test - void getEventByIdWithDataValues() { - Event event = event(enrollment(trackedEntity())); - event.getEventDataValues().add(dv); - manager.update(event); - - JsonEvent eventJson = - GET("/tracker/events/{id}?fields=dataValues", event.getUid()) - .content(HttpStatus.OK) - .as(JsonEvent.class); - - assertHasOnlyMembers(eventJson, "dataValues"); - JsonDataValue dataValue = eventJson.getDataValues().get(0); - assertEquals(de.getUid(), dataValue.getDataElement()); - assertEquals(dv.getValue(), dataValue.getValue()); - assertHasMember(dataValue, "createdAt"); - assertHasMember(dataValue, "updatedAt"); - assertHasMember(dataValue, "storedBy"); - } - - @Test - void getEventByIdWithFieldsRelationships() { - TrackedEntity to = trackedEntity(); - Event from = event(enrollment(to)); - Relationship relationship = relationship(from, to); - - JsonList relationships = - GET("/tracker/events/{id}?fields=relationships", from.getUid()) - .content(HttpStatus.OK) - .getList("relationships", JsonRelationship.class); - - JsonRelationship jsonRelationship = relationships.get(0); - assertEquals(relationship.getUid(), jsonRelationship.getRelationship()); - - JsonRelationshipItem.JsonEvent event = jsonRelationship.getFrom().getEvent(); - assertEquals(relationship.getFrom().getEvent().getUid(), event.getEvent()); - assertEquals(relationship.getFrom().getEvent().getEnrollment().getUid(), event.getEnrollment()); - - JsonRelationshipItem.JsonTrackedEntity trackedEntity = - jsonRelationship.getTo().getTrackedEntity(); - assertEquals( - relationship.getTo().getTrackedEntity().getUid(), trackedEntity.getTrackedEntity()); - - assertHasMember(jsonRelationship, "relationshipName"); - assertHasMember(jsonRelationship, "relationshipType"); - assertHasMember(jsonRelationship, "createdAt"); - assertHasMember(jsonRelationship, "updatedAt"); - assertHasMember(jsonRelationship, "bidirectional"); - } - - @Test - void getEventByIdRelationshipsNoAccessToRelationshipType() { - TrackedEntity to = trackedEntity(); - Event from = event(enrollment(to)); - relationship(relationshipTypeNotAccessible(), from, to); - this.switchContextToUser(user); - - JsonList relationships = - GET("/tracker/events/{id}?fields=relationships", from.getUid()) - .content(HttpStatus.OK) - .getList("relationships", JsonRelationship.class); - - assertEquals(0, relationships.size()); - } - - @Test - void getEventByIdRelationshipsNoAccessToRelationshipItemTo() { - TrackedEntityType type = trackedEntityTypeNotAccessible(); - TrackedEntity to = trackedEntity(type); - Event from = event(enrollment(to)); - relationship(from, to); - this.switchContextToUser(user); - - JsonList relationships = - GET("/tracker/events/{id}?fields=relationships", from.getUid()) - .content(HttpStatus.OK) - .getList("relationships", JsonRelationship.class); - - assertEquals(0, relationships.size()); - } - - @Test - void getEventByIdRelationshipsNoAccessToBothRelationshipItems() { - TrackedEntity to = trackedEntityNotInSearchScope(); - Event from = event(enrollment(to)); - relationship(from, to); - this.switchContextToUser(user); - - assertTrue( - GET("/tracker/events/{id}", from.getUid()) - .error(HttpStatus.FORBIDDEN) - .getMessage() - .contains("OWNERSHIP_ACCESS_DENIED")); - } - - @Test - void getEventByIdRelationshipsNoAccessToRelationshipItemFrom() { - TrackedEntityType type = trackedEntityTypeNotAccessible(); - TrackedEntity from = trackedEntity(type); - Event to = event(enrollment(from)); - relationship(from, to); - this.switchContextToUser(user); - - JsonList relationships = - GET("/tracker/events/{id}?fields=relationships", to.getUid()) - .content(HttpStatus.OK) - .getList("relationships", JsonRelationship.class); - - assertEquals(0, relationships.size()); - } - - @Test - void getEventByIdContainsCreatedByAndUpdateByAndAssignedUserInDataValues() { - - TrackedEntity te = trackedEntity(); - Enrollment enrollment = enrollment(te); - Event event = event(enrollment); - event.setCreatedByUserInfo(UserInfoSnapshot.from(user)); - event.setLastUpdatedByUserInfo(UserInfoSnapshot.from(user)); - event.setAssignedUser(user); - EventDataValue eventDataValue = new EventDataValue(); - eventDataValue.setValue("6"); - - eventDataValue.setDataElement(de.getUid()); - eventDataValue.setCreatedByUserInfo(UserInfoSnapshot.from(user)); - eventDataValue.setLastUpdatedByUserInfo(UserInfoSnapshot.from(user)); - Set eventDataValues = Set.of(eventDataValue); - event.setEventDataValues(eventDataValues); - manager.save(event); - - JsonObject jsonEvent = GET("/tracker/events/{id}", event.getUid()).content(HttpStatus.OK); - - assertTrue(jsonEvent.isObject()); - assertFalse(jsonEvent.isEmpty()); - assertEquals(event.getUid(), jsonEvent.getString("event").string()); - assertEquals(enrollment.getUid(), jsonEvent.getString("enrollment").string()); - assertEquals(orgUnit.getUid(), jsonEvent.getString("orgUnit").string()); - assertEquals(user.getUsername(), jsonEvent.getString("createdBy.username").string()); - assertEquals(user.getUsername(), jsonEvent.getString("updatedBy.username").string()); - assertEquals(user.getDisplayName(), jsonEvent.getString("assignedUser.displayName").string()); - assertFalse(jsonEvent.getArray("dataValues").isEmpty()); - assertEquals( - user.getUsername(), - jsonEvent.getArray("dataValues").getObject(0).getString("createdBy.username").string()); - assertEquals( - user.getUsername(), - jsonEvent.getArray("dataValues").getObject(0).getString("updatedBy.username").string()); - } - - @Test - void getEventByIdNotFound() { - assertEquals( - "Event with id Hq3Kc6HK4OZ could not be found.", - GET("/tracker/events/Hq3Kc6HK4OZ").error(HttpStatus.NOT_FOUND).getMessage()); } @Test @@ -350,72 +113,52 @@ void getEventsFailsIfGivenAttributeCcAndAttributeCategoryCombo() { .getMessage()); } - private TrackedEntityType trackedEntityTypeAccessible() { - TrackedEntityType type = trackedEntityType('A'); - type.getSharing().addUserAccess(userAccess()); - manager.save(type, false); - return type; - } - - private TrackedEntityType trackedEntityTypeNotAccessible() { - TrackedEntityType type = trackedEntityType('B'); - manager.save(type, false); - return type; - } - - private TrackedEntityType trackedEntityType(char uniqueChar) { - TrackedEntityType type = createTrackedEntityType(uniqueChar); - type.getSharing().setOwner(owner); - type.getSharing().setPublicAccess(AccessStringHelper.DEFAULT); - return type; - } + @Test + void shouldMatchZipContent_whenEventJsonZipEndpointIsInvoked() + throws ForbiddenException, BadRequestException { + when(eventService.getEvents(any())).thenReturn(List.of()); - private TrackedEntity trackedEntity() { - TrackedEntity te = trackedEntity(orgUnit); - manager.save(te, false); - return te; + injectSecurityContext(user); + HttpResponse res = GET("/tracker/events.json.zip?attachment=file.json.zip"); + assertEquals(HttpStatus.OK, res.status()); + assertEquals("application/json+zip", res.header("Content-Type")); + assertEquals("attachment; filename=file.json.zip", res.header("Content-Disposition")); } - private TrackedEntity trackedEntityNotInSearchScope() { - TrackedEntity te = trackedEntity(anotherOrgUnit); - manager.save(te, false); - return te; - } + @Test + void shouldMatchZipContent_whenEventJsonZipEndpointIsInvokedWithNoAttachment() + throws ForbiddenException, BadRequestException { + when(eventService.getEvents(any())).thenReturn(List.of()); - private TrackedEntity trackedEntity(TrackedEntityType trackedEntityType) { - TrackedEntity te = trackedEntity(orgUnit, trackedEntityType); - manager.save(te, false); - return te; + injectSecurityContext(user); + HttpResponse res = GET("/tracker/events.json.zip"); + assertEquals(HttpStatus.OK, res.status()); + assertEquals("application/json+zip", res.header("Content-Type")); + assertEquals("attachment; filename=events.json.zip", res.header("Content-Disposition")); } - private TrackedEntity trackedEntity(OrganisationUnit orgUnit) { - return trackedEntity(orgUnit, trackedEntityType); - } + @Test + void shouldMatchGZipContent_whenEventJsonGZipEndpointIsInvoked() + throws ForbiddenException, BadRequestException { + when(eventService.getEvents(any())).thenReturn(List.of()); - private TrackedEntity trackedEntity( - OrganisationUnit orgUnit, TrackedEntityType trackedEntityType) { - TrackedEntity te = createTrackedEntity(orgUnit); - te.setTrackedEntityType(trackedEntityType); - te.getSharing().setPublicAccess(AccessStringHelper.DEFAULT); - te.getSharing().setOwner(owner); - return te; + injectSecurityContext(user); + HttpResponse res = GET("/tracker/events.json.gz?attachment=file.json.gzip"); + assertEquals(HttpStatus.OK, res.status()); + assertEquals("application/json+gzip", res.header("Content-Type")); + assertEquals("attachment; filename=file.json.gzip", res.header("Content-Disposition")); } - private Enrollment enrollment(TrackedEntity te) { - Enrollment enrollment = new Enrollment(program, te, te.getOrganisationUnit()); - enrollment.setAutoFields(); - enrollment.setEnrollmentDate(new Date()); - enrollment.setOccurredDate(new Date()); - enrollment.setStatus(ProgramStatus.COMPLETED); - manager.save(enrollment); - return enrollment; - } + @Test + void shouldMatchGZipContent_whenEventJsonGZipEndpointIsInvokedWithNoAttachment() + throws ForbiddenException, BadRequestException { + when(eventService.getEvents(any())).thenReturn(List.of()); - private Event event(Enrollment enrollment) { - Event event = new Event(enrollment, programStage, enrollment.getOrganisationUnit()); - event.setAutoFields(); - manager.save(event); - return event; + injectSecurityContext(user); + HttpResponse res = GET("/tracker/events.json.gz"); + assertEquals(HttpStatus.OK, res.status()); + assertEquals("application/json+gzip", res.header("Content-Type")); + assertEquals("attachment; filename=events.json.gzip", res.header("Content-Disposition")); } private UserAccess userAccess() { @@ -424,119 +167,4 @@ private UserAccess userAccess() { a.setAccess(AccessStringHelper.FULL); return a; } - - private RelationshipType relationshipTypeAccessible( - RelationshipEntity from, RelationshipEntity to) { - RelationshipType type = relationshipType(from, to); - type.getSharing().addUserAccess(userAccess()); - manager.save(type, false); - return type; - } - - private RelationshipType relationshipTypeNotAccessible() { - return relationshipType( - RelationshipEntity.PROGRAM_STAGE_INSTANCE, RelationshipEntity.TRACKED_ENTITY_INSTANCE); - } - - private RelationshipType relationshipType(RelationshipEntity from, RelationshipEntity to) { - RelationshipType type = createRelationshipType('A'); - type.getFromConstraint().setRelationshipEntity(from); - type.getToConstraint().setRelationshipEntity(to); - type.getSharing().setOwner(owner); - type.getSharing().setPublicAccess(AccessStringHelper.DEFAULT); - manager.save(type, false); - return type; - } - - private Relationship relationship(Event from, TrackedEntity to) { - return relationship( - relationshipTypeAccessible( - RelationshipEntity.PROGRAM_STAGE_INSTANCE, RelationshipEntity.TRACKED_ENTITY_INSTANCE), - from, - to); - } - - private Relationship relationship(RelationshipType type, Event from, TrackedEntity to) { - Relationship r = new Relationship(); - - RelationshipItem fromItem = new RelationshipItem(); - fromItem.setEvent(from); - from.getRelationshipItems().add(fromItem); - fromItem.setRelationship(r); - r.setFrom(fromItem); - - RelationshipItem toItem = new RelationshipItem(); - toItem.setTrackedEntity(to); - to.getRelationshipItems().add(toItem); - r.setTo(toItem); - toItem.setRelationship(r); - - r.setRelationshipType(type); - r.setKey(type.getUid()); - r.setInvertedKey(type.getUid()); - r.setAutoFields(); - r.getSharing().setOwner(owner); - manager.save(r, false); - return r; - } - - private void relationship(TrackedEntity from, Event to) { - Relationship r = new Relationship(); - - RelationshipItem fromItem = new RelationshipItem(); - fromItem.setTrackedEntity(from); - from.getRelationshipItems().add(fromItem); - r.setFrom(fromItem); - fromItem.setRelationship(r); - - RelationshipItem toItem = new RelationshipItem(); - toItem.setEvent(to); - to.getRelationshipItems().add(toItem); - r.setTo(toItem); - toItem.setRelationship(r); - - RelationshipType type = - relationshipTypeAccessible( - RelationshipEntity.PROGRAM_STAGE_INSTANCE, RelationshipEntity.TRACKED_ENTITY_INSTANCE); - r.setRelationshipType(type); - r.setKey(type.getUid()); - r.setInvertedKey(type.getUid()); - - r.setAutoFields(); - r.getSharing().setOwner(owner); - manager.save(r, false); - } - - private Note note(String uid, String value, String storedBy) { - Note note = new Note(value, storedBy); - note.setUid(uid); - manager.save(note, false); - return note; - } - - private void assertDefaultResponse(JsonObject json, Event event) { - // note that some fields are not included in the response because they - // are not part of the setup - // i.e. attributeOptionCombo, ... - assertTrue(json.isObject()); - assertFalse(json.isEmpty()); - assertEquals(event.getUid(), json.getString("event").string(), "event UID"); - assertEquals("ACTIVE", json.getString("status").string()); - assertEquals(program.getUid(), json.getString("program").string()); - assertEquals(programStage.getUid(), json.getString("programStage").string()); - assertEquals(event.getEnrollment().getUid(), json.getString("enrollment").string()); - assertEquals(orgUnit.getUid(), json.getString("orgUnit").string()); - assertFalse(json.getBoolean("followUp").booleanValue()); - assertFalse(json.getBoolean("followup").booleanValue()); - assertFalse(json.getBoolean("deleted").booleanValue()); - assertHasMember(json, "createdAt"); - assertHasMember(json, "createdAtClient"); - assertHasMember(json, "updatedAt"); - assertHasMember(json, "updatedAtClient"); - assertHasMember(json, "dataValues"); - assertHasMember(json, "notes"); - assertHasNoMember(json, "attributeOptionCombo"); - assertHasNoMember(json, "attributeCategoryOptions"); - assertHasNoMember(json, "relationships"); - } } diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerUnitTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerUnitTest.java index 4812d526be1f..0c09ff2792cf 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerUnitTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerUnitTest.java @@ -61,6 +61,8 @@ class EventsExportControllerUnitTest { @Mock private EventFieldsParamMapper eventsMapper; + @Mock private CompressedEventService compressedEventService; + @Test void shouldFailInstantiatingControllerIfAnyOrderableFieldIsUnsupported() { // pretend the service does not support 2 of the orderable fields the web advocates @@ -84,7 +86,8 @@ void shouldFailInstantiatingControllerIfAnyOrderableFieldIsUnsupported() { eventParamsMapper, csvEventService, fieldFilterService, - eventsMapper)); + eventsMapper, + compressedEventService)); assertAll( () -> assertStartsWith("event controller supports ordering by", exception.getMessage()), diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CompressedEventService.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CompressedEventService.java new file mode 100644 index 000000000000..04ee6f264d34 --- /dev/null +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CompressedEventService.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.tracker.export.event; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; +import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import lombok.RequiredArgsConstructor; +import org.hisp.dhis.webapi.controller.tracker.view.Event; +import org.springframework.stereotype.Service; + +@Service("org.hisp.dhis.webapi.controller.tracker.export.event.CompressedEventService") +@RequiredArgsConstructor +public class CompressedEventService { + + private final ObjectMapper objectMapper; + + public void writeZip(OutputStream requestOutputStream, List events, String attachment) + throws IOException { + ZipOutputStream outputStream = new ZipOutputStream(requestOutputStream); + outputStream.putNextEntry(new ZipEntry(attachment)); + + objectMapper.writer().writeValue(outputStream, events); + outputStream.close(); + } + + public void writeGzip(OutputStream requestOutputStream, List events) throws IOException { + GZIPOutputStream outputStream = new GZIPOutputStream(requestOutputStream); + + objectMapper.writer().writeValue(outputStream, events); + outputStream.close(); + } +} diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventRequestParams.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventRequestParams.java index 5f3d53f7e963..dbbedd27c1b0 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventRequestParams.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventRequestParams.java @@ -142,6 +142,8 @@ public class EventRequestParams implements PageRequestParams { private EventStatus status; + private String attachment; + /** * @deprecated use {@link #attributeCategoryCombo} */ diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportController.java index 019730bd27fc..4f0de95a8a7b 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportController.java @@ -34,13 +34,15 @@ import static org.hisp.dhis.webapi.controller.tracker.export.event.EventRequestParams.DEFAULT_FIELDS_PARAM; import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_CSV; import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_CSV_GZIP; +import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_JSON_GZIP; +import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_JSON_ZIP; import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_TEXT_CSV; -import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import java.io.OutputStream; import java.util.List; +import java.util.Objects; import java.util.zip.GZIPOutputStream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -95,24 +97,30 @@ class EventsExportController { private final EventFieldsParamMapper eventsMapper; + private final CompressedEventService compressedEventService; + + private static final String HEADER_CONTENT_TRANSFER_ENCODING = "binary"; + public EventsExportController( EventService eventService, EventRequestParamsMapper eventParamsMapper, CsvService csvEventService, FieldFilterService fieldFilterService, - EventFieldsParamMapper eventsMapper) { + EventFieldsParamMapper eventsMapper, + CompressedEventService compressedEventService) { this.eventService = eventService; this.eventParamsMapper = eventParamsMapper; this.csvEventService = csvEventService; this.fieldFilterService = fieldFilterService; this.eventsMapper = eventsMapper; + this.compressedEventService = compressedEventService; assertUserOrderableFieldsAreSupported( "event", EventMapper.ORDERABLE_FIELDS, eventService.getOrderableFields()); } @OpenApi.Response(status = Status.OK, value = OpenApiExport.ListResponse.class) - @GetMapping(produces = APPLICATION_JSON_VALUE) + @GetMapping(produces = "application/json") PagingWrapper getEvents(EventRequestParams eventRequestParams) throws BadRequestException, ForbiddenException { validatePaginationParameters(eventRequestParams); @@ -157,6 +165,56 @@ PagingWrapper getEvents(EventRequestParams eventRequestParams) return pagingWrapper.withInstances(objectNodes); } + @GetMapping(produces = CONTENT_TYPE_JSON_GZIP) + void getEventsAsGzip(EventRequestParams eventRequestParams, HttpServletResponse response) + throws BadRequestException, IOException, ForbiddenException { + validatePaginationParameters(eventRequestParams); + + EventOperationParams eventOperationParams = eventParamsMapper.map(eventRequestParams); + + List events = eventService.getEvents(eventOperationParams); + + String attachment = getAttachmentOrDefault(eventRequestParams.getAttachment(), "gzip"); + + response.addHeader( + ContextUtils.HEADER_CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); + response.addHeader( + ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, HEADER_CONTENT_TRANSFER_ENCODING); + response.setContentType(CONTENT_TYPE_JSON_GZIP); + + compressedEventService.writeGzip( + response.getOutputStream(), EVENTS_MAPPER.fromCollection(events)); + } + + @GetMapping(produces = CONTENT_TYPE_JSON_ZIP) + void getEventsAsZip(EventRequestParams eventRequestParams, HttpServletResponse response) + throws BadRequestException, ForbiddenException, IOException { + validatePaginationParameters(eventRequestParams); + + EventOperationParams eventOperationParams = eventParamsMapper.map(eventRequestParams); + + List events = eventService.getEvents(eventOperationParams); + + String attachment = getAttachmentOrDefault(eventRequestParams.getAttachment(), "zip"); + + response.addHeader( + ContextUtils.HEADER_CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); + response.addHeader( + ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, HEADER_CONTENT_TRANSFER_ENCODING); + response.setContentType(CONTENT_TYPE_JSON_ZIP); + + compressedEventService.writeZip( + response.getOutputStream(), EVENTS_MAPPER.fromCollection(events), attachment); + } + + private String getAttachmentOrDefault(String filename, String compression) { + return Objects.toString(filename, "events.json." + compression); + } + + public String getContentDispositionHeaderValue(String filename) { + return "attachment; filename=" + filename; + } + @GetMapping(produces = {CONTENT_TYPE_CSV, CONTENT_TYPE_CSV_GZIP, CONTENT_TYPE_TEXT_CSV}) void getEventsAsCsv( EventRequestParams eventRequestParams, @@ -173,7 +231,8 @@ void getEventsAsCsv( response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"events.csv\""); if (ContextUtils.isAcceptCsvGzip(request)) { - response.addHeader(ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, "binary"); + response.addHeader( + ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, HEADER_CONTENT_TRANSFER_ENCODING); outputStream = new GZIPOutputStream(outputStream); response.setContentType(CONTENT_TYPE_CSV_GZIP); response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"events.csv.gz\""); diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/ContextUtils.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/ContextUtils.java index a6e1f6d3e110..4101a629215b 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/ContextUtils.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/ContextUtils.java @@ -59,7 +59,8 @@ public class ContextUtils { public static final String CONTENT_TYPE_PDF = "application/pdf"; - public static final String CONTENT_TYPE_ZIP = "application/zip"; + public static final String CONTENT_TYPE_JSON_ZIP = "application/json+zip"; + public static final String CONTENT_TYPE_JSON_GZIP = "application/json+gzip"; public static final String CONTENT_TYPE_GZIP = "application/gzip"; diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CompressedEventServiceTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CompressedEventServiceTest.java new file mode 100644 index 000000000000..9927ad7b17e1 --- /dev/null +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CompressedEventServiceTest.java @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.tracker.export.event; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.List; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import org.hisp.dhis.common.CodeGenerator; +import org.hisp.dhis.webapi.controller.tracker.view.Event; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class CompressedEventServiceTest { + + private CompressedEventService compressedEventService; + private static final Event FIRST_EVENT = new Event(); + private static final Event SECOND_EVENT = new Event(); + @InjectMocks private ObjectMapper objectMapper; + + @BeforeEach + void setUp() { + compressedEventService = new CompressedEventService(objectMapper); + FIRST_EVENT.setEvent(CodeGenerator.generateUid()); + SECOND_EVENT.setEvent(CodeGenerator.generateUid()); + } + + @Test + void shouldUnzipFileAndMatchEvents_whenCreateZipFileFromEventList() throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + List eventToZip = getEvents(); + + compressedEventService.writeZip(outputStream, eventToZip, "file.json.zip"); + + ZipInputStream zipInputStream = + new ZipInputStream(new ByteArrayInputStream(outputStream.toByteArray())); + var buff = new byte[1024]; + List eventsFromZip; + + ZipEntry zipEntry = zipInputStream.getNextEntry(); + + assertNotNull(zipEntry, "Events Zip file has no entry"); + assertEquals("file.json.zip", zipEntry.getName(), "Events Zip file has a wrong name"); + + var byteArrayOutputStream = new ByteArrayOutputStream(); + int l; + while ((l = zipInputStream.read(buff)) > 0) { + byteArrayOutputStream.write(buff, 0, l); + } + eventsFromZip = + objectMapper.readValue(byteArrayOutputStream.toString(), new TypeReference<>() {}); + + assertNull(zipInputStream.getNextEntry()); // assert only one file is created + assertEquals(eventToZip.size(), eventsFromZip.size()); + assertEquals( + FIRST_EVENT, + eventsFromZip.stream() + .filter(e -> e.getEvent().equals(FIRST_EVENT.getEvent())) + .findAny() + .orElse(null), + "The event does not match or does not exist in the Zip File."); + assertEquals( + SECOND_EVENT, + eventsFromZip.stream() + .filter(e -> e.getEvent().equals(SECOND_EVENT.getEvent())) + .findAny() + .orElse(null), + "The event does not match or does not exist in the Zip File."); + } + + @Test + void shouldGUnzipFileAndMatchEvents_whenCreateGZipFileFromEventList() throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + List eventToGZip = getEvents(); + + compressedEventService.writeGzip(outputStream, eventToGZip); + + GZIPInputStream gzipInputStream = + new GZIPInputStream(new ByteArrayInputStream(outputStream.toByteArray())); + var buff = new byte[1024]; + List eventsFromGZip; + + var byteArrayOutputStream = new ByteArrayOutputStream(); + int l; + while ((l = gzipInputStream.read(buff)) > 0) { + byteArrayOutputStream.write(buff, 0, l); + } + eventsFromGZip = + objectMapper.readValue(byteArrayOutputStream.toString(), new TypeReference<>() {}); + + assertEquals(eventToGZip.size(), eventsFromGZip.size()); + assertEquals( + FIRST_EVENT, + eventToGZip.stream() + .filter(e -> e.getEvent().equals(FIRST_EVENT.getEvent())) + .findAny() + .orElse(null), + "The event does not match or does not exist in the GZip File."); + assertEquals( + SECOND_EVENT, + eventToGZip.stream() + .filter(e -> e.getEvent().equals(SECOND_EVENT.getEvent())) + .findAny() + .orElse(null), + "The event does not match or does not exist in the GZip File."); + } + + List getEvents() { + return List.of(FIRST_EVENT, SECOND_EVENT); + } +} From f2f0494c7948eb59b8aa70bcde61a2cfc7567e33 Mon Sep 17 00:00:00 2001 From: luca Date: Tue, 19 Dec 2023 15:14:01 +0100 Subject: [PATCH 2/9] fix: clean code --- .../tracker/export/event/CompressedEventServiceTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CompressedEventServiceTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CompressedEventServiceTest.java index 9927ad7b17e1..8007ae062f84 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CompressedEventServiceTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CompressedEventServiceTest.java @@ -74,7 +74,6 @@ void shouldUnzipFileAndMatchEvents_whenCreateZipFileFromEventList() throws IOExc ZipInputStream zipInputStream = new ZipInputStream(new ByteArrayInputStream(outputStream.toByteArray())); var buff = new byte[1024]; - List eventsFromZip; ZipEntry zipEntry = zipInputStream.getNextEntry(); @@ -86,7 +85,8 @@ void shouldUnzipFileAndMatchEvents_whenCreateZipFileFromEventList() throws IOExc while ((l = zipInputStream.read(buff)) > 0) { byteArrayOutputStream.write(buff, 0, l); } - eventsFromZip = + + List eventsFromZip = objectMapper.readValue(byteArrayOutputStream.toString(), new TypeReference<>() {}); assertNull(zipInputStream.getNextEntry()); // assert only one file is created @@ -118,14 +118,14 @@ void shouldGUnzipFileAndMatchEvents_whenCreateGZipFileFromEventList() throws IOE GZIPInputStream gzipInputStream = new GZIPInputStream(new ByteArrayInputStream(outputStream.toByteArray())); var buff = new byte[1024]; - List eventsFromGZip; var byteArrayOutputStream = new ByteArrayOutputStream(); int l; while ((l = gzipInputStream.read(buff)) > 0) { byteArrayOutputStream.write(buff, 0, l); } - eventsFromGZip = + + List eventsFromGZip = objectMapper.readValue(byteArrayOutputStream.toString(), new TypeReference<>() {}); assertEquals(eventToGZip.size(), eventsFromGZip.size()); From 3322ad9fb99f8a53ee4f60e1a29ac8bba4e7bf7d Mon Sep 17 00:00:00 2001 From: luca Date: Tue, 19 Dec 2023 15:36:25 +0100 Subject: [PATCH 3/9] fix: sonar --- .../event/EventsExportControllerTest.java | 69 +++++++++---------- 1 file changed, 31 insertions(+), 38 deletions(-) diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java index eea603c55573..616262c80035 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java @@ -29,11 +29,13 @@ import static org.hisp.dhis.utils.Assertions.assertStartsWith; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import java.util.List; import java.util.Set; +import java.util.stream.Stream; import org.hisp.dhis.common.CodeGenerator; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.dataelement.DataElement; @@ -50,6 +52,9 @@ import org.hisp.dhis.webapi.DhisControllerConvenienceTest; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; @@ -113,52 +118,40 @@ void getEventsFailsIfGivenAttributeCcAndAttributeCategoryCombo() { .getMessage()); } - @Test - void shouldMatchZipContent_whenEventJsonZipEndpointIsInvoked() - throws ForbiddenException, BadRequestException { - when(eventService.getEvents(any())).thenReturn(List.of()); - - injectSecurityContext(user); - HttpResponse res = GET("/tracker/events.json.zip?attachment=file.json.zip"); - assertEquals(HttpStatus.OK, res.status()); - assertEquals("application/json+zip", res.header("Content-Type")); - assertEquals("attachment; filename=file.json.zip", res.header("Content-Disposition")); + static Stream + shouldMatchContentTypeAndAttachment_whenEndpointForCompressedEventJsonIsInvoked() { + return Stream.of( + arguments( + "/tracker/events.json.zip?attachment=file.json.zip", + "application/json+zip", + "attachment; filename=file.json.zip"), + arguments( + "/tracker/events.json.zip", + "application/json+zip", + "attachment; filename=events.json.zip"), + arguments( + "/tracker/events.json.gz?attachment=file.json.gzip", + "application/json+gzip", + "attachment; filename=file.json.gzip"), + arguments( + "/tracker/events.json.gz", + "application/json+gzip", + "attachment; filename=events.json.gzip")); } - @Test - void shouldMatchZipContent_whenEventJsonZipEndpointIsInvokedWithNoAttachment() + @ParameterizedTest + @MethodSource + void shouldMatchContentTypeAndAttachment_whenEndpointForCompressedEventJsonIsInvoked( + String url, String expectedContentType, String expectedAttachment) throws ForbiddenException, BadRequestException { - when(eventService.getEvents(any())).thenReturn(List.of()); - - injectSecurityContext(user); - HttpResponse res = GET("/tracker/events.json.zip"); - assertEquals(HttpStatus.OK, res.status()); - assertEquals("application/json+zip", res.header("Content-Type")); - assertEquals("attachment; filename=events.json.zip", res.header("Content-Disposition")); - } - @Test - void shouldMatchGZipContent_whenEventJsonGZipEndpointIsInvoked() - throws ForbiddenException, BadRequestException { when(eventService.getEvents(any())).thenReturn(List.of()); - injectSecurityContext(user); - HttpResponse res = GET("/tracker/events.json.gz?attachment=file.json.gzip"); - assertEquals(HttpStatus.OK, res.status()); - assertEquals("application/json+gzip", res.header("Content-Type")); - assertEquals("attachment; filename=file.json.gzip", res.header("Content-Disposition")); - } - @Test - void shouldMatchGZipContent_whenEventJsonGZipEndpointIsInvokedWithNoAttachment() - throws ForbiddenException, BadRequestException { - when(eventService.getEvents(any())).thenReturn(List.of()); - - injectSecurityContext(user); - HttpResponse res = GET("/tracker/events.json.gz"); + HttpResponse res = GET(url); assertEquals(HttpStatus.OK, res.status()); - assertEquals("application/json+gzip", res.header("Content-Type")); - assertEquals("attachment; filename=events.json.gzip", res.header("Content-Disposition")); + assertEquals(expectedContentType, res.header("Content-Type")); + assertEquals(expectedAttachment, res.header("Content-Disposition")); } private UserAccess userAccess() { From 251c45351e52d4f39e81465230f2916658d274a4 Mon Sep 17 00:00:00 2001 From: luca Date: Thu, 21 Dec 2023 11:50:04 +0100 Subject: [PATCH 4/9] fix: harmonize compressione support events/trackedentities --- .../event/EventsExportControllerTest.java | 50 ++++++- .../event/EventsExportControllerUnitTest.java | 5 +- .../TrackedEntitiesExportControllerTest.java | 8 +- ...EventService.java => CompressionUtil.java} | 43 ++++-- .../controller/tracker/export/CsvService.java | 6 + .../tracker/export/event/CsvEventService.java | 32 ++++- .../export/event/EventsExportController.java | 123 ++++++++++++------ .../CsvTrackedEntityService.java | 33 ++++- .../TrackedEntitiesExportController.java | 111 +++++++++++----- .../TrackedEntityRequestParams.java | 2 + .../hisp/dhis/webapi/utils/ContextUtils.java | 2 + ...viceTest.java => CompressionUtilTest.java} | 16 +-- 12 files changed, 311 insertions(+), 120 deletions(-) rename dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/{event/CompressedEventService.java => CompressionUtil.java} (64%) rename dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/{event/CompressedEventServiceTest.java => CompressionUtilTest.java} (90%) diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java index 616262c80035..0433a06a4411 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java @@ -50,6 +50,7 @@ import org.hisp.dhis.user.sharing.UserAccess; import org.hisp.dhis.web.HttpStatus; import org.hisp.dhis.webapi.DhisControllerConvenienceTest; +import org.hisp.dhis.webapi.utils.ContextUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -124,25 +125,59 @@ void getEventsFailsIfGivenAttributeCcAndAttributeCategoryCombo() { arguments( "/tracker/events.json.zip?attachment=file.json.zip", "application/json+zip", - "attachment; filename=file.json.zip"), + "attachment; filename=file.json.zip", + "binary"), arguments( "/tracker/events.json.zip", "application/json+zip", - "attachment; filename=events.json.zip"), + "attachment; filename=events.json.zip", + "binary"), arguments( - "/tracker/events.json.gz?attachment=file.json.gzip", + "/tracker/events.json.gz?attachment=file.json.gz", "application/json+gzip", - "attachment; filename=file.json.gzip"), + "attachment; filename=file.json.gz", + "binary"), arguments( "/tracker/events.json.gz", "application/json+gzip", - "attachment; filename=events.json.gzip")); + "attachment; filename=events.json.gz", + "binary"), + arguments( + "/tracker/events.csv", + "application/csv; charset=UTF-8", + "attachment; filename=events.csv", + null), + arguments( + "/tracker/events.csv?attachment=file.csv", + "application/csv; charset=UTF-8", + "attachment; filename=file.csv", + null), + arguments( + "/tracker/events.csv.gz", + "application/csv+gzip", + "attachment; filename=events.csv.gz", + "binary"), + arguments( + "/tracker/events.csv.gz?attachment=file.csv.gz", + "application/csv+gzip", + "attachment; filename=file.csv.gz", + "binary"), + arguments( + "/tracker/events.csv.zip", + "application/csv+zip", + "attachment; filename=events.csv.zip", + "binary"), + arguments( + "/tracker/events.csv.zip?attachment=file.csv.zip", + "application/csv+zip", + "attachment; filename=file.csv.zip", + "binary")); } @ParameterizedTest @MethodSource void shouldMatchContentTypeAndAttachment_whenEndpointForCompressedEventJsonIsInvoked( - String url, String expectedContentType, String expectedAttachment) + String url, String expectedContentType, String expectedAttachment, String encoding) throws ForbiddenException, BadRequestException { when(eventService.getEvents(any())).thenReturn(List.of()); @@ -151,7 +186,8 @@ void shouldMatchContentTypeAndAttachment_whenEndpointForCompressedEventJsonIsInv HttpResponse res = GET(url); assertEquals(HttpStatus.OK, res.status()); assertEquals(expectedContentType, res.header("Content-Type")); - assertEquals(expectedAttachment, res.header("Content-Disposition")); + assertEquals(expectedAttachment, res.header(ContextUtils.HEADER_CONTENT_DISPOSITION)); + assertEquals(encoding, res.header(ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING)); } private UserAccess userAccess() { diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerUnitTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerUnitTest.java index 0c09ff2792cf..121d76f064bc 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerUnitTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerUnitTest.java @@ -33,6 +33,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.when; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -61,7 +62,7 @@ class EventsExportControllerUnitTest { @Mock private EventFieldsParamMapper eventsMapper; - @Mock private CompressedEventService compressedEventService; + @Mock private ObjectMapper objectMapper; @Test void shouldFailInstantiatingControllerIfAnyOrderableFieldIsUnsupported() { @@ -87,7 +88,7 @@ void shouldFailInstantiatingControllerIfAnyOrderableFieldIsUnsupported() { csvEventService, fieldFilterService, eventsMapper, - compressedEventService)); + objectMapper)); assertAll( () -> assertStartsWith("event controller supports ordering by", exception.getMessage()), diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportControllerTest.java index 94576a1d418d..4988e2f01492 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportControllerTest.java @@ -424,9 +424,7 @@ void getTrackedEntityReturnsCsvFormat() { () -> assertTrue(response.header("content-type").contains(ContextUtils.CONTENT_TYPE_CSV)), () -> assertTrue( - response - .header("content-disposition") - .contains("filename=\"trackedEntities.csv\"")), + response.header("content-disposition").contains("filename=trackedEntities.csv")), () -> assertTrue(response.content().toString().contains("trackedEntity,trackedEntityType"))); } @@ -450,7 +448,7 @@ void getTrackedEntityReturnsCsvZipFormat() { assertTrue( response .header("content-disposition") - .contains("filename=\"trackedEntities.csv.zip\""))); + .contains("filename=trackedEntities.csv.zip"))); } @Test @@ -473,7 +471,7 @@ void getTrackedEntityReturnsCsvGZipFormat() { assertTrue( response .header("content-disposition") - .contains("filename=\"trackedEntities.csv.gz\""))); + .contains("filename=trackedEntities.csv.gz"))); } @Test diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CompressedEventService.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtil.java similarity index 64% rename from dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CompressedEventService.java rename to dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtil.java index 04ee6f264d34..f6d66d83e6a2 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CompressedEventService.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtil.java @@ -25,38 +25,53 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.hisp.dhis.webapi.controller.tracker.export.event; +package org.hisp.dhis.webapi.controller.tracker.export; -import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import java.io.IOException; import java.io.OutputStream; -import java.util.List; import java.util.zip.GZIPOutputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -import lombok.RequiredArgsConstructor; -import org.hisp.dhis.webapi.controller.tracker.view.Event; -import org.springframework.stereotype.Service; -@Service("org.hisp.dhis.webapi.controller.tracker.export.event.CompressedEventService") -@RequiredArgsConstructor -public class CompressedEventService { +public class CompressionUtil { - private final ObjectMapper objectMapper; + private CompressionUtil() { + throw new IllegalStateException( + "Utility class to compress exported objects in Zip o GZip format"); + } - public void writeZip(OutputStream requestOutputStream, List events, String attachment) + /** + * @param requestOutputStream Output stream from request + * @param toCompress Objects to compress + * @param objectWriter Object writer from a mapper + * @param attachment Attachment file name + * @param + * @throws IOException + */ + public static void writeZip( + OutputStream requestOutputStream, T toCompress, ObjectWriter objectWriter, String attachment) throws IOException { ZipOutputStream outputStream = new ZipOutputStream(requestOutputStream); outputStream.putNextEntry(new ZipEntry(attachment)); - objectMapper.writer().writeValue(outputStream, events); + objectWriter.writeValue(outputStream, toCompress); outputStream.close(); } - public void writeGzip(OutputStream requestOutputStream, List events) throws IOException { + /** + * @param requestOutputStream Output stream from request + * @param toCompress Objects to compress + * @param objectWriter Object writer from a mapper + * @param + * @throws IOException + */ + public static void writeGzip( + OutputStream requestOutputStream, T toCompress, ObjectWriter objectWriter) + throws IOException { GZIPOutputStream outputStream = new GZIPOutputStream(requestOutputStream); - objectMapper.writer().writeValue(outputStream, events); + objectWriter.writeValue(outputStream, toCompress); outputStream.close(); } } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/CsvService.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/CsvService.java index 540b34c6d53d..e110fcc84b64 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/CsvService.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/CsvService.java @@ -35,6 +35,12 @@ public interface CsvService { void write(OutputStream outputStream, List events, boolean withHeader) throws IOException; + void writeZip(OutputStream outputStream, List toCompress, boolean withHeader, String file) + throws IOException; + + void writeGzip(OutputStream outputStream, List toCompress, boolean withHeader) + throws IOException; + List read(InputStream inputStream, boolean skipFirst) throws IOException, org.locationtech.jts.io.ParseException; } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventService.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventService.java index d34954bf38f8..2ef352df00e1 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventService.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventService.java @@ -44,6 +44,7 @@ import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.event.EventStatus; import org.hisp.dhis.util.DateUtils; +import org.hisp.dhis.webapi.controller.tracker.export.CompressionUtil; import org.hisp.dhis.webapi.controller.tracker.export.CsvService; import org.hisp.dhis.webapi.controller.tracker.view.DataValue; import org.hisp.dhis.webapi.controller.tracker.view.Event; @@ -65,13 +66,7 @@ class CsvEventService implements CsvService { @Override public void write(OutputStream outputStream, List events, boolean withHeader) throws IOException { - final CsvSchema csvSchema = - CSV_MAPPER - .schemaFor(CsvEventDataValue.class) - .withLineSeparator("\n") - .withUseHeader(withHeader); - - ObjectWriter writer = CSV_MAPPER.writer(csvSchema.withUseHeader(withHeader)); + ObjectWriter writer = getObjectWriter(withHeader); List dataValues = new ArrayList<>(); @@ -91,6 +86,29 @@ public void write(OutputStream outputStream, List events, boolean withHea writer.writeValue(outputStream, dataValues); } + @Override + public void writeZip( + OutputStream outputStream, List toCompress, boolean withHeader, String file) + throws IOException { + CompressionUtil.writeZip(outputStream, toCompress, getObjectWriter(withHeader), file); + } + + @Override + public void writeGzip(OutputStream outputStream, List toCompress, boolean withHeader) + throws IOException { + CompressionUtil.writeGzip(outputStream, toCompress, getObjectWriter(withHeader)); + } + + private ObjectWriter getObjectWriter(boolean withHeader) { + final CsvSchema csvSchema = + CSV_MAPPER + .schemaFor(CsvEventDataValue.class) + .withLineSeparator("\n") + .withUseHeader(withHeader); + + return CSV_MAPPER.writer(csvSchema.withUseHeader(withHeader)); + } + private static CsvEventDataValue map(Event event) { CsvEventDataValue result = new CsvEventDataValue(); result.setEvent(event.getEvent()); diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportController.java index 4f0de95a8a7b..f059a1307e41 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportController.java @@ -30,21 +30,23 @@ import static org.hisp.dhis.common.OpenApi.Response.Status; import static org.hisp.dhis.webapi.controller.tracker.ControllerSupport.RESOURCE_PATH; import static org.hisp.dhis.webapi.controller.tracker.ControllerSupport.assertUserOrderableFieldsAreSupported; +import static org.hisp.dhis.webapi.controller.tracker.export.CompressionUtil.writeGzip; +import static org.hisp.dhis.webapi.controller.tracker.export.CompressionUtil.writeZip; import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validatePaginationParameters; import static org.hisp.dhis.webapi.controller.tracker.export.event.EventRequestParams.DEFAULT_FIELDS_PARAM; import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_CSV; import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_CSV_GZIP; +import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_CSV_ZIP; import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_JSON_GZIP; import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_JSON_ZIP; import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_TEXT_CSV; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import java.io.OutputStream; import java.util.List; import java.util.Objects; -import java.util.zip.GZIPOutputStream; -import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.hisp.dhis.common.DhisApiVersion; import org.hisp.dhis.common.OpenApi; @@ -97,9 +99,7 @@ class EventsExportController { private final EventFieldsParamMapper eventsMapper; - private final CompressedEventService compressedEventService; - - private static final String HEADER_CONTENT_TRANSFER_ENCODING = "binary"; + private final ObjectMapper objectMapper; public EventsExportController( EventService eventService, @@ -107,13 +107,13 @@ public EventsExportController( CsvService csvEventService, FieldFilterService fieldFilterService, EventFieldsParamMapper eventsMapper, - CompressedEventService compressedEventService) { + ObjectMapper objectMapper) { this.eventService = eventService; this.eventParamsMapper = eventParamsMapper; this.csvEventService = csvEventService; this.fieldFilterService = fieldFilterService; this.eventsMapper = eventsMapper; - this.compressedEventService = compressedEventService; + this.objectMapper = objectMapper; assertUserOrderableFieldsAreSupported( "event", EventMapper.ORDERABLE_FIELDS, eventService.getOrderableFields()); @@ -166,7 +166,7 @@ PagingWrapper getEvents(EventRequestParams eventRequestParams) } @GetMapping(produces = CONTENT_TYPE_JSON_GZIP) - void getEventsAsGzip(EventRequestParams eventRequestParams, HttpServletResponse response) + void getEventsAsJsonGzip(EventRequestParams eventRequestParams, HttpServletResponse response) throws BadRequestException, IOException, ForbiddenException { validatePaginationParameters(eventRequestParams); @@ -174,20 +174,21 @@ void getEventsAsGzip(EventRequestParams eventRequestParams, HttpServletResponse List events = eventService.getEvents(eventOperationParams); - String attachment = getAttachmentOrDefault(eventRequestParams.getAttachment(), "gzip"); + String attachment = getAttachmentOrDefault(eventRequestParams.getAttachment(), "json", "gz"); response.addHeader( ContextUtils.HEADER_CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); response.addHeader( - ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, HEADER_CONTENT_TRANSFER_ENCODING); + ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, + ContextUtils.BINARY_HEADER_CONTENT_TRANSFER_ENCODING); response.setContentType(CONTENT_TYPE_JSON_GZIP); - compressedEventService.writeGzip( - response.getOutputStream(), EVENTS_MAPPER.fromCollection(events)); + writeGzip( + response.getOutputStream(), EVENTS_MAPPER.fromCollection(events), objectMapper.writer()); } @GetMapping(produces = CONTENT_TYPE_JSON_ZIP) - void getEventsAsZip(EventRequestParams eventRequestParams, HttpServletResponse response) + void getEventsAsJsonZip(EventRequestParams eventRequestParams, HttpServletResponse response) throws BadRequestException, ForbiddenException, IOException { validatePaginationParameters(eventRequestParams); @@ -195,52 +196,100 @@ void getEventsAsZip(EventRequestParams eventRequestParams, HttpServletResponse r List events = eventService.getEvents(eventOperationParams); - String attachment = getAttachmentOrDefault(eventRequestParams.getAttachment(), "zip"); + String attachment = getAttachmentOrDefault(eventRequestParams.getAttachment(), "json", "zip"); response.addHeader( ContextUtils.HEADER_CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); response.addHeader( - ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, HEADER_CONTENT_TRANSFER_ENCODING); + ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, + ContextUtils.BINARY_HEADER_CONTENT_TRANSFER_ENCODING); response.setContentType(CONTENT_TYPE_JSON_ZIP); - compressedEventService.writeZip( - response.getOutputStream(), EVENTS_MAPPER.fromCollection(events), attachment); - } - - private String getAttachmentOrDefault(String filename, String compression) { - return Objects.toString(filename, "events.json." + compression); - } - - public String getContentDispositionHeaderValue(String filename) { - return "attachment; filename=" + filename; + writeZip( + response.getOutputStream(), + EVENTS_MAPPER.fromCollection(events), + objectMapper.writer(), + attachment); } - @GetMapping(produces = {CONTENT_TYPE_CSV, CONTENT_TYPE_CSV_GZIP, CONTENT_TYPE_TEXT_CSV}) + @GetMapping(produces = {CONTENT_TYPE_CSV, CONTENT_TYPE_TEXT_CSV}) void getEventsAsCsv( EventRequestParams eventRequestParams, HttpServletResponse response, - @RequestParam(required = false, defaultValue = "false") boolean skipHeader, - HttpServletRequest request) + @RequestParam(required = false, defaultValue = "false") boolean skipHeader) throws IOException, BadRequestException, ForbiddenException { EventOperationParams eventOperationParams = eventParamsMapper.map(eventRequestParams); List events = eventService.getEvents(eventOperationParams); + String attachment = getAttachmentOrDefault(eventRequestParams.getAttachment(), "csv"); + OutputStream outputStream = response.getOutputStream(); response.setContentType(CONTENT_TYPE_CSV); - response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"events.csv\""); - - if (ContextUtils.isAcceptCsvGzip(request)) { - response.addHeader( - ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, HEADER_CONTENT_TRANSFER_ENCODING); - outputStream = new GZIPOutputStream(outputStream); - response.setContentType(CONTENT_TYPE_CSV_GZIP); - response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"events.csv.gz\""); - } + response.setHeader( + HttpHeaders.CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); csvEventService.write(outputStream, EVENTS_MAPPER.fromCollection(events), !skipHeader); } + @GetMapping(produces = {CONTENT_TYPE_CSV_GZIP}) + void getEventsAsCsvGZip( + EventRequestParams eventRequestParams, + HttpServletResponse response, + @RequestParam(required = false, defaultValue = "false") boolean skipHeader) + throws IOException, BadRequestException, ForbiddenException { + EventOperationParams eventOperationParams = eventParamsMapper.map(eventRequestParams); + + List events = eventService.getEvents(eventOperationParams); + + String attachment = getAttachmentOrDefault(eventRequestParams.getAttachment(), "csv", "gz"); + + response.addHeader( + ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, + ContextUtils.BINARY_HEADER_CONTENT_TRANSFER_ENCODING); + response.setContentType(CONTENT_TYPE_CSV_GZIP); + response.addHeader( + ContextUtils.HEADER_CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); + + csvEventService.writeGzip( + response.getOutputStream(), EVENTS_MAPPER.fromCollection(events), !skipHeader); + } + + @GetMapping(produces = {CONTENT_TYPE_CSV_ZIP}) + void getEventsAsCsvZip( + EventRequestParams eventRequestParams, + HttpServletResponse response, + @RequestParam(required = false, defaultValue = "false") boolean skipHeader) + throws IOException, BadRequestException, ForbiddenException { + EventOperationParams eventOperationParams = eventParamsMapper.map(eventRequestParams); + + List events = eventService.getEvents(eventOperationParams); + + String attachment = getAttachmentOrDefault(eventRequestParams.getAttachment(), "csv", "zip"); + + response.addHeader( + ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, + ContextUtils.BINARY_HEADER_CONTENT_TRANSFER_ENCODING); + response.setContentType(CONTENT_TYPE_CSV_ZIP); + response.addHeader( + ContextUtils.HEADER_CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); + + csvEventService.writeZip( + response.getOutputStream(), EVENTS_MAPPER.fromCollection(events), !skipHeader, attachment); + } + + private String getAttachmentOrDefault(String filename, String type, String compression) { + return Objects.toString(filename, String.join(".", EVENTS, type, compression)); + } + + private String getAttachmentOrDefault(String filename, String type) { + return Objects.toString(filename, String.join(".", EVENTS, type)); + } + + public String getContentDispositionHeaderValue(String filename) { + return "attachment; filename=" + filename; + } + @OpenApi.Response(OpenApi.EntityType.class) @GetMapping("/{uid}") ResponseEntity getEventByUid( diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/CsvTrackedEntityService.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/CsvTrackedEntityService.java index 6afbe335b088..b88b1a818d41 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/CsvTrackedEntityService.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/CsvTrackedEntityService.java @@ -38,6 +38,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.hisp.dhis.webapi.controller.tracker.export.CompressionUtil; import org.hisp.dhis.webapi.controller.tracker.export.CsvService; import org.hisp.dhis.webapi.controller.tracker.view.Attribute; import org.hisp.dhis.webapi.controller.tracker.view.TrackedEntity; @@ -52,13 +53,7 @@ class CsvTrackedEntityService implements CsvService { public void write( OutputStream outputStream, List trackedEntities, boolean withHeader) throws IOException { - final CsvSchema csvSchema = - CSV_MAPPER - .schemaFor(CsvTrackedEntity.class) - .withLineSeparator("\n") - .withUseHeader(withHeader); - - ObjectWriter writer = CSV_MAPPER.writer(csvSchema.withUseHeader(withHeader)); + ObjectWriter writer = getObjectWriter(withHeader); List attributes = new ArrayList<>(); @@ -99,6 +94,30 @@ public void write( writer.writeValue(outputStream, attributes); } + @Override + public void writeZip( + OutputStream outputStream, List toCompress, boolean withHeader, String file) + throws IOException { + CompressionUtil.writeZip(outputStream, toCompress, getObjectWriter(withHeader), file); + } + + @Override + public void writeGzip( + OutputStream outputStream, List toCompress, boolean withHeader) + throws IOException { + CompressionUtil.writeGzip(outputStream, toCompress, getObjectWriter(withHeader)); + } + + private ObjectWriter getObjectWriter(boolean withHeader) { + final CsvSchema csvSchema = + CSV_MAPPER + .schemaFor(CsvTrackedEntity.class) + .withLineSeparator("\n") + .withUseHeader(withHeader); + + return CSV_MAPPER.writer(csvSchema.withUseHeader(withHeader)); + } + private String checkForNull(Instant instant) { if (instant == null) { return null; diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportController.java index e2da9e97b539..d65e148b5bab 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportController.java @@ -42,9 +42,7 @@ import java.io.IOException; import java.io.OutputStream; import java.util.List; -import java.util.zip.GZIPOutputStream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; +import java.util.Objects; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.hisp.dhis.common.DhisApiVersion; @@ -177,50 +175,97 @@ PagingWrapper getTrackedEntities( return pagingWrapper.withInstances(objectNodes); } - @GetMapping( - produces = { - CONTENT_TYPE_CSV, - CONTENT_TYPE_CSV_GZIP, - CONTENT_TYPE_CSV_ZIP, - CONTENT_TYPE_TEXT_CSV - }) + @GetMapping(produces = {CONTENT_TYPE_CSV, CONTENT_TYPE_TEXT_CSV}) void getTrackedEntitiesAsCsv( TrackedEntityRequestParams trackedEntityRequestParams, HttpServletResponse response, - HttpServletRequest request, @CurrentUser User user, @RequestParam(required = false, defaultValue = "false") boolean skipHeader) throws IOException, BadRequestException, ForbiddenException, NotFoundException { TrackedEntityOperationParams operationParams = paramsMapper.map(trackedEntityRequestParams, user, CSV_FIELDS); - List trackedEntities = + String attachment = getAttachmentOrDefault(trackedEntityRequestParams.getAttachment(), "csv"); + + response.setContentType(CONTENT_TYPE_CSV); + response.setHeader( + HttpHeaders.CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); + + csvEventService.write( + response.getOutputStream(), TRACKED_ENTITY_MAPPER.fromCollection( - trackedEntityService.getTrackedEntities(operationParams)); + trackedEntityService.getTrackedEntities(operationParams)), + !skipHeader); + } + + @GetMapping(produces = {CONTENT_TYPE_CSV_ZIP}) + void getTrackedEntitiesAsCsvZip( + TrackedEntityRequestParams trackedEntityRequestParams, + HttpServletResponse response, + @CurrentUser User user, + @RequestParam(required = false, defaultValue = "false") boolean skipHeader) + throws IOException, BadRequestException, ForbiddenException, NotFoundException { + TrackedEntityOperationParams operationParams = + paramsMapper.map(trackedEntityRequestParams, user, CSV_FIELDS); OutputStream outputStream = response.getOutputStream(); - if (ContextUtils.isAcceptCsvGzip(request)) { - response.addHeader(ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, "binary"); - outputStream = new GZIPOutputStream(outputStream); - response.setContentType(CONTENT_TYPE_CSV_GZIP); - response.setHeader( - HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"trackedEntities.csv.gz\""); - } else if (ContextUtils.isAcceptCsvZip(request)) { - response.addHeader(ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, "binary"); - response.setContentType(CONTENT_TYPE_CSV_ZIP); - response.setHeader( - HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"trackedEntities.csv.zip\""); - ZipOutputStream zos = new ZipOutputStream(outputStream); - zos.putNextEntry(new ZipEntry("trackedEntities.csv")); - outputStream = zos; - } else { - response.setContentType(CONTENT_TYPE_CSV); - response.setHeader( - HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"trackedEntities.csv\""); - } + String attachment = + getAttachmentOrDefault(trackedEntityRequestParams.getAttachment(), "csv", "zip"); + + response.addHeader( + ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, + ContextUtils.BINARY_HEADER_CONTENT_TRANSFER_ENCODING); + response.setContentType(CONTENT_TYPE_CSV_ZIP); + response.setHeader( + HttpHeaders.CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); + + csvEventService.writeZip( + outputStream, + TRACKED_ENTITY_MAPPER.fromCollection( + trackedEntityService.getTrackedEntities(operationParams)), + !skipHeader, + attachment); + } + + @GetMapping(produces = {CONTENT_TYPE_CSV_GZIP}) + void getTrackedEntitiesAsCsvGZip( + TrackedEntityRequestParams trackedEntityRequestParams, + HttpServletResponse response, + HttpServletRequest request, + @CurrentUser User user, + @RequestParam(required = false, defaultValue = "false") boolean skipHeader) + throws IOException, BadRequestException, ForbiddenException, NotFoundException { + TrackedEntityOperationParams operationParams = + paramsMapper.map(trackedEntityRequestParams, user, CSV_FIELDS); + + String attachment = + getAttachmentOrDefault(trackedEntityRequestParams.getAttachment(), "csv", "gz"); + + response.addHeader( + ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING, + ContextUtils.BINARY_HEADER_CONTENT_TRANSFER_ENCODING); + response.setContentType(CONTENT_TYPE_CSV_GZIP); + response.setHeader( + HttpHeaders.CONTENT_DISPOSITION, getContentDispositionHeaderValue(attachment)); + + csvEventService.writeGzip( + response.getOutputStream(), + TRACKED_ENTITY_MAPPER.fromCollection( + trackedEntityService.getTrackedEntities(operationParams)), + !skipHeader); + } + + private String getAttachmentOrDefault(String filename, String type, String compression) { + return Objects.toString(filename, String.join(".", TRACKED_ENTITIES, type, compression)); + } + + private String getAttachmentOrDefault(String filename, String type) { + return Objects.toString(filename, String.join(".", TRACKED_ENTITIES, type)); + } - csvEventService.write(outputStream, trackedEntities, !skipHeader); + public String getContentDispositionHeaderValue(String filename) { + return "attachment; filename=" + filename; } @OpenApi.Response(OpenApi.EntityType.class) diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntityRequestParams.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntityRequestParams.java index 3b31c872e576..9ec1c0472ac1 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntityRequestParams.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntityRequestParams.java @@ -203,4 +203,6 @@ public class TrackedEntityRequestParams implements PageRequestParams { @OpenApi.Property(value = String[].class) private List fields = FieldFilterParser.parse(DEFAULT_FIELDS_PARAM); + + private String attachment; } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/ContextUtils.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/ContextUtils.java index 4101a629215b..3723fd0b08d6 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/ContextUtils.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/ContextUtils.java @@ -106,6 +106,8 @@ public class ContextUtils { public static final String HEADER_CONTENT_TRANSFER_ENCODING = "Content-Transfer-Encoding"; + public static final String BINARY_HEADER_CONTENT_TRANSFER_ENCODING = "binary"; + public static final String HEADER_VALUE_NO_STORE = "no-cache, no-store, max-age=0, must-revalidate"; diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CompressedEventServiceTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtilTest.java similarity index 90% rename from dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CompressedEventServiceTest.java rename to dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtilTest.java index 8007ae062f84..e50b68097e7a 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CompressedEventServiceTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtilTest.java @@ -25,8 +25,10 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.hisp.dhis.webapi.controller.tracker.export.event; +package org.hisp.dhis.webapi.controller.tracker.export; +import static org.hisp.dhis.webapi.controller.tracker.export.CompressionUtil.writeGzip; +import static org.hisp.dhis.webapi.controller.tracker.export.CompressionUtil.writeZip; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -49,27 +51,25 @@ import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) -class CompressedEventServiceTest { +class CompressionUtilTest { - private CompressedEventService compressedEventService; private static final Event FIRST_EVENT = new Event(); private static final Event SECOND_EVENT = new Event(); @InjectMocks private ObjectMapper objectMapper; @BeforeEach void setUp() { - compressedEventService = new CompressedEventService(objectMapper); FIRST_EVENT.setEvent(CodeGenerator.generateUid()); SECOND_EVENT.setEvent(CodeGenerator.generateUid()); } @Test - void shouldUnzipFileAndMatchEvents_whenCreateZipFileFromEventList() throws IOException { + void shouldUnzipFileAndMatchEventsWhenCreateZipFileFromEventList() throws IOException { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); List eventToZip = getEvents(); - compressedEventService.writeZip(outputStream, eventToZip, "file.json.zip"); + writeZip(outputStream, eventToZip, objectMapper.writer(), "file.json.zip"); ZipInputStream zipInputStream = new ZipInputStream(new ByteArrayInputStream(outputStream.toByteArray())); @@ -108,12 +108,12 @@ void shouldUnzipFileAndMatchEvents_whenCreateZipFileFromEventList() throws IOExc } @Test - void shouldGUnzipFileAndMatchEvents_whenCreateGZipFileFromEventList() throws IOException { + void shouldGUnzipFileAndMatchEventsWhenCreateGZipFileFromEventList() throws IOException { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); List eventToGZip = getEvents(); - compressedEventService.writeGzip(outputStream, eventToGZip); + writeGzip(outputStream, eventToGZip, objectMapper.writer()); GZIPInputStream gzipInputStream = new GZIPInputStream(new ByteArrayInputStream(outputStream.toByteArray())); From 08dd585ed517ae41ae5ba2fb234657c2f4b6aab8 Mon Sep 17 00:00:00 2001 From: luca Date: Thu, 21 Dec 2023 12:09:25 +0100 Subject: [PATCH 5/9] fix: sonar --- .../export/trackedentity/TrackedEntitiesExportController.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportController.java index d65e148b5bab..79fd05c6cf67 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportController.java @@ -43,7 +43,6 @@ import java.io.OutputStream; import java.util.List; import java.util.Objects; -import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.hisp.dhis.common.DhisApiVersion; import org.hisp.dhis.common.OpenApi; @@ -232,7 +231,6 @@ void getTrackedEntitiesAsCsvZip( void getTrackedEntitiesAsCsvGZip( TrackedEntityRequestParams trackedEntityRequestParams, HttpServletResponse response, - HttpServletRequest request, @CurrentUser User user, @RequestParam(required = false, defaultValue = "false") boolean skipHeader) throws IOException, BadRequestException, ForbiddenException, NotFoundException { From 145adaaebfa34c02a06242aa47da0f3b744ac203 Mon Sep 17 00:00:00 2001 From: luca Date: Thu, 4 Jan 2024 09:48:40 +0100 Subject: [PATCH 6/9] fix: compress csv object, improve testing --- .../TrackedEntitiesExportControllerTest.java | 36 ++++++++ .../tracker/export/event/CsvEventService.java | 41 +++++---- .../CsvTrackedEntityService.java | 57 +++++++------ .../TrackedEntitiesExportController.java | 5 +- .../tracker/export/CompressionUtilTest.java | 8 +- .../export/event/CsvEventServiceTest.java | 69 +++++++++++++++ .../CsvTrackedEntityServiceTest.java | 83 +++++++++++++++++++ 7 files changed, 248 insertions(+), 51 deletions(-) diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportControllerTest.java index 4988e2f01492..95d72d08a931 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportControllerTest.java @@ -28,6 +28,7 @@ package org.hisp.dhis.webapi.controller.tracker.export.trackedentity; import static org.hisp.dhis.common.OrganisationUnitSelectionMode.ACCESSIBLE; +import static org.hisp.dhis.web.WebClient.Accept; import static org.hisp.dhis.webapi.controller.tracker.JsonAssertions.assertContainsAll; import static org.hisp.dhis.webapi.controller.tracker.JsonAssertions.assertFirstRelationship; import static org.hisp.dhis.webapi.controller.tracker.JsonAssertions.assertHasMember; @@ -429,6 +430,41 @@ void getTrackedEntityReturnsCsvFormat() { assertTrue(response.content().toString().contains("trackedEntity,trackedEntityType"))); } + @Test + void getTrackedEntityCsvById() { + TrackedEntity te = trackedEntity(); + this.switchContextToUser(user); + + WebClient.HttpResponse response = + GET("/tracker/trackedEntities/{id}", te.getUid(), Accept(ContextUtils.CONTENT_TYPE_CSV)); + + String csvResponse = response.content(ContextUtils.CONTENT_TYPE_CSV); + + assertTrue(response.header("content-type").contains(ContextUtils.CONTENT_TYPE_CSV)); + assertTrue(response.header("content-disposition").contains("filename=trackedEntity.csv")); + assertEquals(trackedEntityToCsv(te), csvResponse); + } + + String trackedEntityToCsv(TrackedEntity te) { + return """ + trackedEntity,trackedEntityType,createdAt,createdAtClient,updatedAt,updatedAtClient,orgUnit,inactive,deleted,potentialDuplicate,geometry,latitude,longitude,storedBy,createdBy,updatedBy,attrCreatedAt,attrUpdatedAt,attribute,displayName,value,valueType + """ + .concat( + String.join( + ",", + te.getUid(), + te.getTrackedEntityType().getUid(), + DateUtils.instantFromDate(te.getCreated()).toString(), + DateUtils.instantFromDate(te.getCreatedAtClient()).toString(), + DateUtils.instantFromDate(te.getLastUpdated()).toString(), + DateUtils.instantFromDate(te.getLastUpdatedAtClient()).toString(), + te.getOrganisationUnit().getUid(), + Boolean.toString(te.isInactive()), + Boolean.toString(te.isDeleted()), + Boolean.toString(te.isPotentialDuplicate()), + ",,,,,,,,,,," + "\n")); + } + @Test void getTrackedEntityReturnsCsvZipFormat() { injectSecurityContext(user); diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventService.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventService.java index 2ef352df00e1..f9c91711779f 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventService.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventService.java @@ -68,35 +68,22 @@ public void write(OutputStream outputStream, List events, boolean withHea throws IOException { ObjectWriter writer = getObjectWriter(withHeader); - List dataValues = new ArrayList<>(); - - for (Event event : events) { - CsvEventDataValue templateDataValue = map(event); - - if (event.getDataValues().isEmpty()) { - dataValues.add(templateDataValue); - continue; - } - - for (DataValue value : event.getDataValues()) { - dataValues.add(map(value, templateDataValue)); - } - } - - writer.writeValue(outputStream, dataValues); + writer.writeValue(outputStream, getCsvEventDataValues(events)); } @Override public void writeZip( OutputStream outputStream, List toCompress, boolean withHeader, String file) throws IOException { - CompressionUtil.writeZip(outputStream, toCompress, getObjectWriter(withHeader), file); + CompressionUtil.writeZip( + outputStream, getCsvEventDataValues(toCompress), getObjectWriter(withHeader), file); } @Override public void writeGzip(OutputStream outputStream, List toCompress, boolean withHeader) throws IOException { - CompressionUtil.writeGzip(outputStream, toCompress, getObjectWriter(withHeader)); + CompressionUtil.writeGzip( + outputStream, getCsvEventDataValues(toCompress), getObjectWriter(withHeader)); } private ObjectWriter getObjectWriter(boolean withHeader) { @@ -109,6 +96,24 @@ private ObjectWriter getObjectWriter(boolean withHeader) { return CSV_MAPPER.writer(csvSchema.withUseHeader(withHeader)); } + private List getCsvEventDataValues(List events) { + List dataValues = new ArrayList<>(); + + for (Event event : events) { + CsvEventDataValue templateDataValue = map(event); + + if (event.getDataValues().isEmpty()) { + dataValues.add(templateDataValue); + continue; + } + + for (DataValue value : event.getDataValues()) { + dataValues.add(map(value, templateDataValue)); + } + } + return dataValues; + } + private static CsvEventDataValue map(Event event) { CsvEventDataValue result = new CsvEventDataValue(); result.setEvent(event.getEvent()); diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/CsvTrackedEntityService.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/CsvTrackedEntityService.java index b88b1a818d41..67d964a3075c 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/CsvTrackedEntityService.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/CsvTrackedEntityService.java @@ -55,6 +55,36 @@ public void write( throws IOException { ObjectWriter writer = getObjectWriter(withHeader); + writer.writeValue(outputStream, getCsvTrackedEntities(trackedEntities)); + } + + @Override + public void writeZip( + OutputStream outputStream, List toCompress, boolean withHeader, String file) + throws IOException { + CompressionUtil.writeZip( + outputStream, getCsvTrackedEntities(toCompress), getObjectWriter(withHeader), file); + } + + @Override + public void writeGzip( + OutputStream outputStream, List toCompress, boolean withHeader) + throws IOException { + CompressionUtil.writeGzip( + outputStream, getCsvTrackedEntities(toCompress), getObjectWriter(withHeader)); + } + + private ObjectWriter getObjectWriter(boolean withHeader) { + final CsvSchema csvSchema = + CSV_MAPPER + .schemaFor(CsvTrackedEntity.class) + .withLineSeparator("\n") + .withUseHeader(withHeader); + + return CSV_MAPPER.writer(csvSchema.withUseHeader(withHeader)); + } + + private List getCsvTrackedEntities(List trackedEntities) { List attributes = new ArrayList<>(); for (TrackedEntity trackedEntity : trackedEntities) { @@ -90,32 +120,7 @@ public void write( addAttributes(trackedEntity, trackedEntityValue, attributes); } } - - writer.writeValue(outputStream, attributes); - } - - @Override - public void writeZip( - OutputStream outputStream, List toCompress, boolean withHeader, String file) - throws IOException { - CompressionUtil.writeZip(outputStream, toCompress, getObjectWriter(withHeader), file); - } - - @Override - public void writeGzip( - OutputStream outputStream, List toCompress, boolean withHeader) - throws IOException { - CompressionUtil.writeGzip(outputStream, toCompress, getObjectWriter(withHeader)); - } - - private ObjectWriter getObjectWriter(boolean withHeader) { - final CsvSchema csvSchema = - CSV_MAPPER - .schemaFor(CsvTrackedEntity.class) - .withLineSeparator("\n") - .withUseHeader(withHeader); - - return CSV_MAPPER.writer(csvSchema.withUseHeader(withHeader)); + return attributes; } private String checkForNull(Instant instant) { diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportController.java index 79fd05c6cf67..0631cea7e6dc 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportController.java @@ -288,7 +288,7 @@ ResponseEntity getTrackedEntityByUid( @GetMapping( value = "/{uid}", - produces = {CONTENT_TYPE_CSV, CONTENT_TYPE_CSV_GZIP, CONTENT_TYPE_TEXT_CSV}) + produces = {CONTENT_TYPE_CSV, CONTENT_TYPE_TEXT_CSV}) void getTrackedEntityByUidAsCsv( @PathVariable String uid, HttpServletResponse response, @@ -303,8 +303,7 @@ void getTrackedEntityByUidAsCsv( OutputStream outputStream = response.getOutputStream(); response.setContentType(CONTENT_TYPE_CSV); - response.setHeader( - HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"trackedEntity.csv\""); + response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=trackedEntity.csv"); csvEventService.write(outputStream, List.of(trackedEntity), !skipHeader); } } diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtilTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtilTest.java index e50b68097e7a..c1e4764dadc3 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtilTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/CompressionUtilTest.java @@ -97,14 +97,14 @@ void shouldUnzipFileAndMatchEventsWhenCreateZipFileFromEventList() throws IOExce .filter(e -> e.getEvent().equals(FIRST_EVENT.getEvent())) .findAny() .orElse(null), - "The event does not match or does not exist in the Zip File."); + "The event does not match or not exists in the Zip File."); assertEquals( SECOND_EVENT, eventsFromZip.stream() .filter(e -> e.getEvent().equals(SECOND_EVENT.getEvent())) .findAny() .orElse(null), - "The event does not match or does not exist in the Zip File."); + "The event does not match or not exists in the Zip File."); } @Test @@ -135,14 +135,14 @@ void shouldGUnzipFileAndMatchEventsWhenCreateGZipFileFromEventList() throws IOEx .filter(e -> e.getEvent().equals(FIRST_EVENT.getEvent())) .findAny() .orElse(null), - "The event does not match or does not exist in the GZip File."); + "The event does not match or not exists in the GZip File."); assertEquals( SECOND_EVENT, eventToGZip.stream() .filter(e -> e.getEvent().equals(SECOND_EVENT.getEvent())) .findAny() .orElse(null), - "The event does not match or does not exist in the GZip File."); + "The event does not match or not exists in the GZip File."); } List getEvents() { diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventServiceTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventServiceTest.java index 1e9c234f4bfd..b133abe93df2 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventServiceTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventServiceTest.java @@ -29,6 +29,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -42,6 +43,9 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.hisp.dhis.event.EventStatus; import org.hisp.dhis.webapi.controller.tracker.view.DataValue; import org.hisp.dhis.webapi.controller.tracker.view.Event; @@ -224,4 +228,69 @@ void testReadEventsParsesGeometryEvenIfQuoted(String csv) throws IOException, Pa geometryFactory.createPoint(new Coordinate(-11.4283223849698, 8.06311527044516)); assertEquals(expected, events.get(0).getGeometry()); } + + @Test + void zipFileAndMatchCsvEventDataValuesWhenCreateZipFileFromEventList() + throws IOException, ParseException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + File event = new File("src/test/resources/controller/tracker/csv/completeEvent.csv"); + InputStream inputStream = Files.asByteSource(event).openStream(); + + List events = service.read(inputStream, false); + + service.writeZip(outputStream, events, false, "file.json.zip"); + + ZipInputStream zipInputStream = + new ZipInputStream(new ByteArrayInputStream(outputStream.toByteArray())); + var buff = new byte[1024]; + + ZipEntry zipEntry = zipInputStream.getNextEntry(); + + assertNotNull(zipEntry, "Events Zip file has no entry"); + assertEquals("file.json.zip", zipEntry.getName(), "Events Zip file has a wrong name"); + + var csvStream = new ByteArrayOutputStream(); + int l; + while ((l = zipInputStream.read(buff)) > 0) { + csvStream.write(buff, 0, l); + } + + assertEquals( + """ +eventId,COMPLETED,programId,programStageId,enrollmentId,orgUnitId,2020-02-26T23:01:00Z,2020-02-26T23:02:00Z,,,,false,false,2020-02-26T23:03:00Z,,2020-02-26T23:05:00Z,,admin,2020-02-26T23:07:00Z,,attributeOptionCombo,,,dataElement,value,admin,false,,2020-02-26T23:08:00Z,2020-02-26T23:09:00Z +""", + csvStream.toString(), + "The event does not match or not exists in the Zip File."); + } + + @Test + void gzipFileAndMatchCsvEventDataValuesWhenCreateGZipFileFromEventList() + throws IOException, ParseException { + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + File event = new File("src/test/resources/controller/tracker/csv/completeEvent.csv"); + InputStream inputStream = Files.asByteSource(event).openStream(); + + List events = service.read(inputStream, false); + service.writeGzip(outputStream, events, false); + + GZIPInputStream gzipInputStream = + new GZIPInputStream(new ByteArrayInputStream(outputStream.toByteArray())); + var buff = new byte[1024]; + + var csvStream = new ByteArrayOutputStream(); + int l; + while ((l = gzipInputStream.read(buff)) > 0) { + csvStream.write(buff, 0, l); + } + + assertEquals( + """ +eventId,COMPLETED,programId,programStageId,enrollmentId,orgUnitId,2020-02-26T23:01:00Z,2020-02-26T23:02:00Z,,,,false,false,2020-02-26T23:03:00Z,,2020-02-26T23:05:00Z,,admin,2020-02-26T23:07:00Z,,attributeOptionCombo,,,dataElement,value,admin,false,,2020-02-26T23:08:00Z,2020-02-26T23:09:00Z +""", + csvStream.toString(), + "The event does not match or not exists in the GZip File."); + } } diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/CsvTrackedEntityServiceTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/CsvTrackedEntityServiceTest.java index b1772787b8d9..0a40c0a539d7 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/CsvTrackedEntityServiceTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/CsvTrackedEntityServiceTest.java @@ -28,7 +28,9 @@ package org.hisp.dhis.webapi.controller.tracker.export.trackedentity; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.time.Instant; @@ -36,6 +38,9 @@ import java.util.Arrays; import java.util.List; import java.util.stream.IntStream; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.geotools.geojson.geom.GeometryJSON; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.webapi.controller.tracker.view.Attribute; @@ -146,6 +151,84 @@ void multipleTrackedEntitiesWithAttributesAreWritten() throws IOException { out.toString()); } + @Test + void zipFileAndMatchCsvTeWhenCreateZipFileFromTrackedEntityList() throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + List trackedEntities = new ArrayList<>(); + + trackedEntities.add(getTrackedEntityToCompress()); + + service.writeZip(outputStream, trackedEntities, false, "file.json.zip"); + + ZipInputStream zipInputStream = + new ZipInputStream(new ByteArrayInputStream(outputStream.toByteArray())); + var buff = new byte[1024]; + + ZipEntry zipEntry = zipInputStream.getNextEntry(); + + assertNotNull(zipEntry, "Events Zip file has no entry"); + assertEquals("file.json.zip", zipEntry.getName(), "Events Zip file has a wrong name"); + + var csvStream = new ByteArrayOutputStream(); + int l; + while ((l = zipInputStream.read(buff)) > 0) { + csvStream.write(buff, 0, l); + } + + assertEquals( + """ +"Test tracked entity",,2022-09-29T15:15:30Z,,,,"Test org unit",false,false,false,"POINT (40 5)",5.0,40.0,,,,,,"attribute 1",,"Age test",AGE +"Test tracked entity",,2022-09-29T15:15:30Z,,,,"Test org unit",false,false,false,"POINT (40 5)",5.0,40.0,,,,,,"attribute 2",,"Text test",TEXT +""", + csvStream.toString(), + "The tracked entity does not match or not exists in the Zip File."); + } + + @Test + void gzipFileAndMatchCsvTeWhenCreateGZipFileFromTrackedEntityList() throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + List trackedEntities = new ArrayList<>(); + + trackedEntities.add(getTrackedEntityToCompress()); + + service.writeGzip(outputStream, trackedEntities, false); + + GZIPInputStream gzipInputStream = + new GZIPInputStream(new ByteArrayInputStream(outputStream.toByteArray())); + var buff = new byte[1024]; + + var csvStream = new ByteArrayOutputStream(); + int l; + while ((l = gzipInputStream.read(buff)) > 0) { + csvStream.write(buff, 0, l); + } + + assertEquals( + """ +"Test tracked entity",,2022-09-29T15:15:30Z,,,,"Test org unit",false,false,false,"POINT (40 5)",5.0,40.0,,,,,,"attribute 1",,"Age test",AGE +"Test tracked entity",,2022-09-29T15:15:30Z,,,,"Test org unit",false,false,false,"POINT (40 5)",5.0,40.0,,,,,,"attribute 2",,"Text test",TEXT +""", + csvStream.toString(), + "The tracked entity does not match or not exists in the Zip File."); + } + + private TrackedEntity getTrackedEntityToCompress() throws IOException { + TrackedEntity trackedEntity = createTrackedEntity(); + + GeometryJSON geometryJSON = new GeometryJSON(); + String pointCoordinates = "[40,5]"; + Geometry geometry = + geometryJSON.read("{\"type\":\"Point\", \"coordinates\": " + pointCoordinates + " }"); + trackedEntity.setGeometry(geometry); + + Attribute attribute1 = createAttribute("attribute 1", ValueType.AGE, "Age test"); + Attribute attribute2 = createAttribute("attribute 2", ValueType.TEXT, "Text test"); + trackedEntity.setAttributes(Arrays.asList(attribute1, attribute2)); + return trackedEntity; + } + private Attribute createAttribute(String attr, ValueType valueType, String value) { Attribute attribute = new Attribute(); attribute.setAttribute(attr); From 31d4efa97b0bc1d347d645419406d75a4a53c8c3 Mon Sep 17 00:00:00 2001 From: luca Date: Fri, 5 Jan 2024 13:41:04 +0100 Subject: [PATCH 7/9] fix: improve tests --- .../tracker/export/event/EventsExportControllerTest.java | 2 ++ .../TrackedEntitiesExportControllerTest.java | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java index 0433a06a4411..478e01345e49 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java @@ -29,6 +29,7 @@ import static org.hisp.dhis.utils.Assertions.assertStartsWith; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; @@ -188,6 +189,7 @@ void shouldMatchContentTypeAndAttachment_whenEndpointForCompressedEventJsonIsInv assertEquals(expectedContentType, res.header("Content-Type")); assertEquals(expectedAttachment, res.header(ContextUtils.HEADER_CONTENT_DISPOSITION)); assertEquals(encoding, res.header(ContextUtils.HEADER_CONTENT_TRANSFER_ENCODING)); + assertNotNull(res.content(expectedContentType)); } private UserAccess userAccess() { diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportControllerTest.java index 95d72d08a931..14ee70553c67 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportControllerTest.java @@ -37,6 +37,7 @@ import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import java.time.LocalDate; @@ -484,7 +485,8 @@ void getTrackedEntityReturnsCsvZipFormat() { assertTrue( response .header("content-disposition") - .contains("filename=trackedEntities.csv.zip"))); + .contains("filename=trackedEntities.csv.zip")), + () -> assertNotNull(response.content(ContextUtils.CONTENT_TYPE_CSV_ZIP))); } @Test @@ -505,9 +507,8 @@ void getTrackedEntityReturnsCsvGZipFormat() { response.header("content-type").contains(ContextUtils.CONTENT_TYPE_CSV_GZIP)), () -> assertTrue( - response - .header("content-disposition") - .contains("filename=trackedEntities.csv.gz"))); + response.header("content-disposition").contains("filename=trackedEntities.csv.gz")), + () -> assertNotNull(response.content(ContextUtils.CONTENT_TYPE_CSV_GZIP))); } @Test From 2fda449f6d82b72f48597d6611bb06e81a9cad93 Mon Sep 17 00:00:00 2001 From: luca Date: Fri, 5 Jan 2024 13:41:49 +0100 Subject: [PATCH 8/9] fix: add to open api --- .../src/main/resources/openapi/EventsExportController.md | 5 +++++ .../resources/openapi/TrackedEntitiesExportController.md | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/dhis-2/dhis-web-api/src/main/resources/openapi/EventsExportController.md b/dhis-2/dhis-web-api/src/main/resources/openapi/EventsExportController.md index 2fb1b0349b81..c58b2135d120 100644 --- a/dhis-2/dhis-web-api/src/main/resources/openapi/EventsExportController.md +++ b/dhis-2/dhis-web-api/src/main/resources/openapi/EventsExportController.md @@ -259,3 +259,8 @@ Valid operators are: - `NLIKE` - not like - `SW` - starts with - `EW` - ends with + +### `*.parameter.EventRequestParams.attachment` + +It allows you to specify the attachment file name when extracting in a binary format such as CSV, zip, or gzip. +If not specified, it defaults to `events..` (for example, `events.csv.zip` for zip compression of a csv list) \ No newline at end of file diff --git a/dhis-2/dhis-web-api/src/main/resources/openapi/TrackedEntitiesExportController.md b/dhis-2/dhis-web-api/src/main/resources/openapi/TrackedEntitiesExportController.md index 4feb8287a826..9697f5aff610 100644 --- a/dhis-2/dhis-web-api/src/main/resources/openapi/TrackedEntitiesExportController.md +++ b/dhis-2/dhis-web-api/src/main/resources/openapi/TrackedEntitiesExportController.md @@ -199,3 +199,8 @@ Valid operators are: - `NLIKE` - not like - `SW` - starts with - `EW` - ends with + +### `*.parameter.TrackedEntityRequestParams.attachment` + +It allows you to specify the attachment file name when extracting in a binary format such as CSV, zip, or gzip. +If not specified, it defaults to `trackedentitites..` (for example, `trackedentitites.csv.zip` for zip compression of a csv list) \ No newline at end of file From e5911b9477426ae67fd5d1dae25efff5c8e26c8f Mon Sep 17 00:00:00 2001 From: luca Date: Fri, 5 Jan 2024 13:48:29 +0100 Subject: [PATCH 9/9] fix: typo --- .../main/resources/openapi/TrackedEntitiesExportController.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-web-api/src/main/resources/openapi/TrackedEntitiesExportController.md b/dhis-2/dhis-web-api/src/main/resources/openapi/TrackedEntitiesExportController.md index 9697f5aff610..0484f8b06fe6 100644 --- a/dhis-2/dhis-web-api/src/main/resources/openapi/TrackedEntitiesExportController.md +++ b/dhis-2/dhis-web-api/src/main/resources/openapi/TrackedEntitiesExportController.md @@ -203,4 +203,4 @@ Valid operators are: ### `*.parameter.TrackedEntityRequestParams.attachment` It allows you to specify the attachment file name when extracting in a binary format such as CSV, zip, or gzip. -If not specified, it defaults to `trackedentitites..` (for example, `trackedentitites.csv.zip` for zip compression of a csv list) \ No newline at end of file +If not specified, it defaults to `trackedEntities..` (for example, `trackedEntities.csv.zip` for zip compression of a csv list) \ No newline at end of file