fix: Use new start and end date types in tracker exporter [DHIS2-1601…

…9] (2.40) (#17648)

* feat: Create StartDate and EndDate types [DHIS2-16019] (#17454)

* fix: Use new start and end date types in tracker exporter [DHIS2-16019] (#17522)

* Fix tests

dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/DateUtils.java

@@ -33,7 +33,9 @@
 import java.time.Duration;
 import java.time.Instant;
 import java.time.LocalDateTime;
+import java.time.LocalTime;
 import java.time.OffsetDateTime;
+import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.temporal.ChronoUnit;
 import java.util.Calendar;
@@ -109,6 +111,9 @@ public class DateUtils {
       ObjectArrays.concat(
           SUPPORTED_DATE_ONLY_PARSERS, SUPPORTED_DATE_TIME_FORMAT_PARSERS, DateTimeParser.class);
 
+  private static final DateTimeFormatter ONLY_DATE_FORMATTER =
+      new DateTimeFormatterBuilder().append(null, SUPPORTED_DATE_ONLY_PARSERS).toFormatter();
+
   private static final DateTimeFormatter DATE_FORMATTER =
       new DateTimeFormatterBuilder().append(null, SUPPORTED_DATE_FORMAT_PARSERS).toFormatter();
 
@@ -137,6 +142,9 @@ public class DateUtils {
   private static final DateTimeFormatter LONG_DATE_FORMAT =
       DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss");
 
+  private static final DateTimeFormatter LONG_DATE_FORMAT_WITH_MILLIS =
+      DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS");
+
   private static final DateTimeFormatter HTTP_DATE_FORMAT =
       DateTimeFormat.forPattern("EEE, dd MMM yyyy HH:mm:ss 'GMT'").withLocale(Locale.ENGLISH);
 
@@ -178,6 +186,16 @@ public static String getLongGmtDateString(Date date) {
     return date != null ? TIMESTAMP_UTC_TZ_FORMAT.print(new DateTime(date)) : null;
   }
 
+  /**
+   * Formats a Date to the format yyyy-MM-dd HH:mm:ss.SSS.
+   *
+   * @param date the Date to parse.
+   * @return A formatted date string.
+   */
+  public static String toLongDateWithMillis(Date date) {
+    return date != null ? LONG_DATE_FORMAT_WITH_MILLIS.print(new DateTime(date)) : null;
+  }
+
   /**
    * Formats a Date to the format yyyy-MM-dd HH:mm:ss.
    *
@@ -648,8 +666,8 @@ public static String getPrettyInterval(Date start, Date end) {
   }
 
   /**
-   * Parses the given string into a Date using the supported date formats. Returns null if the
-   * string cannot be parsed.
+   * Parses the given string into a Date using the supported date formats. Add time at the beginning
+   * of the day if no time was provided. Returns null if the string cannot be parsed.
    *
    * @param dateString the date string.
    * @return a date.
@@ -658,6 +676,29 @@ public static Date parseDate(String dateString) {
     return safeParseDateTime(dateString, DATE_FORMATTER);
   }
 
+  /**
+   * Parses the given string into a Date using the supported date formats. Add time at the end of
+   * the day if no time was provided. Returns null if the string cannot be parsed.
+   *
+   * @param dateString the date string.
+   * @return a date.
+   */
+  public static Date parseDateEndOfTheDay(String dateString) {
+    if (StringUtils.isEmpty(dateString)) {
+      return null;
+    }
+
+    try {
+      Date date = safeParseDateTime(dateString, ONLY_DATE_FORMATTER);
+      LocalDateTime localDateTime =
+          LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()).with(LocalTime.MAX);
+      return Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant());
+    } catch (IllegalArgumentException e) {
+      // dateString has time defined
+    }
+    return safeParseDateTime(dateString, DATE_FORMATTER);
+  }
+
   /**
    * Parses the given string into a Date using the supported date formats. Returns null if the
    * string cannot be parsed.

dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/ObjectUtils.java

@@ -29,10 +29,13 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
 import org.apache.commons.lang3.StringUtils;
 
 /**
@@ -168,4 +171,48 @@ public static <T, U extends RuntimeException> T throwIfNull(T object, Supplier<U
 
     return object;
   }
+
+  /**
+   * If object is null return null, otherwise return function applied to object.
+   *
+   * @param object to check.
+   * @param <U> the function return type.
+   * @param <T> the object type.
+   * @param function the function to be applied to non-null object.
+   */
+  public static <T, U> U applyIfNotNull(T object, Function<T, U> function) {
+    if (object == null) {
+      return null;
+    }
+
+    return function.apply(object);
+  }
+
+  /**
+   * Util method that always returns a new Set, either instantiated from a non-null Set passed as an
+   * argument, or if a null arg is passed then returning an empty Set. This helps reduce possible
+   * NullPointerExceptions when trying to instantiate a Set with a null value.
+   *
+   * @param set
+   * @return
+   * @param <T>
+   */
+  @Nonnull
+  public static <T> Set<T> copyOf(Set<T> set) {
+    return set != null ? new HashSet<>(set) : new HashSet<>();
+  }
+
+  /**
+   * Util method that always returns a new List, either instantiated from a non-null Set passed as
+   * an argument, or if a null arg is passed then returning an empty Set. This helps reduce possible
+   * NullPointerExceptions when trying to instantiate a Set with a null value.
+   *
+   * @param list
+   * @return
+   * @param <T>
+   */
+  @Nonnull
+  public static <T> List<T> copyOf(List<T> list) {
+    return list != null ? new ArrayList<>(list) : new ArrayList<>();
+  }
 }

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/hibernate/HibernateProgramInstanceStore.java

@@ -30,9 +30,9 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids;
 import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString;
-import static org.hisp.dhis.util.DateUtils.getLongDateString;
 import static org.hisp.dhis.util.DateUtils.getLongGmtDateString;
 import static org.hisp.dhis.util.DateUtils.nowMinusDuration;
+import static org.hisp.dhis.util.DateUtils.toLongDateWithMillis;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
@@ -157,7 +157,10 @@ private QueryWithOrderBy buildProgramInstanceHql(ProgramInstanceQueryParams para
               + "'";
     } else if (params.hasLastUpdated()) {
       hql +=
-          hlp.whereAnd() + "pi.lastUpdated >= '" + getLongDateString(params.getLastUpdated()) + "'";
+          hlp.whereAnd()
+              + "pi.lastUpdated >= '"
+              + toLongDateWithMillis(params.getLastUpdated())
+              + "'";
     }
 
     if (params.hasTrackedEntityInstance()) {
@@ -211,15 +214,15 @@ private QueryWithOrderBy buildProgramInstanceHql(ProgramInstanceQueryParams para
       hql +=
           hlp.whereAnd()
               + "pi.enrollmentDate >= '"
-              + getLongDateString(params.getProgramStartDate())
+              + toLongDateWithMillis(params.getProgramStartDate())
               + "'";
     }
 
     if (params.hasProgramEndDate()) {
       hql +=
           hlp.whereAnd()
               + "pi.enrollmentDate <= '"
-              + getLongDateString(params.getProgramEndDate())
+              + toLongDateWithMillis(params.getProgramEndDate())
               + "'";
     }
 

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/trackedentity/hibernate/HibernateTrackedEntityInstanceStore.java

@@ -46,8 +46,8 @@
 import static org.hisp.dhis.trackedentity.TrackedEntityInstanceQueryParams.PROGRAM_INSTANCE_ALIAS;
 import static org.hisp.dhis.trackedentity.TrackedEntityInstanceQueryParams.TRACKED_ENTITY_ID;
 import static org.hisp.dhis.trackedentity.TrackedEntityInstanceQueryParams.TRACKED_ENTITY_INSTANCE_ID;
-import static org.hisp.dhis.util.DateUtils.getLongDateString;
 import static org.hisp.dhis.util.DateUtils.getLongGmtDateString;
+import static org.hisp.dhis.util.DateUtils.toLongDateWithMillis;
 
 import com.google.common.collect.Lists;
 import java.util.ArrayList;
@@ -647,14 +647,14 @@ private String getFromSubQueryTrackedEntityConditions(
         trackedEntity
             .append(whereAnd.whereAnd())
             .append(" TEI.lastupdated >= '")
-            .append(getLongDateString(params.getLastUpdatedStartDate()))
+            .append(toLongDateWithMillis(params.getLastUpdatedStartDate()))
             .append(SINGLE_QUOTE);
       }
       if (params.hasLastUpdatedEndDate()) {
         trackedEntity
             .append(whereAnd.whereAnd())
             .append(" TEI.lastupdated <= '")
-            .append(getLongDateString(params.getLastUpdatedEndDate()))
+            .append(toLongDateWithMillis(params.getLastUpdatedEndDate()))
             .append(SINGLE_QUOTE);
       }
     }
@@ -663,7 +663,7 @@ private String getFromSubQueryTrackedEntityConditions(
       if (params.getSkipChangedBefore() != null) {
         trackedEntity
             .append(" AND TEI.lastupdated >= '")
-            .append(getLongDateString(params.getSkipChangedBefore()))
+            .append(toLongDateWithMillis(params.getSkipChangedBefore()))
             .append(SINGLE_QUOTE);
       }
     }
@@ -985,28 +985,28 @@ private String getFromSubQueryProgramInstanceConditions(
     if (params.hasProgramEnrollmentStartDate()) {
       program
           .append("AND PI.enrollmentdate >= '")
-          .append(getLongDateString(params.getProgramEnrollmentStartDate()))
+          .append(toLongDateWithMillis(params.getProgramEnrollmentStartDate()))
           .append("' ");
     }
 
     if (params.hasProgramEnrollmentEndDate()) {
       program
           .append("AND PI.enrollmentdate <= '")
-          .append(getLongDateString(params.getProgramEnrollmentEndDate()))
+          .append(toLongDateWithMillis(params.getProgramEnrollmentEndDate()))
           .append("' ");
     }
 
     if (params.hasProgramIncidentStartDate()) {
       program
           .append("AND PI.incidentdate >= '")
-          .append(getLongDateString(params.getProgramIncidentStartDate()))
+          .append(toLongDateWithMillis(params.getProgramIncidentStartDate()))
           .append("' ");
     }
 
     if (params.hasProgramIncidentEndDate()) {
       program
           .append("AND PI.incidentdate <= '")
-          .append(getLongDateString(params.getProgramIncidentEndDate()))
+          .append(toLongDateWithMillis(params.getProgramIncidentEndDate()))
           .append("' ");
     }
 
@@ -1047,8 +1047,8 @@ private String getFromSubQueryProgramStageInstance(TrackedEntityInstanceQueryPar
     }
 
     if (params.hasEventStatus()) {
-      String start = getLongDateString(params.getEventStartDate());
-      String end = getLongDateString(params.getEventEndDate());
+      String start = toLongDateWithMillis(params.getEventStartDate());
+      String end = toLongDateWithMillis(params.getEventEndDate());
 
       if (params.isEventStatus(EventStatus.COMPLETED)) {
         events

dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/JdbcEventStore.java

@@ -1284,7 +1284,7 @@ private StringBuilder getFromWhereClause(
     }
 
     if (params.getStartDate() != null) {
-      mapSqlParameterSource.addValue("startDate", params.getStartDate(), Types.DATE);
+      mapSqlParameterSource.addValue("startDate", params.getStartDate(), Types.TIMESTAMP);
 
       fromBuilder
           .append(hlp.whereAnd())
@@ -1706,7 +1706,7 @@ private String getFromWhereClause(
     }
 
     if (params.getStartDate() != null) {
-      mapSqlParameterSource.addValue("startDate", params.getStartDate(), Types.DATE);
+      mapSqlParameterSource.addValue("startDate", params.getStartDate(), Types.TIMESTAMP);
 
       sqlBuilder
           .append(hlp.whereAnd())
@@ -1745,7 +1745,7 @@ private String getFromWhereClause(
     }
 
     if (params.getDueDateStart() != null) {
-      mapSqlParameterSource.addValue("startDueDate", params.getDueDateStart(), Types.DATE);
+      mapSqlParameterSource.addValue("startDueDate", params.getDueDateStart(), Types.TIMESTAMP);
 
       sqlBuilder
           .append(hlp.whereAnd())
@@ -1755,7 +1755,7 @@ private String getFromWhereClause(
     }
 
     if (params.getDueDateEnd() != null) {
-      mapSqlParameterSource.addValue("endDueDate", params.getDueDateEnd(), Types.DATE);
+      mapSqlParameterSource.addValue("endDueDate", params.getDueDateEnd(), Types.TIMESTAMP);
 
       sqlBuilder
           .append(hlp.whereAnd())

dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/export/TrackerExportTests.java

@@ -467,7 +467,7 @@ public void shouldNotReturnRelationshipsWhenEventHasRelationshipsAndFieldsExclud
   public void shouldReturnFilteredEvent() {
     trackerActions
         .get(
-            "events?enrollmentOccurredAfter=2019-08-16&enrollmentOccurredBefore=2019-08-20&event=ZwwuwNp6gVd")
+            "events?enrollmentOccurredAfter=2019-08-19&enrollmentOccurredBefore=2019-08-19&event=ZwwuwNp6gVd")
         .validate()
         .statusCode(200)
         .rootPath("instances[0]")

dhis-2/dhis-test-e2e/src/test/resources/tracker/importer/teis/teisWithEnrollmentsAndEvents.json

@@ -12,7 +12,7 @@
           "enrollment": "MNWZ6hnuhSw",
           "enrolledAt": "2019-08-19T00:00:00.000",
           "deleted": false,
-          "occurredAt": "2019-08-19T00:00:00.000",
+          "occurredAt": "2019-08-19T12:00:00.000",
           "status": "ACTIVE",
           "notes": [],
           "relationships": [],

dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEnrollmentCriteria.java

@@ -29,7 +29,6 @@
 
 import static org.hisp.dhis.webapi.controller.event.webrequest.tracker.FieldTranslatorSupport.translate;
 
-import java.util.Date;
 import java.util.Optional;
 import lombok.Data;
 import lombok.Getter;
@@ -38,6 +37,8 @@
 import org.hisp.dhis.common.OrganisationUnitSelectionMode;
 import org.hisp.dhis.program.ProgramStatus;
 import org.hisp.dhis.webapi.controller.event.webrequest.PagingAndSortingCriteriaAdapter;
+import org.hisp.dhis.webapi.webdomain.EndDateTime;
+import org.hisp.dhis.webapi.webdomain.StartDateTime;
 
 /** Represents query parameters sent to {@link TrackerEnrollmentsExportController}. */
 @Data
@@ -53,13 +54,13 @@ public class TrackerEnrollmentCriteria extends PagingAndSortingCriteriaAdapter {
 
   private Boolean followUp;
 
-  private Date updatedAfter;
+  private StartDateTime updatedAfter;
 
   private String updatedWithin;
 
-  private Date enrolledAfter;
+  private StartDateTime enrolledAfter;
 
-  private Date enrolledBefore;
+  private EndDateTime enrolledBefore;
 
   private String trackedEntityType;
 

dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/TrackerEnrollmentCriteriaMapper.java

@@ -30,6 +30,7 @@
 import static org.apache.commons.lang3.BooleanUtils.toBooleanDefaultIfNull;
 import static org.apache.commons.lang3.StringUtils.isNotEmpty;
 import static org.hisp.dhis.common.OrganisationUnitSelectionMode.ALL;
+import static org.hisp.dhis.util.ObjectUtils.applyIfNotNull;
 import static org.hisp.dhis.webapi.controller.event.mapper.OrderParamsHelper.toOrderParams;
 import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamUtils.applyIfNonEmpty;
 import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamUtils.parseUids;
@@ -54,6 +55,8 @@
 import org.hisp.dhis.trackedentity.TrackedEntityTypeService;
 import org.hisp.dhis.user.CurrentUserService;
 import org.hisp.dhis.user.User;
+import org.hisp.dhis.webapi.webdomain.EndDateTime;
+import org.hisp.dhis.webapi.webdomain.StartDateTime;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
@@ -101,10 +104,10 @@ public ProgramInstanceQueryParams map(TrackerEnrollmentCriteria criteria)
     params.setProgram(program);
     params.setProgramStatus(criteria.getProgramStatus());
     params.setFollowUp(criteria.getFollowUp());
-    params.setLastUpdated(criteria.getUpdatedAfter());
+    params.setLastUpdated(applyIfNotNull(criteria.getUpdatedAfter(), StartDateTime::toDate));
     params.setLastUpdatedDuration(criteria.getUpdatedWithin());
-    params.setProgramStartDate(criteria.getEnrolledAfter());
-    params.setProgramEndDate(criteria.getEnrolledBefore());
+    params.setProgramStartDate(applyIfNotNull(criteria.getEnrolledAfter(), StartDateTime::toDate));
+    params.setProgramEndDate(applyIfNotNull(criteria.getEnrolledBefore(), EndDateTime::toDate));
     params.setTrackedEntityType(trackedEntityType);
     params.setTrackedEntityInstanceUid(
         Optional.ofNullable(trackedEntity).map(BaseIdentifiableObject::getUid).orElse(null));