feat: add support for non-dimension params as headers [DHIS2-16191] (#…

…15957) * fix: properly parsed headers for static dimensions [DHIS2-16191] * test: fixes and some e2e test added [DHIS2-16191] * fix: peer review [DHIS2-16191]
dhis2 · Dec 20, 2023 · 533f996 · 533f996
1 parent 8c031f4
commit 533f996
Show file tree

Hide file tree

Showing 17 changed files with 325 additions and 65 deletions.
diff --git a/...s-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/CommonParams.java b/...s-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/CommonParams.java
@@ -79,6 +79,14 @@ public class CommonParams {
    */
   @Builder.Default private final Set<String> headers = new LinkedHashSet<>();
 
+  /**
+   * Data structure containing parsed versions of the headers. If present, they will represent the
+   * columns to be retrieved. Cannot be repeated and should keep ordering, hence a {@link
+   * LinkedHashSet}.
+   */
+  @Builder.Default
+  private final Set<DimensionIdentifier<DimensionParam>> parsedHeaders = new LinkedHashSet<>();
+
   /** The object that groups the paging and sorting parameters. */
   @Builder.Default
   private final AnalyticsPagingParams pagingParams = AnalyticsPagingParams.builder().build();

diff --git a/...alytics/src/main/java/org/hisp/dhis/analytics/common/params/dimension/DimensionParam.java b/...alytics/src/main/java/org/hisp/dhis/analytics/common/params/dimension/DimensionParam.java
@@ -30,15 +30,22 @@
 import static java.util.Objects.nonNull;
 import static lombok.AccessLevel.PRIVATE;
 import static org.apache.commons.collections4.CollectionUtils.isNotEmpty;
-import static org.apache.commons.lang3.StringUtils.lowerCase;
+import static org.apache.commons.lang3.StringUtils.equalsIgnoreCase;
 import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamObjectType.ORGANISATION_UNIT;
+import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamObjectType.STATIC;
 import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamObjectType.byForeignType;
 import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamType.DATE_FILTERS;
 import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamType.DIMENSIONS;
 import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamType.FILTERS;
+import static org.hisp.dhis.analytics.tei.query.context.TeiStaticField.ORG_UNIT_CODE;
+import static org.hisp.dhis.analytics.tei.query.context.TeiStaticField.ORG_UNIT_NAME;
+import static org.hisp.dhis.analytics.tei.query.context.TeiStaticField.ORG_UNIT_NAME_HIERARCHY;
+import static org.hisp.dhis.analytics.tei.query.context.TeiStaticField.TRACKED_ENTITY_INSTANCE;
 import static org.hisp.dhis.common.DimensionType.PERIOD;
 import static org.hisp.dhis.common.QueryOperator.EQ;
+import static org.hisp.dhis.common.ValueType.COORDINATE;
 import static org.hisp.dhis.common.ValueType.DATETIME;
+import static org.hisp.dhis.common.ValueType.GEOJSON;
 import static org.hisp.dhis.common.ValueType.TEXT;
 
 import java.util.ArrayList;
@@ -51,7 +58,6 @@
 import lombok.Data;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
-import org.apache.commons.lang3.StringUtils;
 import org.hisp.dhis.analytics.tei.query.context.TeiHeaderProvider;
 import org.hisp.dhis.analytics.tei.query.context.TeiStaticField;
 import org.hisp.dhis.common.DimensionalObject;
@@ -132,10 +138,6 @@ public static DimensionParam ofObject(
             + " instead");
   }
 
-  public static boolean isStaticDimensionIdentifier(String dimensionIdentifier) {
-    return StaticDimension.of(dimensionIdentifier).isPresent();
-  }
-
   /**
    * @return true if this DimensionParams has some items on it.
    */
@@ -212,7 +214,7 @@ public String getUid() {
       return queryItem.getItem().getUid();
     }
 
-    return staticDimension.getColumnName();
+    return staticDimension.getHeaderName();
   }
 
   public boolean isPeriodDimension() {
@@ -235,20 +237,30 @@ public String getName() {
 
   @RequiredArgsConstructor
   public enum StaticDimension implements TeiHeaderProvider {
-    OUNAME(TEXT, ORGANISATION_UNIT, TeiStaticField.ORG_UNIT_NAME),
-    OUCODE(TEXT, ORGANISATION_UNIT, TeiStaticField.ORG_UNIT_CODE),
-    OUNAMEHIERARCHY(TEXT, ORGANISATION_UNIT, TeiStaticField.ORG_UNIT_NAME_HIERARCHY),
+    TRACKEDENTITYINSTANCEUID(TEXT, STATIC, TRACKED_ENTITY_INSTANCE),
+    GEOMETRY(GEOJSON, STATIC, TeiStaticField.GEOMETRY),
+    LONGITUDE(COORDINATE, STATIC, TeiStaticField.LONGITUDE),
+    LATITUDE(COORDINATE, STATIC, TeiStaticField.LATITUDE),
+    OUNAME(TEXT, ORGANISATION_UNIT, ORG_UNIT_NAME),
+    OUCODE(TEXT, ORGANISATION_UNIT, ORG_UNIT_CODE),
+    OUNAMEHIERARCHY(TEXT, ORGANISATION_UNIT, ORG_UNIT_NAME_HIERARCHY),
     ENROLLMENTDATE(DATETIME, DimensionParamObjectType.PERIOD),
     ENDDATE(DATETIME, DimensionParamObjectType.PERIOD),
     INCIDENTDATE(DATETIME, DimensionParamObjectType.PERIOD),
     EXECUTIONDATE(DATETIME, DimensionParamObjectType.PERIOD),
-    LASTUPDATED(DATETIME, DimensionParamObjectType.PERIOD),
-    LASTUPDATEDBYDISPLAYNAME(TEXT, DimensionParamObjectType.STATIC),
+    LASTUPDATED(DATETIME, DimensionParamObjectType.PERIOD, TeiStaticField.LAST_UPDATED),
+    LASTUPDATEDBYDISPLAYNAME(TEXT, STATIC),
     CREATED(DATETIME, DimensionParamObjectType.PERIOD),
-    CREATEDBYDISPLAYNAME(TEXT, DimensionParamObjectType.STATIC),
-    STOREDBY(TEXT, DimensionParamObjectType.STATIC),
-    ENROLLMENT_STATUS(TEXT, DimensionParamObjectType.STATIC, null, "enrollmentstatus"),
-    EVENT_STATUS(TEXT, DimensionParamObjectType.STATIC, null, "status");
+    CREATEDBYDISPLAYNAME(TEXT, STATIC),
+    STOREDBY(TEXT, STATIC),
+    ENROLLMENT_STATUS(TEXT, STATIC, null, "enrollmentstatus"),
+    PROGRAM_STATUS(
+        TEXT,
+        STATIC,
+        null,
+        "enrollmentstatus",
+        "programstatus"), /* this enum is an alias for ENROLLMENT_STATUS */
+    EVENT_STATUS(TEXT, STATIC, null, "status", "eventstatus");
 
     private final ValueType valueType;
 
@@ -258,6 +270,8 @@ public enum StaticDimension implements TeiHeaderProvider {
 
     private final TeiStaticField teiStaticField;
 
+    @Getter private final String headerName;
+
     StaticDimension(ValueType valueType, DimensionParamObjectType dimensionParamObjectType) {
       this(valueType, dimensionParamObjectType, null);
     }
@@ -269,27 +283,47 @@ public enum StaticDimension implements TeiHeaderProvider {
       this.valueType = valueType;
 
       // By default, columnName is its own "name" in lowercase.
-      this.columnName = lowerCase(name());
+      this.columnName = normalizedName();
 
       this.dimensionParamObjectType = dimensionParamObjectType;
 
       this.teiStaticField = teiStaticField;
+
+      this.headerName = this.columnName;
     }
 
     StaticDimension(
         ValueType valueType,
         DimensionParamObjectType dimensionParamObjectType,
         TeiStaticField teiStaticField,
         String columnName) {
+      this(valueType, dimensionParamObjectType, teiStaticField, columnName, columnName);
+    }
+
+    StaticDimension(
+        ValueType valueType,
+        DimensionParamObjectType dimensionParamObjectType,
+        TeiStaticField teiStaticField,
+        String columnName,
+        String headerName) {
       this.valueType = valueType;
-      this.columnName = columnName;
       this.dimensionParamObjectType = dimensionParamObjectType;
       this.teiStaticField = teiStaticField;
+      this.columnName = columnName;
+      this.headerName = headerName;
     }
 
-    static Optional<StaticDimension> of(String value) {
+    public String normalizedName() {
+      return name().toLowerCase().replace("_", "");
+    }
+
+    public static Optional<StaticDimension> of(String value) {
       return Arrays.stream(StaticDimension.values())
-          .filter(sd -> StringUtils.equalsIgnoreCase(sd.name(), value))
+          .filter(
+              sd ->
+                  equalsIgnoreCase(sd.columnName, value)
+                      || equalsIgnoreCase(sd.name(), value)
+                      || equalsIgnoreCase(sd.normalizedName(), value))
           .findFirst();
     }
 
@@ -307,5 +341,9 @@ public String getFullName() {
     public ValueType getType() {
       return valueType;
     }
+
+    public boolean isTeiStaticField() {
+      return nonNull(teiStaticField);
+    }
   }
 }
diff --git a/...ics/src/main/java/org/hisp/dhis/analytics/common/processing/CommonQueryRequestMapper.java b/...ics/src/main/java/org/hisp/dhis/analytics/common/processing/CommonQueryRequestMapper.java
@@ -30,7 +30,6 @@
 import static java.util.Collections.emptySet;
 import static java.util.Collections.unmodifiableList;
 import static org.hisp.dhis.analytics.EventOutputType.TRACKED_ENTITY_INSTANCE;
-import static org.hisp.dhis.analytics.common.params.dimension.DimensionParam.isStaticDimensionIdentifier;
 import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamType.DATE_FILTERS;
 import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamType.DIMENSIONS;
 import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamType.FILTERS;
@@ -66,6 +65,7 @@
 import org.hisp.dhis.analytics.common.params.CommonParams;
 import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier;
 import org.hisp.dhis.analytics.common.params.dimension.DimensionParam;
+import org.hisp.dhis.analytics.common.params.dimension.DimensionParam.StaticDimension;
 import org.hisp.dhis.analytics.common.params.dimension.DimensionParamType;
 import org.hisp.dhis.analytics.common.params.dimension.StringUid;
 import org.hisp.dhis.analytics.event.EventDataQueryService;
@@ -144,6 +144,7 @@ public CommonParams map(CommonQueryRequest request) {
         .orderParams(getSortingParams(request, programs, userOrgUnits))
         .headers(getHeaders(request))
         .dimensionIdentifiers(retrieveDimensionParams(request, programs, userOrgUnits))
+        .parsedHeaders(parseHeaders(request, programs, userOrgUnits))
         .skipMeta(request.isSkipMeta())
         .includeMetadataDetails(request.isIncludeMetadataDetails())
         .hierarchyMeta(request.isHierarchyMeta())
@@ -154,6 +155,14 @@ public CommonParams map(CommonQueryRequest request) {
         .build();
   }
 
+  private Set<DimensionIdentifier<DimensionParam>> parseHeaders(
+      CommonQueryRequest request, List<Program> programs, List<OrganisationUnit> userOrgUnits) {
+
+    return HEADERS.getUidsGetter().apply(request).stream()
+        .map(header -> toDimensionIdentifier(header, HEADERS, request, programs, userOrgUnits))
+        .collect(Collectors.toSet());
+  }
+
   /**
    * Returns the headers specified in the given request.
    *
@@ -362,8 +371,11 @@ public DimensionIdentifier<DimensionParam> toDimensionIdentifier(
         dimensionIdentifierConverter.fromString(programs, dimensionId);
     List<String> items = getDimensionItemsFromParam(dimensionOrFilter);
 
+    Optional<StaticDimension> staticDimension =
+        StaticDimension.of(dimensionIdentifier.getDimension().getUid());
+
     // Then we check if it's a static dimension.
-    if (isStaticDimensionIdentifier(dimensionIdentifier.getDimension().getUid())) {
+    if (staticDimension.isPresent()) {
       return parseAsStaticDimension(dimensionParamType, dimensionIdentifier, items);
     }
 

diff --git a/...is-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/PeriodCondition.java b/...is-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/PeriodCondition.java
@@ -35,6 +35,7 @@
 
 import java.util.Date;
 import java.util.List;
+import java.util.Objects;
 import org.apache.commons.lang3.tuple.Pair;
 import org.hisp.dhis.analytics.TimeField;
 import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier;
@@ -67,13 +68,15 @@ private PeriodCondition(
         dimensionIdentifier.getDimension().getDimensionalObject().getItems().stream()
             .map(Period.class::cast)
             .map(Period::getStartDate)
+            .filter(Objects::nonNull)
             .reduce(DateUtils::min)
             .orElse(null);
 
     Date maxDate =
         dimensionIdentifier.getDimension().getDimensionalObject().getItems().stream()
             .map(Period.class::cast)
             .map(Period::getEndDate)
+            .filter(Objects::nonNull)
             .map(this::nextDay)
             .reduce(DateUtils::max)
             .orElse(null);

diff --git a/...ces/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/TeiFields.java b/...ces/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/TeiFields.java
@@ -162,7 +162,10 @@ public static Set<GridHeader> getGridHeaders(TeiQueryParams teiQueryParams, List
         .map(
             field ->
                 findDimensionParamForField(
-                    field, teiQueryParams.getCommonParams().getDimensionIdentifiers()))
+                    field,
+                    Stream.concat(
+                        teiQueryParams.getCommonParams().getDimensionIdentifiers().stream(),
+                        getEligibleParsedHeaders(teiQueryParams))))
         .filter(Objects::nonNull)
         .map(
             dimIdentifier ->
@@ -173,6 +176,33 @@ public static Set<GridHeader> getGridHeaders(TeiQueryParams teiQueryParams, List
     return reorder(headersMap, fields);
   }
 
+  /**
+   * Since TeiStaticFields are already added to the grid headers, we need to filter them out from
+   * the list of parsed headers.
+   *
+   * @param teiQueryParams the {@link TeiQueryParams}.
+   * @return a {@link Stream} of {@link DimensionIdentifier}.
+   */
+  private static Stream<DimensionIdentifier<DimensionParam>> getEligibleParsedHeaders(
+      TeiQueryParams teiQueryParams) {
+    return teiQueryParams.getCommonParams().getParsedHeaders().stream()
+        .filter(TeiFields::isEligible);
+  }
+
+  /**
+   * Checks if the given {@link DimensionIdentifier} is eligible to be added as a header. It is
+   * eligible if it is a static dimension and it is either an event or enrollment dimension, and it
+   * is not a TEI static field (which is already added to the grid headers).
+   *
+   * @param parsedHeader the {@link DimensionIdentifier}.
+   * @return true if it is eligible, false otherwise.
+   */
+  private static boolean isEligible(DimensionIdentifier<DimensionParam> parsedHeader) {
+    return parsedHeader.getDimension().isStaticDimension()
+        && (parsedHeader.isEventDimension() || parsedHeader.isEnrollmentDimension())
+        && !parsedHeader.getDimension().getStaticDimension().isTeiStaticField();
+  }
+
   /**
    * Based on the given map of {@link GridHeader}, it will return a set of headers, reordering the
    * headers respecting the given fields ordering. Only elements inside the given map are returned.
@@ -331,8 +361,8 @@ private static GridHeader getCustomGridHeaderForItem(
    * @throws IllegalStateException if nothing is found.
    */
   private static DimensionIdentifier<DimensionParam> findDimensionParamForField(
-      Field field, List<DimensionIdentifier<DimensionParam>> dimensionIdentifiers) {
-    return dimensionIdentifiers.stream()
+      Field field, Stream<DimensionIdentifier<DimensionParam>> dimensionIdentifiers) {
+    return dimensionIdentifiers
         .filter(di -> di.toString().equals(field.getDimensionIdentifier()))
         .findFirst()
         .orElse(null);

diff --git a/.../java/org/hisp/dhis/analytics/tei/query/context/querybuilder/DataElementQueryBuilder.java b/.../java/org/hisp/dhis/analytics/tei/query/context/querybuilder/DataElementQueryBuilder.java
@@ -34,7 +34,6 @@
 
 import java.util.List;
 import java.util.function.Predicate;
-import java.util.stream.Stream;
 import lombok.Getter;
 import org.apache.commons.lang3.StringUtils;
 import org.hisp.dhis.analytics.common.params.AnalyticsSortingParams;
@@ -56,6 +55,11 @@
 @Service
 @org.springframework.core.annotation.Order(999)
 public class DataElementQueryBuilder implements SqlQueryBuilder {
+
+  @Getter
+  private final List<Predicate<DimensionIdentifier<DimensionParam>>> headerFilters =
+      List.of(DataElementQueryBuilder::isDataElement);
+
   @Getter
   private final List<Predicate<DimensionIdentifier<DimensionParam>>> dimensionFilters =
       List.of(DataElementQueryBuilder::isDataElement);
@@ -67,13 +71,13 @@ public class DataElementQueryBuilder implements SqlQueryBuilder {
   @Override
   public RenderableSqlQuery buildSqlQuery(
       QueryContext queryContext,
+      List<DimensionIdentifier<DimensionParam>> acceptedHeaders,
       List<DimensionIdentifier<DimensionParam>> acceptedDimensions,
       List<AnalyticsSortingParams> acceptedSortingParams) {
     RenderableSqlQuery.RenderableSqlQueryBuilder builder = RenderableSqlQuery.builder();
 
-    Stream.concat(
-            acceptedDimensions.stream(),
-            acceptedSortingParams.stream().map(AnalyticsSortingParams::getOrderBy))
+    // Select fields are the union of headers, dimensions and sorting params
+    streamDimensions(acceptedHeaders, acceptedDimensions, acceptedSortingParams)
         .map(
             dimensionIdentifier ->
                 Field.ofUnquoted(
@@ -85,6 +89,7 @@ public RenderableSqlQuery buildSqlQuery(
                     dimensionIdentifier.toString()))
         .forEach(builder::selectField);
 
+    // Groupable conditions comes from dimensions
     acceptedDimensions.stream()
         .filter(SqlQueryBuilders::hasRestrictions)
         .map(
@@ -93,6 +98,7 @@ public RenderableSqlQuery buildSqlQuery(
                     dimId.getGroupId(), DataElementCondition.of(queryContext, dimId)))
         .forEach(builder::groupableCondition);
 
+    // Order clause comes from sorting params
     acceptedSortingParams.forEach(
         analyticsSortingParams ->
             builder.orderClause(

diff --git a/...in/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/LeftJoinsQueryBuilder.java b/...in/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/LeftJoinsQueryBuilder.java
@@ -43,7 +43,6 @@
 import java.util.Set;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 import javax.annotation.Nonnull;
 import org.apache.commons.lang3.tuple.Pair;
 import org.hisp.dhis.analytics.common.params.AnalyticsSortingParams;
@@ -64,6 +63,7 @@
 /** This class is responsible for building the SQL statement for the main TEI table. */
 @Service
 public class LeftJoinsQueryBuilder implements SqlQueryBuilder {
+
   @Nonnull
   @Override
   public List<Predicate<DimensionIdentifier<DimensionParam>>> getDimensionFilters() {
@@ -86,14 +86,13 @@ public List<Predicate<AnalyticsSortingParams>> getSortingFilters() {
   @Override
   public RenderableSqlQuery buildSqlQuery(
       QueryContext queryContext,
+      List<DimensionIdentifier<DimensionParam>> headerIdentifiers,
       List<DimensionIdentifier<DimensionParam>> dimensionIdentifiers,
       List<AnalyticsSortingParams> analyticsSortingParams) {
     RenderableSqlQueryBuilder renderableSqlQuery = RenderableSqlQuery.builder();
 
     List<DimensionIdentifier<DimensionParam>> allDimensions =
-        Stream.concat(
-                dimensionIdentifiers.stream(),
-                analyticsSortingParams.stream().map(AnalyticsSortingParams::getOrderBy))
+        streamDimensions(headerIdentifiers, dimensionIdentifiers, analyticsSortingParams)
             .filter(dimensionIdentifier -> !isOfType(dimensionIdentifier, PROGRAM_ATTRIBUTE))
             .collect(Collectors.toList());