From 533f996d4933b4cdd59e51928b436875bbbc813b Mon Sep 17 00:00:00 2001 From: Giuseppe Nespolino Date: Wed, 20 Dec 2023 10:08:19 +0100 Subject: [PATCH] feat: add support for non-dimension params as headers [DHIS2-16191] (#15957) * fix: properly parsed headers for static dimensions [DHIS2-16191] * test: fixes and some e2e test added [DHIS2-16191] * fix: peer review [DHIS2-16191] --- .../analytics/common/params/CommonParams.java | 8 ++ .../params/dimension/DimensionParam.java | 78 +++++++++---- .../processing/CommonQueryRequestMapper.java | 16 ++- .../analytics/tei/query/PeriodCondition.java | 3 + .../dhis/analytics/tei/query/TeiFields.java | 36 +++++- .../querybuilder/DataElementQueryBuilder.java | 14 ++- .../querybuilder/LeftJoinsQueryBuilder.java | 7 +- .../querybuilder/LimitOffsetQueryBuilder.java | 1 + .../querybuilder/MainTableQueryBuilder.java | 1 + .../querybuilder/OrgUnitQueryBuilder.java | 6 +- .../querybuilder/PeriodQueryBuilder.java | 26 +++-- .../ProgramIndicatorQueryBuilder.java | 7 +- .../querybuilder/StatusQueryBuilder.java | 18 +-- .../query/context/sql/SqlQueryBuilder.java | 31 ++++- .../context/sql/SqlQueryBuilderAdaptor.java | 1 + .../context/sql/SqlQueryCreatorService.java | 27 ++++- .../analytics/tei/TrackedEntityQueryTest.java | 110 ++++++++++++++++++ 17 files changed, 325 insertions(+), 65 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/CommonParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/CommonParams.java index b2e6d99c03de..81b8ca5f30fb 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/CommonParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/CommonParams.java @@ -79,6 +79,14 @@ public class CommonParams { */ @Builder.Default private final Set headers = new LinkedHashSet<>(); + /** + * Data structure containing parsed versions of the headers. If present, they will represent the + * columns to be retrieved. Cannot be repeated and should keep ordering, hence a {@link + * LinkedHashSet}. + */ + @Builder.Default + private final Set> parsedHeaders = new LinkedHashSet<>(); + /** The object that groups the paging and sorting parameters. */ @Builder.Default private final AnalyticsPagingParams pagingParams = AnalyticsPagingParams.builder().build(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/dimension/DimensionParam.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/dimension/DimensionParam.java index 5386130114ed..d46ccdeebdd6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/dimension/DimensionParam.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/dimension/DimensionParam.java @@ -30,15 +30,22 @@ import static java.util.Objects.nonNull; import static lombok.AccessLevel.PRIVATE; import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; -import static org.apache.commons.lang3.StringUtils.lowerCase; +import static org.apache.commons.lang3.StringUtils.equalsIgnoreCase; import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamObjectType.ORGANISATION_UNIT; +import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamObjectType.STATIC; import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamObjectType.byForeignType; import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamType.DATE_FILTERS; import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamType.DIMENSIONS; import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamType.FILTERS; +import static org.hisp.dhis.analytics.tei.query.context.TeiStaticField.ORG_UNIT_CODE; +import static org.hisp.dhis.analytics.tei.query.context.TeiStaticField.ORG_UNIT_NAME; +import static org.hisp.dhis.analytics.tei.query.context.TeiStaticField.ORG_UNIT_NAME_HIERARCHY; +import static org.hisp.dhis.analytics.tei.query.context.TeiStaticField.TRACKED_ENTITY_INSTANCE; import static org.hisp.dhis.common.DimensionType.PERIOD; import static org.hisp.dhis.common.QueryOperator.EQ; +import static org.hisp.dhis.common.ValueType.COORDINATE; import static org.hisp.dhis.common.ValueType.DATETIME; +import static org.hisp.dhis.common.ValueType.GEOJSON; import static org.hisp.dhis.common.ValueType.TEXT; import java.util.ArrayList; @@ -51,7 +58,6 @@ import lombok.Data; import lombok.Getter; import lombok.RequiredArgsConstructor; -import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.tei.query.context.TeiHeaderProvider; import org.hisp.dhis.analytics.tei.query.context.TeiStaticField; import org.hisp.dhis.common.DimensionalObject; @@ -132,10 +138,6 @@ public static DimensionParam ofObject( + " instead"); } - public static boolean isStaticDimensionIdentifier(String dimensionIdentifier) { - return StaticDimension.of(dimensionIdentifier).isPresent(); - } - /** * @return true if this DimensionParams has some items on it. */ @@ -212,7 +214,7 @@ public String getUid() { return queryItem.getItem().getUid(); } - return staticDimension.getColumnName(); + return staticDimension.getHeaderName(); } public boolean isPeriodDimension() { @@ -235,20 +237,30 @@ public String getName() { @RequiredArgsConstructor public enum StaticDimension implements TeiHeaderProvider { - OUNAME(TEXT, ORGANISATION_UNIT, TeiStaticField.ORG_UNIT_NAME), - OUCODE(TEXT, ORGANISATION_UNIT, TeiStaticField.ORG_UNIT_CODE), - OUNAMEHIERARCHY(TEXT, ORGANISATION_UNIT, TeiStaticField.ORG_UNIT_NAME_HIERARCHY), + TRACKEDENTITYINSTANCEUID(TEXT, STATIC, TRACKED_ENTITY_INSTANCE), + GEOMETRY(GEOJSON, STATIC, TeiStaticField.GEOMETRY), + LONGITUDE(COORDINATE, STATIC, TeiStaticField.LONGITUDE), + LATITUDE(COORDINATE, STATIC, TeiStaticField.LATITUDE), + OUNAME(TEXT, ORGANISATION_UNIT, ORG_UNIT_NAME), + OUCODE(TEXT, ORGANISATION_UNIT, ORG_UNIT_CODE), + OUNAMEHIERARCHY(TEXT, ORGANISATION_UNIT, ORG_UNIT_NAME_HIERARCHY), ENROLLMENTDATE(DATETIME, DimensionParamObjectType.PERIOD), ENDDATE(DATETIME, DimensionParamObjectType.PERIOD), INCIDENTDATE(DATETIME, DimensionParamObjectType.PERIOD), EXECUTIONDATE(DATETIME, DimensionParamObjectType.PERIOD), - LASTUPDATED(DATETIME, DimensionParamObjectType.PERIOD), - LASTUPDATEDBYDISPLAYNAME(TEXT, DimensionParamObjectType.STATIC), + LASTUPDATED(DATETIME, DimensionParamObjectType.PERIOD, TeiStaticField.LAST_UPDATED), + LASTUPDATEDBYDISPLAYNAME(TEXT, STATIC), CREATED(DATETIME, DimensionParamObjectType.PERIOD), - CREATEDBYDISPLAYNAME(TEXT, DimensionParamObjectType.STATIC), - STOREDBY(TEXT, DimensionParamObjectType.STATIC), - ENROLLMENT_STATUS(TEXT, DimensionParamObjectType.STATIC, null, "enrollmentstatus"), - EVENT_STATUS(TEXT, DimensionParamObjectType.STATIC, null, "status"); + CREATEDBYDISPLAYNAME(TEXT, STATIC), + STOREDBY(TEXT, STATIC), + ENROLLMENT_STATUS(TEXT, STATIC, null, "enrollmentstatus"), + PROGRAM_STATUS( + TEXT, + STATIC, + null, + "enrollmentstatus", + "programstatus"), /* this enum is an alias for ENROLLMENT_STATUS */ + EVENT_STATUS(TEXT, STATIC, null, "status", "eventstatus"); private final ValueType valueType; @@ -258,6 +270,8 @@ public enum StaticDimension implements TeiHeaderProvider { private final TeiStaticField teiStaticField; + @Getter private final String headerName; + StaticDimension(ValueType valueType, DimensionParamObjectType dimensionParamObjectType) { this(valueType, dimensionParamObjectType, null); } @@ -269,11 +283,13 @@ public enum StaticDimension implements TeiHeaderProvider { this.valueType = valueType; // By default, columnName is its own "name" in lowercase. - this.columnName = lowerCase(name()); + this.columnName = normalizedName(); this.dimensionParamObjectType = dimensionParamObjectType; this.teiStaticField = teiStaticField; + + this.headerName = this.columnName; } StaticDimension( @@ -281,15 +297,33 @@ public enum StaticDimension implements TeiHeaderProvider { DimensionParamObjectType dimensionParamObjectType, TeiStaticField teiStaticField, String columnName) { + this(valueType, dimensionParamObjectType, teiStaticField, columnName, columnName); + } + + StaticDimension( + ValueType valueType, + DimensionParamObjectType dimensionParamObjectType, + TeiStaticField teiStaticField, + String columnName, + String headerName) { this.valueType = valueType; - this.columnName = columnName; this.dimensionParamObjectType = dimensionParamObjectType; this.teiStaticField = teiStaticField; + this.columnName = columnName; + this.headerName = headerName; } - static Optional of(String value) { + public String normalizedName() { + return name().toLowerCase().replace("_", ""); + } + + public static Optional of(String value) { return Arrays.stream(StaticDimension.values()) - .filter(sd -> StringUtils.equalsIgnoreCase(sd.name(), value)) + .filter( + sd -> + equalsIgnoreCase(sd.columnName, value) + || equalsIgnoreCase(sd.name(), value) + || equalsIgnoreCase(sd.normalizedName(), value)) .findFirst(); } @@ -307,5 +341,9 @@ public String getFullName() { public ValueType getType() { return valueType; } + + public boolean isTeiStaticField() { + return nonNull(teiStaticField); + } } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/processing/CommonQueryRequestMapper.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/processing/CommonQueryRequestMapper.java index 67f76a96ee2f..f735f5ce3e3f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/processing/CommonQueryRequestMapper.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/processing/CommonQueryRequestMapper.java @@ -30,7 +30,6 @@ import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableList; import static org.hisp.dhis.analytics.EventOutputType.TRACKED_ENTITY_INSTANCE; -import static org.hisp.dhis.analytics.common.params.dimension.DimensionParam.isStaticDimensionIdentifier; import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamType.DATE_FILTERS; import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamType.DIMENSIONS; import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamType.FILTERS; @@ -66,6 +65,7 @@ import org.hisp.dhis.analytics.common.params.CommonParams; import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier; import org.hisp.dhis.analytics.common.params.dimension.DimensionParam; +import org.hisp.dhis.analytics.common.params.dimension.DimensionParam.StaticDimension; import org.hisp.dhis.analytics.common.params.dimension.DimensionParamType; import org.hisp.dhis.analytics.common.params.dimension.StringUid; import org.hisp.dhis.analytics.event.EventDataQueryService; @@ -144,6 +144,7 @@ public CommonParams map(CommonQueryRequest request) { .orderParams(getSortingParams(request, programs, userOrgUnits)) .headers(getHeaders(request)) .dimensionIdentifiers(retrieveDimensionParams(request, programs, userOrgUnits)) + .parsedHeaders(parseHeaders(request, programs, userOrgUnits)) .skipMeta(request.isSkipMeta()) .includeMetadataDetails(request.isIncludeMetadataDetails()) .hierarchyMeta(request.isHierarchyMeta()) @@ -154,6 +155,14 @@ public CommonParams map(CommonQueryRequest request) { .build(); } + private Set> parseHeaders( + CommonQueryRequest request, List programs, List userOrgUnits) { + + return HEADERS.getUidsGetter().apply(request).stream() + .map(header -> toDimensionIdentifier(header, HEADERS, request, programs, userOrgUnits)) + .collect(Collectors.toSet()); + } + /** * Returns the headers specified in the given request. * @@ -362,8 +371,11 @@ public DimensionIdentifier toDimensionIdentifier( dimensionIdentifierConverter.fromString(programs, dimensionId); List items = getDimensionItemsFromParam(dimensionOrFilter); + Optional staticDimension = + StaticDimension.of(dimensionIdentifier.getDimension().getUid()); + // Then we check if it's a static dimension. - if (isStaticDimensionIdentifier(dimensionIdentifier.getDimension().getUid())) { + if (staticDimension.isPresent()) { return parseAsStaticDimension(dimensionParamType, dimensionIdentifier, items); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/PeriodCondition.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/PeriodCondition.java index 81d407e18743..9b5916fda83d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/PeriodCondition.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/PeriodCondition.java @@ -35,6 +35,7 @@ import java.util.Date; import java.util.List; +import java.util.Objects; import org.apache.commons.lang3.tuple.Pair; import org.hisp.dhis.analytics.TimeField; import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier; @@ -67,6 +68,7 @@ private PeriodCondition( dimensionIdentifier.getDimension().getDimensionalObject().getItems().stream() .map(Period.class::cast) .map(Period::getStartDate) + .filter(Objects::nonNull) .reduce(DateUtils::min) .orElse(null); @@ -74,6 +76,7 @@ private PeriodCondition( dimensionIdentifier.getDimension().getDimensionalObject().getItems().stream() .map(Period.class::cast) .map(Period::getEndDate) + .filter(Objects::nonNull) .map(this::nextDay) .reduce(DateUtils::max) .orElse(null); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/TeiFields.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/TeiFields.java index b28776bee483..058cc13ff405 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/TeiFields.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/TeiFields.java @@ -162,7 +162,10 @@ public static Set getGridHeaders(TeiQueryParams teiQueryParams, List .map( field -> findDimensionParamForField( - field, teiQueryParams.getCommonParams().getDimensionIdentifiers())) + field, + Stream.concat( + teiQueryParams.getCommonParams().getDimensionIdentifiers().stream(), + getEligibleParsedHeaders(teiQueryParams)))) .filter(Objects::nonNull) .map( dimIdentifier -> @@ -173,6 +176,33 @@ public static Set getGridHeaders(TeiQueryParams teiQueryParams, List return reorder(headersMap, fields); } + /** + * Since TeiStaticFields are already added to the grid headers, we need to filter them out from + * the list of parsed headers. + * + * @param teiQueryParams the {@link TeiQueryParams}. + * @return a {@link Stream} of {@link DimensionIdentifier}. + */ + private static Stream> getEligibleParsedHeaders( + TeiQueryParams teiQueryParams) { + return teiQueryParams.getCommonParams().getParsedHeaders().stream() + .filter(TeiFields::isEligible); + } + + /** + * Checks if the given {@link DimensionIdentifier} is eligible to be added as a header. It is + * eligible if it is a static dimension and it is either an event or enrollment dimension, and it + * is not a TEI static field (which is already added to the grid headers). + * + * @param parsedHeader the {@link DimensionIdentifier}. + * @return true if it is eligible, false otherwise. + */ + private static boolean isEligible(DimensionIdentifier parsedHeader) { + return parsedHeader.getDimension().isStaticDimension() + && (parsedHeader.isEventDimension() || parsedHeader.isEnrollmentDimension()) + && !parsedHeader.getDimension().getStaticDimension().isTeiStaticField(); + } + /** * Based on the given map of {@link GridHeader}, it will return a set of headers, reordering the * headers respecting the given fields ordering. Only elements inside the given map are returned. @@ -331,8 +361,8 @@ private static GridHeader getCustomGridHeaderForItem( * @throws IllegalStateException if nothing is found. */ private static DimensionIdentifier findDimensionParamForField( - Field field, List> dimensionIdentifiers) { - return dimensionIdentifiers.stream() + Field field, Stream> dimensionIdentifiers) { + return dimensionIdentifiers .filter(di -> di.toString().equals(field.getDimensionIdentifier())) .findFirst() .orElse(null); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/DataElementQueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/DataElementQueryBuilder.java index 590723149aa5..e26d3bf762e9 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/DataElementQueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/DataElementQueryBuilder.java @@ -34,7 +34,6 @@ import java.util.List; import java.util.function.Predicate; -import java.util.stream.Stream; import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.common.params.AnalyticsSortingParams; @@ -56,6 +55,11 @@ @Service @org.springframework.core.annotation.Order(999) public class DataElementQueryBuilder implements SqlQueryBuilder { + + @Getter + private final List>> headerFilters = + List.of(DataElementQueryBuilder::isDataElement); + @Getter private final List>> dimensionFilters = List.of(DataElementQueryBuilder::isDataElement); @@ -67,13 +71,13 @@ public class DataElementQueryBuilder implements SqlQueryBuilder { @Override public RenderableSqlQuery buildSqlQuery( QueryContext queryContext, + List> acceptedHeaders, List> acceptedDimensions, List acceptedSortingParams) { RenderableSqlQuery.RenderableSqlQueryBuilder builder = RenderableSqlQuery.builder(); - Stream.concat( - acceptedDimensions.stream(), - acceptedSortingParams.stream().map(AnalyticsSortingParams::getOrderBy)) + // Select fields are the union of headers, dimensions and sorting params + streamDimensions(acceptedHeaders, acceptedDimensions, acceptedSortingParams) .map( dimensionIdentifier -> Field.ofUnquoted( @@ -85,6 +89,7 @@ public RenderableSqlQuery buildSqlQuery( dimensionIdentifier.toString())) .forEach(builder::selectField); + // Groupable conditions comes from dimensions acceptedDimensions.stream() .filter(SqlQueryBuilders::hasRestrictions) .map( @@ -93,6 +98,7 @@ public RenderableSqlQuery buildSqlQuery( dimId.getGroupId(), DataElementCondition.of(queryContext, dimId))) .forEach(builder::groupableCondition); + // Order clause comes from sorting params acceptedSortingParams.forEach( analyticsSortingParams -> builder.orderClause( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/LeftJoinsQueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/LeftJoinsQueryBuilder.java index 98c79a770634..8fafbfb191f6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/LeftJoinsQueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/LeftJoinsQueryBuilder.java @@ -43,7 +43,6 @@ import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; -import java.util.stream.Stream; import javax.annotation.Nonnull; import org.apache.commons.lang3.tuple.Pair; import org.hisp.dhis.analytics.common.params.AnalyticsSortingParams; @@ -64,6 +63,7 @@ /** This class is responsible for building the SQL statement for the main TEI table. */ @Service public class LeftJoinsQueryBuilder implements SqlQueryBuilder { + @Nonnull @Override public List>> getDimensionFilters() { @@ -86,14 +86,13 @@ public List> getSortingFilters() { @Override public RenderableSqlQuery buildSqlQuery( QueryContext queryContext, + List> headerIdentifiers, List> dimensionIdentifiers, List analyticsSortingParams) { RenderableSqlQueryBuilder renderableSqlQuery = RenderableSqlQuery.builder(); List> allDimensions = - Stream.concat( - dimensionIdentifiers.stream(), - analyticsSortingParams.stream().map(AnalyticsSortingParams::getOrderBy)) + streamDimensions(headerIdentifiers, dimensionIdentifiers, analyticsSortingParams) .filter(dimensionIdentifier -> !isOfType(dimensionIdentifier, PROGRAM_ATTRIBUTE)) .collect(Collectors.toList()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/LimitOffsetQueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/LimitOffsetQueryBuilder.java index 70e37ce3465f..a323a264c16d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/LimitOffsetQueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/LimitOffsetQueryBuilder.java @@ -44,6 +44,7 @@ public class LimitOffsetQueryBuilder implements SqlQueryBuilder { @Override public RenderableSqlQuery buildSqlQuery( QueryContext queryContext, + List> unusedHeaders, List> unusedOne, List unusedTwo) { AnalyticsPagingParams pagingParams = diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/MainTableQueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/MainTableQueryBuilder.java index f9ed81ba3984..bbe65a1a21e9 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/MainTableQueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/MainTableQueryBuilder.java @@ -45,6 +45,7 @@ public class MainTableQueryBuilder implements SqlQueryBuilder { @Override public RenderableSqlQuery buildSqlQuery( QueryContext queryContext, + List> unused, List> unusedOne, List unusedTwo) { return RenderableSqlQuery.builder() diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/OrgUnitQueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/OrgUnitQueryBuilder.java index 1c674552fd2f..487c1ff75246 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/OrgUnitQueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/OrgUnitQueryBuilder.java @@ -33,7 +33,6 @@ import java.util.List; import java.util.function.Predicate; -import java.util.stream.Stream; import lombok.Getter; import org.hisp.dhis.analytics.common.params.AnalyticsSortingParams; import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier; @@ -68,13 +67,12 @@ public class OrgUnitQueryBuilder implements SqlQueryBuilder { @Override public RenderableSqlQuery buildSqlQuery( QueryContext queryContext, + List> acceptedHeaders, List> acceptedDimensions, List acceptedSortingParams) { RenderableSqlQuery.RenderableSqlQueryBuilder builder = RenderableSqlQuery.builder(); - Stream.concat( - acceptedDimensions.stream(), - acceptedSortingParams.stream().map(AnalyticsSortingParams::getOrderBy)) + streamDimensions(acceptedHeaders, acceptedDimensions, acceptedSortingParams) .filter( dimensionIdentifier -> dimensionIdentifier.isEventDimension() diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/PeriodQueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/PeriodQueryBuilder.java index 673e216e8cd7..f668779ae87c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/PeriodQueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/PeriodQueryBuilder.java @@ -33,14 +33,15 @@ import java.util.List; import java.util.Optional; +import java.util.function.Function; import java.util.function.Predicate; -import java.util.stream.Stream; import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.TimeField; import org.hisp.dhis.analytics.common.params.AnalyticsSortingParams; import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier; import org.hisp.dhis.analytics.common.params.dimension.DimensionParam; +import org.hisp.dhis.analytics.common.params.dimension.DimensionParam.StaticDimension; import org.hisp.dhis.analytics.common.query.Field; import org.hisp.dhis.analytics.common.query.GroupableCondition; import org.hisp.dhis.analytics.common.query.IndexedOrder; @@ -49,6 +50,7 @@ import org.hisp.dhis.analytics.tei.query.context.sql.QueryContext; import org.hisp.dhis.analytics.tei.query.context.sql.RenderableSqlQuery; import org.hisp.dhis.analytics.tei.query.context.sql.SqlQueryBuilderAdaptor; +import org.hisp.dhis.common.DimensionalObject; import org.hisp.dhis.period.Period; import org.springframework.stereotype.Service; @@ -73,21 +75,21 @@ public class PeriodQueryBuilder extends SqlQueryBuilderAdaptor { @Override public RenderableSqlQuery buildSqlQuery( QueryContext ctx, + List> acceptedHeaders, List> acceptedDimensions, List acceptedSortingParams) { RenderableSqlQuery.RenderableSqlQueryBuilder builder = RenderableSqlQuery.builder(); - Stream.concat( - acceptedDimensions.stream(), - acceptedSortingParams.stream().map(AnalyticsSortingParams::getOrderBy)) + streamDimensions(acceptedHeaders, acceptedDimensions, acceptedSortingParams) .map( dimensionIdentifier -> { - String field = getTimeField(dimensionIdentifier); + String field = getTimeField(dimensionIdentifier, StaticDimension::getColumnName); + String alias = getTimeField(dimensionIdentifier, StaticDimension::getHeaderName); String prefix = getPrefix(dimensionIdentifier, false); return Field.ofUnquoted( - doubleQuote(prefix), () -> field, prefix + DIMENSION_SEPARATOR + field); + doubleQuote(prefix), () -> field, prefix + DIMENSION_SEPARATOR + alias); }) .forEach(builder::selectField); @@ -99,7 +101,7 @@ public RenderableSqlQuery buildSqlQuery( acceptedSortingParams.forEach( sortingParam -> { DimensionIdentifier dimensionIdentifier = sortingParam.getOrderBy(); - String fieldName = getTimeField(dimensionIdentifier); + String fieldName = getTimeField(dimensionIdentifier, StaticDimension::getColumnName); Field field = Field.ofUnquoted( @@ -112,15 +114,21 @@ public RenderableSqlQuery buildSqlQuery( return builder.build(); } - private static String getTimeField(DimensionIdentifier dimensionIdentifier) { + private static String getTimeField( + DimensionIdentifier dimensionIdentifier, + Function staticDimensionNameExtractor) { return Optional.of(dimensionIdentifier) .map(DimensionIdentifier::getDimension) .map(DimensionParam::getDimensionalObject) + .filter(DimensionalObject::hasItems) .map(d -> d.getItems().get(0)) .map(Period.class::cast) .map(Period::getDateField) .map(TimeField::valueOf) .map(TimeField::getField) - .orElseGet(() -> dimensionIdentifier.getDimension().getStaticDimension().getColumnName()); + .orElseGet( + () -> + staticDimensionNameExtractor.apply( + dimensionIdentifier.getDimension().getStaticDimension())); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/ProgramIndicatorQueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/ProgramIndicatorQueryBuilder.java index 9c444d450d99..41edff6f5ce7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/ProgramIndicatorQueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/ProgramIndicatorQueryBuilder.java @@ -42,7 +42,6 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; -import java.util.stream.Stream; import lombok.Getter; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.StringUtils; @@ -93,12 +92,12 @@ public class ProgramIndicatorQueryBuilder implements SqlQueryBuilder { @Override public RenderableSqlQuery buildSqlQuery( QueryContext queryContext, + List> acceptedHeaders, List> acceptedDimensions, List acceptedSortingParams) { + List allDimensionIdentifiers = - Stream.concat( - acceptedDimensions.stream(), - acceptedSortingParams.stream().map(AnalyticsSortingParams::getOrderBy)) + streamDimensions(acceptedHeaders, acceptedDimensions, acceptedSortingParams) .map(ProgramIndicatorDimensionIdentifier::of) .distinct() .collect(toList()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/StatusQueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/StatusQueryBuilder.java index 5c4918d8e524..ec753bdb4f0d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/StatusQueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/StatusQueryBuilder.java @@ -31,18 +31,19 @@ import static org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifierHelper.getPrefix; import static org.hisp.dhis.analytics.common.params.dimension.DimensionParam.StaticDimension.ENROLLMENT_STATUS; import static org.hisp.dhis.analytics.common.params.dimension.DimensionParam.StaticDimension.EVENT_STATUS; +import static org.hisp.dhis.analytics.common.params.dimension.DimensionParam.StaticDimension.PROGRAM_STATUS; import static org.hisp.dhis.commons.util.TextUtils.doubleQuote; import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.function.Predicate; -import java.util.stream.Stream; import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.common.params.AnalyticsSortingParams; import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier; import org.hisp.dhis.analytics.common.params.dimension.DimensionParam; +import org.hisp.dhis.analytics.common.params.dimension.DimensionParam.StaticDimension; import org.hisp.dhis.analytics.common.query.Field; import org.hisp.dhis.analytics.common.query.GroupableCondition; import org.hisp.dhis.analytics.common.query.IndexedOrder; @@ -57,7 +58,7 @@ public class StatusQueryBuilder extends SqlQueryBuilderAdaptor { /** The supported status dimensions. */ private static final Collection SUPPORTED_STATUS_DIMENSIONS = - List.of(ENROLLMENT_STATUS, EVENT_STATUS); + List.of(ENROLLMENT_STATUS, PROGRAM_STATUS, EVENT_STATUS); @Getter private final List>> dimensionFilters = @@ -79,21 +80,22 @@ private static boolean isStatusDimension( @Override public RenderableSqlQuery buildSqlQuery( QueryContext ctx, + List> acceptedHeaders, List> acceptedDimensions, List acceptedSortingParams) { RenderableSqlQuery.RenderableSqlQueryBuilder builder = RenderableSqlQuery.builder(); - Stream.concat( - acceptedDimensions.stream(), - acceptedSortingParams.stream().map(AnalyticsSortingParams::getOrderBy)) + streamDimensions(acceptedHeaders, acceptedDimensions, acceptedSortingParams) .map( dimensionIdentifier -> { - String field = - dimensionIdentifier.getDimension().getStaticDimension().getColumnName(); + StaticDimension staticDimension = + dimensionIdentifier.getDimension().getStaticDimension(); String prefix = getPrefix(dimensionIdentifier, false); return Field.ofUnquoted( - doubleQuote(prefix), () -> field, prefix + DIMENSION_SEPARATOR + field); + doubleQuote(prefix), + staticDimension::getColumnName, + prefix + DIMENSION_SEPARATOR + staticDimension.getHeaderName()); }) .forEach(builder::selectField); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/SqlQueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/SqlQueryBuilder.java index 0f11c8df9bb6..c8f14d1885df 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/SqlQueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/SqlQueryBuilder.java @@ -28,7 +28,9 @@ package org.hisp.dhis.analytics.tei.query.context.sql; import java.util.List; +import java.util.function.Function; import java.util.function.Predicate; +import java.util.stream.Stream; import javax.annotation.Nonnull; import org.hisp.dhis.analytics.common.params.AnalyticsSortingParams; import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier; @@ -41,14 +43,28 @@ public interface SqlQueryBuilder { * Builds a {@link RenderableSqlQuery} based on the given arguments. * * @param queryContext the {@link QueryContext}. - * @param dimensions the list of {@link DimensionIdentifier}. - * @param sortingParams the list of {@link AnalyticsSortingParams}. + * @param acceptedHeaders the list of {@link DimensionIdentifier}. + * @param acceptedDimensions the list of {@link DimensionIdentifier}. + * @param acceptedSortingParams the list of {@link AnalyticsSortingParams}. */ RenderableSqlQuery buildSqlQuery( @Nonnull QueryContext queryContext, + @Nonnull List> acceptedHeaders, @Nonnull List> acceptedDimensions, @Nonnull List acceptedSortingParams); + /** + * Provides the list of {@link Predicate} functions for {@link DimensionIdentifier} (headers). + * They act as filters and are used to build the final {@link RenderableSqlQuery} query. By + * default, it returns the same as {@link #getDimensionFilters()}. + * + * @return the list of filter dimensions or empty. + */ + @Nonnull + default List>> getHeaderFilters() { + return getDimensionFilters(); + } + /** * Provides the list of {@link Predicate} functions for {@link DimensionIdentifier}. They act as * filters and are used to build the final {@link RenderableSqlQuery} query. @@ -74,4 +90,15 @@ default List> getSortingFilters() { default boolean alwaysRun() { return false; } + + default Stream> streamDimensions( + List> headers, + List> dimensions, + List sortingParams) { + return Stream.of( + headers.stream(), + dimensions.stream(), + sortingParams.stream().map(AnalyticsSortingParams::getOrderBy)) + .flatMap(Function.identity()); + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/SqlQueryBuilderAdaptor.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/SqlQueryBuilderAdaptor.java index 0c56101b1637..7df1e600931d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/SqlQueryBuilderAdaptor.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/SqlQueryBuilderAdaptor.java @@ -45,6 +45,7 @@ public abstract class SqlQueryBuilderAdaptor implements SqlQueryBuilder { @Override public RenderableSqlQuery buildSqlQuery( QueryContext queryContext, + List> acceptedHeaders, List> acceptedDimensions, List acceptedSortingParams) { RenderableSqlQuery.RenderableSqlQueryBuilder builder = RenderableSqlQuery.builder(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/SqlQueryCreatorService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/SqlQueryCreatorService.java index baff299da0d6..0295d32cbbe8 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/SqlQueryCreatorService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/SqlQueryCreatorService.java @@ -27,11 +27,10 @@ */ package org.hisp.dhis.analytics.tei.query.context.sql; -import static java.util.stream.Collectors.toList; - import java.util.List; import java.util.function.Predicate; import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.common.params.AnalyticsSortingParams; import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier; import org.hisp.dhis.analytics.common.params.dimension.DimensionParam; @@ -61,26 +60,44 @@ public SqlQueryCreator getSqlQueryCreator(TeiQueryParams teiQueryParams) { List> acceptedDimensions = teiQueryParams.getCommonParams().getDimensionIdentifiers().stream() .filter(provider.getDimensionFilters().stream().reduce(x -> true, Predicate::and)) - .collect(toList()); + .toList(); List acceptedSortingParams = teiQueryParams.getCommonParams().getOrderParams().stream() .filter(provider.getSortingFilters().stream().reduce(x -> true, Predicate::and)) - .collect(toList()); + .toList(); + + List> acceptedHeaders = + teiQueryParams.getCommonParams().getParsedHeaders().stream() + .filter(provider.getHeaderFilters().stream().reduce(x -> true, Predicate::and)) + .filter(parsedHeader -> notContains(acceptedDimensions, parsedHeader)) + .toList(); if (provider.alwaysRun() + || !CollectionUtils.isEmpty(acceptedHeaders) || !CollectionUtils.isEmpty(acceptedDimensions) || !CollectionUtils.isEmpty(acceptedSortingParams)) { renderableSqlQuery = mergeQueries( renderableSqlQuery, - provider.buildSqlQuery(queryContext, acceptedDimensions, acceptedSortingParams)); + provider.buildSqlQuery( + queryContext, acceptedHeaders, acceptedDimensions, acceptedSortingParams)); } } return SqlQueryCreator.of(queryContext, renderableSqlQuery); } + private boolean notContains( + List> acceptedDimensions, + DimensionIdentifier parsedHeader) { + return acceptedDimensions.stream() + .noneMatch( + dimensionIdentifier -> + StringUtils.equalsIgnoreCase( + dimensionIdentifier.toString(), parsedHeader.toString())); + } + private RenderableSqlQuery mergeQueries( RenderableSqlQuery initial, RenderableSqlQuery contribution) { RenderableSqlQuery.RenderableSqlQueryBuilder sqlQueryContextBuilder = initial.toBuilder(); diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/tei/TrackedEntityQueryTest.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/tei/TrackedEntityQueryTest.java index bfdc147001d6..8141515f9e98 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/tei/TrackedEntityQueryTest.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/tei/TrackedEntityQueryTest.java @@ -39,6 +39,7 @@ import java.util.List; import java.util.Map; +import java.util.stream.IntStream; import net.minidev.json.JSONObject; import org.hisp.dhis.AnalyticsApiTest; import org.hisp.dhis.actions.analytics.AnalyticsTeiActions; @@ -2779,4 +2780,113 @@ public void queryProgramIndicator() { "", "2994.5")); } + + @Test + public void headerParamProgramStatus() { + // Given + QueryParamsBuilder params = + new QueryParamsBuilder() + .add("programStatus=IpHINAT79UW.ACTIVE") + .add("headers=IpHINAT79UW.programstatus") + .add("lastUpdated=LAST_YEAR") + .add("relativePeriodDate=2016-01-01"); + + // When + ApiResponse response = analyticsTeiActions.query().get("nEenWmSyUEp", JSON, JSON, params); + + // Then + response + .validate() + .statusCode(200) + .body("rows", hasSize(equalTo(50))) + .body("height", equalTo(50)) + .body("width", equalTo(1)) + .body("headerWidth", equalTo(1)) + .body("headers", hasSize(equalTo(1))); + + validateHeader( + response, + 0, + "IpHINAT79UW.programstatus", + "Enrollment PROGRAM_STATUS", + "TEXT", + "java.lang.String", + false, + true); + + IntStream.range(0, 50).forEach(i -> validateRow(response, i, List.of("ACTIVE"))); + } + + @Test + public void headerParamEnrollmentStatus() { + // Given + QueryParamsBuilder params = + new QueryParamsBuilder() + .add("enrollmentStatus=IpHINAT79UW.ACTIVE") + .add("headers=IpHINAT79UW.enrollmentstatus") + .add("lastUpdated=LAST_YEAR") + .add("relativePeriodDate=2016-01-01"); + + // When + ApiResponse response = analyticsTeiActions.query().get("nEenWmSyUEp", JSON, JSON, params); + + // Then + response + .validate() + .statusCode(200) + .body("rows", hasSize(equalTo(50))) + .body("height", equalTo(50)) + .body("width", equalTo(1)) + .body("headerWidth", equalTo(1)) + .body("headers", hasSize(equalTo(1))); + + validateHeader( + response, + 0, + "IpHINAT79UW.enrollmentstatus", + "Enrollment ENROLLMENT_STATUS", + "TEXT", + "java.lang.String", + false, + true); + + IntStream.range(0, 50).forEach(i -> validateRow(response, i, List.of("ACTIVE"))); + } + + @Test + public void headerParamIncidentDate() { + // Given + QueryParamsBuilder params = + new QueryParamsBuilder() + .add("incidentDate=IpHINAT79UW.2022-01-01") + .add("headers=IpHINAT79UW.incidentdate") + .add("asc=IpHINAT79UW.incidentdate") + .add("lastUpdated=LAST_YEAR") + .add("relativePeriodDate=2016-01-01"); + + // When + ApiResponse response = analyticsTeiActions.query().get("nEenWmSyUEp", JSON, JSON, params); + + // Then + response + .validate() + .statusCode(200) + .body("rows", hasSize(equalTo(50))) + .body("height", equalTo(50)) + .body("width", equalTo(1)) + .body("headerWidth", equalTo(1)) + .body("headers", hasSize(equalTo(1))); + + validateHeader( + response, + 0, + "IpHINAT79UW.incidentdate", + "Enrollment INCIDENTDATE", + "DATETIME", + "java.time.LocalDateTime", + false, + true); + + validateRow(response, 0, List.of("2022-01-01 12:05:00.0")); + } }