diff --git a/.github/workflows/deploy-instance.yml b/.github/workflows/deploy-instance.yml new file mode 100644 index 000000000000..cbb49a3b2e82 --- /dev/null +++ b/.github/workflows/deploy-instance.yml @@ -0,0 +1,57 @@ +name: Deploy instance + +on: + pull_request: + types: [opened, reopened, labeled, synchronize] + +# Cancel previous runs of the same workflow and PR number or branch/tag +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + deploy-instance: + if: contains(github.event.pull_request.labels.*.name, 'deploy') + runs-on: ubuntu-latest + env: + INSTANCE_HOST: 'https://dev.im.dhis2.org' + INSTANCE_NAME: pr-${{ github.event.pull_request.number }} + steps: + - name: Wait for API tests + uses: lewagon/wait-on-check-action@v1.3.1 + with: + ref: ${{ github.head_ref }} + check-name: 'api-test' + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/checkout@v4 + with: + repository: dhis2-sre/im-manager + sparse-checkout: scripts/instances + + - name: Install HTTPie + run: python -m pip install httpie + + - name: Deploy DHIS2 instance + working-directory: scripts/instances + env: + HTTP: https --check-status + USER_EMAIL: ${{ secrets.IM_BOT_EMAIL }} + PASSWORD: ${{ secrets.IM_BOT_PASSWORD }} + IM_HOST: 'https://api.im.dhis2.org' + IMAGE_REPOSITORY: core-pr + IMAGE_TAG: ${{ github.event.pull_request.number }} + IMAGE_PULL_POLICY: Always + DATABASE_ID: sl-sierra-leone-dev-sql-gz + run: ./findByName.sh dev $INSTANCE_NAME && ./restart.sh dev $INSTANCE_NAME || ./deploy-dhis2.sh dev $INSTANCE_NAME + + - name: Wait for instance + run: timeout 300 bash -c 'while [[ "$(curl -fsSL -o /dev/null -w %{http_code} $INSTANCE_HOST/$INSTANCE_NAME)" != "200" ]]; do sleep 5; done' + + - name: Generate analytics + run: curl -X POST -u "${{ secrets.DHIS2_USERNAME }}:${{ secrets.DHIS2_PASSWORD }}" "$INSTANCE_HOST/$INSTANCE_NAME/api/resourceTables/analytics" -d 'executeTei=true' + + - name: Comment instance URL + uses: actions-cool/maintain-one-comment@v3 + with: + body: "Instance deployed to https://dev.im.dhis2.org/pr-${{ github.event.pull_request.number }}" diff --git a/.github/workflows/destroy-instance.yml b/.github/workflows/destroy-instance.yml new file mode 100644 index 000000000000..756923da224d --- /dev/null +++ b/.github/workflows/destroy-instance.yml @@ -0,0 +1,32 @@ +name: Destroy instance + +on: + pull_request: + types: [closed] + +# Cancel previous runs of the same workflow and PR number or branch/tag +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + destroy-instance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + repository: dhis2-sre/im-manager + sparse-checkout: scripts/instances + + - name: Install HTTPie + run: python -m pip install httpie + + - name: Destroy DHIS2 instance + working-directory: scripts/instances + env: + HTTP: https --check-status + USER_EMAIL: ${{ secrets.IM_BOT_EMAIL }} + PASSWORD: ${{ secrets.IM_BOT_PASSWORD }} + IM_HOST: 'https://api.im.dhis2.org' + INSTANCE_NAME: pr-${{ github.event.number }} + run: ./findByName.sh dev $INSTANCE_NAME && ./destroy.sh dev $INSTANCE_NAME diff --git a/.github/workflows/run-api-analytics-tests.yml b/.github/workflows/run-api-analytics-tests.yml index f31c0cbc9cf9..9a5c3a27a004 100644 --- a/.github/workflows/run-api-analytics-tests.yml +++ b/.github/workflows/run-api-analytics-tests.yml @@ -12,7 +12,7 @@ concurrency: group: ${{ github.workflow}}-${{ github.ref }} cancel-in-progress: true jobs: - api-test: + api-analytics-test: env: CORE_IMAGE_NAME: "dhis2/core-dev:local" PR_NUMBER: ${{ github.event.number }} @@ -88,7 +88,7 @@ jobs: contains(needs.*.result, 'failure') && github.ref == 'refs/heads/master' - needs: [ api-test ] + needs: [ api-analytics-test ] steps: - uses: rtCamp/action-slack-notify@v2 env: diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/AggregationType.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/AggregationType.java index 32be2d37813b..72e3b54741b7 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/AggregationType.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/AggregationType.java @@ -63,13 +63,6 @@ public enum AggregationType { private static final EnumSet FIRST_TYPES = EnumSet.of(FIRST, FIRST_AVERAGE_ORG_UNIT, FIRST_FIRST_ORG_UNIT); - /** - * Types that allow non-numeric output such as String or boolean (and possibly also numeric - * output) - */ - private static final EnumSet ALLOWS_NONNUMERIC_TYPES = - EnumSet.of(LAST, FIRST, MIN, MAX, NONE, CUSTOM, DEFAULT); - private final String value; private boolean aggregatable; @@ -99,10 +92,6 @@ public boolean isFirst() { return FIRST_TYPES.contains(this); } - public boolean allowsNonnumeric() { - return ALLOWS_NONNUMERIC_TYPES.contains(this); - } - public static AggregationType fromValue(String value) { for (AggregationType type : AggregationType.values()) { if (type.value.equalsIgnoreCase(value)) { diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/AnalyticsMetaDataKey.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/AnalyticsMetaDataKey.java index a78c85c28fea..c9cbf833181b 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/AnalyticsMetaDataKey.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/AnalyticsMetaDataKey.java @@ -36,11 +36,16 @@ public enum AnalyticsMetaDataKey { ITEMS("items"), DIMENSIONS("dimensions"), PAGER("pager"), + ORG_UNITS("organisationUnits"), ORG_UNIT_HIERARCHY("ouHierarchy"), ORG_UNIT_NAME_HIERARCHY("ouNameHierarchy"), - ORG_UNIT_ANCESTORS("ouAncestors"); + ORG_UNIT_ANCESTORS("ouAncestors"), + USER_ORGUNIT("USER_ORGUNIT"), - private String key; + USER_ORGUNIT_CHILDREN("USER_ORGUNIT_CHILDREN"), + USER_ORGUNIT_GRANDCHILDREN("USER_ORGUNIT_GRANDCHILDREN"); + + private final String key; AnalyticsMetaDataKey(String key) { this.key = key; diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/BaseAnalyticalObject.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/BaseAnalyticalObject.java index 849d9005f7f2..26ec362d8f05 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/BaseAnalyticalObject.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/BaseAnalyticalObject.java @@ -41,6 +41,8 @@ import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; import static org.hisp.dhis.common.DimensionalObject.PERIOD_DIM_ID; import static org.hisp.dhis.common.DimensionalObject.STATIC_DIMS; +import static org.hisp.dhis.common.DimensionalObjectUtils.asActualDimension; +import static org.hisp.dhis.common.DimensionalObjectUtils.linkAssociations; import static org.hisp.dhis.common.DxfNamespaces.DXF_2_0; import static org.hisp.dhis.organisationunit.OrganisationUnit.KEY_LEVEL; import static org.hisp.dhis.organisationunit.OrganisationUnit.KEY_ORGUNIT_GROUP; @@ -79,7 +81,6 @@ import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.dataelement.DataElementGroupSetDimension; import org.hisp.dhis.eventvisualization.Attribute; -import org.hisp.dhis.eventvisualization.EventRepetition; import org.hisp.dhis.eventvisualization.SimpleDimension; import org.hisp.dhis.eventvisualization.SimpleDimension.Type; import org.hisp.dhis.eventvisualization.SimpleDimensionHandler; @@ -587,18 +588,20 @@ protected DimensionalObject getDimensionalObject( * (one that is not defined anywhere and only exists for very specific use cases. See {@link * SimpleDimension}). * - * @param eventAnalyticalObject the object of type EventAnalyticalObject - * @param dimension the dimension, ie: dx, pe, eventDate + * @param eventAnalyticalObject the object of type {@link EventAnalyticalObject}. + * @param dimension the dimension, ie: dx, pe, eventDate. It can be a qualified one like + * program.eventDate, or program.stage.dimension. * @param parent the parent attribute * @return the dimensional object related to the given dimension and attribute. */ protected DimensionalObject getDimensionalObject( EventAnalyticalObject eventAnalyticalObject, String dimension, Attribute parent) { Optional optionalDimensionalObject = getDimensionalObject(dimension); + String actualDim = asActualDimension(dimension); if (optionalDimensionalObject.isPresent()) { return linkAssociations(eventAnalyticalObject, optionalDimensionalObject.get(), parent); - } else if (Type.contains(dimension)) { + } else if (Type.contains(actualDim)) { DimensionalObject dimensionalObject = new SimpleDimensionHandler(eventAnalyticalObject).getDimensionalObject(dimension, parent); @@ -608,35 +611,6 @@ protected DimensionalObject getDimensionalObject( } } - /** - * This method links existing associations between objects. This is mainly needed in cases where - * attributes need to be programmatically associated to fulfill client requirements. - * - * @param eventAnalyticalObject the source object - * @param dimensionalObject where the associations will happen - * @param parent the parent attribute, where the association object should be appended to - * @return the dimensional object containing the correct associations. - */ - private DimensionalObject linkAssociations( - EventAnalyticalObject eventAnalyticalObject, - DimensionalObject dimensionalObject, - Attribute parent) { - // Associating event repetitions. - List repetitions = eventAnalyticalObject.getEventRepetitions(); - - if (isNotEmpty(repetitions)) { - for (EventRepetition eventRepetition : repetitions) { - if (eventRepetition.getDimension() != null - && eventRepetition.getDimension().equals(dimensionalObject.getDimension()) - && parent == eventRepetition.getParent()) { - ((BaseDimensionalObject) dimensionalObject).setEventRepetition(eventRepetition); - } - } - } - - return dimensionalObject; - } - /** * Populates the given dimensionalObjects list based on the respective dimensions provided. * @@ -693,11 +667,13 @@ protected void populateDimensions( * @return a list of DimensionalObjects. */ protected Optional getDimensionalObject(String dimension) { - if (DATA_X_DIM_ID.equals(dimension)) { + String actualDim = asActualDimension(dimension); + + if (DATA_X_DIM_ID.equals(actualDim)) { return Optional.of( new BaseDimensionalObject( dimension, DimensionType.DATA_X, getDataDimensionNameableObjects())); - } else if (PERIOD_DIM_ID.equals(dimension)) { + } else if (PERIOD_DIM_ID.equals(actualDim)) { List periodList = new ArrayList<>(periods); if (hasRelativePeriods()) { @@ -709,7 +685,7 @@ protected Optional getDimensionalObject(String dimension) { } return Optional.of(new BaseDimensionalObject(dimension, DimensionType.PERIOD, periodList)); - } else if (ORGUNIT_DIM_ID.equals(dimension)) { + } else if (ORGUNIT_DIM_ID.equals(actualDim)) { List ouList = new ArrayList<>(); ouList.addAll(organisationUnits); ouList.addAll(transientOrganisationUnits); @@ -744,14 +720,14 @@ protected Optional getDimensionalObject(String dimension) { return Optional.of( new BaseDimensionalObject(dimension, DimensionType.ORGANISATION_UNIT, ouList)); - } else if (CATEGORYOPTIONCOMBO_DIM_ID.equals(dimension)) { + } else if (CATEGORYOPTIONCOMBO_DIM_ID.equals(actualDim)) { return Optional.of( new BaseDimensionalObject( dimension, DimensionType.CATEGORY_OPTION_COMBO, new ArrayList<>())); - } else if (DATA_COLLAPSED_DIM_ID.equals(dimension)) { + } else if (DATA_COLLAPSED_DIM_ID.equals(actualDim)) { return Optional.of( new BaseDimensionalObject(dimension, DimensionType.DATA_COLLAPSED, new ArrayList<>())); - } else if (STATIC_DIMS.contains(dimension)) { + } else if (STATIC_DIMS.contains(actualDim)) { return Optional.of( new BaseDimensionalObject(dimension, DimensionType.STATIC, new ArrayList<>())); } else { @@ -788,13 +764,15 @@ private Optional getDimensionFromEmbeddedObjects(String dimen } private Optional getTrackedEntityDimension(String dimension) { + String actualDim = asActualDimension(dimension); + // Tracked entity attribute. Map attributes = attributeDimensions.stream() .collect(toMap(TrackedEntityAttributeDimension::getUid, Function.identity())); - if (attributes.containsKey(dimension)) { - TrackedEntityAttributeDimension tead = attributes.get(dimension); + if (attributes.containsKey(actualDim)) { + TrackedEntityAttributeDimension tead = attributes.get(actualDim); if (tead != null) { ValueType valueType = @@ -819,7 +797,7 @@ private Optional getTrackedEntityDimension(String dimension) // Tracked entity data element. for (TrackedEntityDataElementDimension tedd : dataElementDimensions) { - if (tedd != null && dimension.equals(tedd.getUid())) { + if (tedd != null && actualDim.equals(tedd.getUid())) { ValueType valueType = tedd.getDataElement() != null ? tedd.getDataElement().getValueType() : null; @@ -827,7 +805,7 @@ private Optional getTrackedEntityDimension(String dimension) tedd.getDataElement() != null ? tedd.getDataElement().getOptionSet() : null; String elementProgramStage = - dimension + (tedd.getProgramStage() != null ? tedd.getProgramStage().getUid() : EMPTY); + actualDim + (tedd.getProgramStage() != null ? tedd.getProgramStage().getUid() : EMPTY); // Return dimensions with distinct program stages. if (!addedElementsProgramStages.contains(elementProgramStage)) { @@ -854,8 +832,8 @@ private Optional getTrackedEntityDimension(String dimension) programIndicatorDimensions.stream() .collect(toMap(TrackedEntityProgramIndicatorDimension::getUid, Function.identity())); - if (programIndicators.containsKey(dimension)) { - TrackedEntityProgramIndicatorDimension teid = programIndicators.get(dimension); + if (programIndicators.containsKey(actualDim)) { + TrackedEntityProgramIndicatorDimension teid = programIndicators.get(actualDim); return Optional.of( new BaseDimensionalObject( @@ -883,11 +861,13 @@ private Optional getTrackedEntityDimension(String dimension) private Optional getDimensionFromEmbeddedObjects( String dimension, DimensionType dimensionType, List embeddedObjects) { + String actualDim = asActualDimension(dimension); + Map dimensions = Maps.uniqueIndex(embeddedObjects, d -> d.getDimension().getDimension()); - if (dimensions.containsKey(dimension)) { - DimensionalEmbeddedObject object = dimensions.get(dimension); + if (dimensions.containsKey(actualDim)) { + DimensionalEmbeddedObject object = dimensions.get(actualDim); if (object != null) { return Optional.of( diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/BaseDimensionalObject.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/BaseDimensionalObject.java index f3e2115ed959..cc8d13aca877 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/BaseDimensionalObject.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/BaseDimensionalObject.java @@ -27,6 +27,7 @@ */ package org.hisp.dhis.common; +import static org.hisp.dhis.common.DimensionalObjectUtils.asBaseObjects; import static org.hisp.dhis.common.DisplayProperty.SHORTNAME; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -44,11 +45,13 @@ import java.util.stream.Collectors; import lombok.Getter; import lombok.Setter; +import org.apache.commons.lang3.tuple.Triple; import org.hisp.dhis.analytics.AggregationType; import org.hisp.dhis.analytics.QueryKey; import org.hisp.dhis.eventvisualization.EventRepetition; import org.hisp.dhis.legend.LegendSet; import org.hisp.dhis.option.OptionSet; +import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramStage; @JacksonXmlRootElement(localName = "dimension", namespace = DxfNamespaces.DXF_2_0) @@ -95,6 +98,9 @@ public class BaseDimensionalObject extends BaseNameableObject implements Dimensi /** The program stage for this dimension. */ private ProgramStage programStage; + /** The program for this dimension. */ + private Program program; + /** The aggregation type for this dimension. */ protected AggregationType aggregationType; @@ -129,12 +135,29 @@ public class BaseDimensionalObject extends BaseNameableObject implements Dimensi public BaseDimensionalObject() {} public BaseDimensionalObject(String dimension) { - this.uid = dimension; + if (dimension != null) { + with(dimension); + } + } + + /** + * This method will split the given dimension into individual "uid" and assign each one to the + * respective object, for each internal association. + * + * @param qualifiedDimension the dimension. It can be a simple uid like "dimUid", or a qualified + * value like "programUid.stageUid.dimUid". + */ + private void with(String qualifiedDimension) { + Triple tripe = asBaseObjects(qualifiedDimension); + + this.program = tripe.getLeft() != null ? tripe.getLeft() : null; + this.programStage = tripe.getMiddle() != null ? tripe.getMiddle() : null; + this.uid = tripe.getRight() != null ? tripe.getRight().getUid() : qualifiedDimension; } public BaseDimensionalObject( String dimension, DimensionType dimensionType, List items) { - this.uid = dimension; + this(dimension); this.dimensionType = dimensionType; this.items = new ArrayList<>(items); } @@ -164,6 +187,7 @@ public BaseDimensionalObject( DimensionType dimensionType, List items, ValueType valueType) { + this(dimension, dimensionType, null, items); this.uid = dimension; this.dimensionType = dimensionType; this.items = new ArrayList<>(items); @@ -231,6 +255,8 @@ public BaseDimensionalObject( this.filter = filter; this.valueType = valueType; this.optionSet = optionSet; + + setProgram(); } // TODO aggregationType in constructors @@ -258,6 +284,12 @@ public DimensionalObject instance() { return object; } + private void setProgram() { + if (programStage != null) { + program = programStage.getProgram(); + } + } + @Override public String getDimensionName() { return dimensionName != null ? dimensionName : uid; @@ -432,6 +464,18 @@ public void setProgramStage(ProgramStage programStage) { this.programStage = programStage; } + @Override + @JsonProperty + @JsonSerialize(as = BaseIdentifiableObject.class) + @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0) + public Program getProgram() { + return program; + } + + public void setProgram(Program program) { + this.program = program; + } + @Override @JsonProperty @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0) diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DataQueryRequest.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DataQueryRequest.java index 42def8cdb8e3..d3c8df4a8d87 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DataQueryRequest.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DataQueryRequest.java @@ -27,6 +27,8 @@ */ package org.hisp.dhis.common; +import static org.hisp.dhis.util.OrganisationUnitCriteriaUtils.getAnalyticsQueryCriteria; + import java.util.Date; import java.util.Set; import lombok.Getter; @@ -101,6 +103,8 @@ public class DataQueryRequest { protected DhisApiVersion apiVersion; + protected String userOrganisationUnitCriteria; + public boolean hasAggregationType() { return aggregationType != null; } @@ -319,6 +323,8 @@ public DataQueryRequestBuilder fromCriteria(AggregateAnalyticsQueryCriteria crit this.request.startDate = criteria.getStartDate(); this.request.timeField = criteria.getTimeField(); this.request.userOrgUnit = criteria.getUserOrgUnit(); + this.request.userOrganisationUnitCriteria = + getAnalyticsQueryCriteria(criteria.getDimension()); this.request.userOrgUnitType = criteria.getUserOrgUnitType(); return this; } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalItemObject.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalItemObject.java index ae9ee1ef422e..c12c4e954df7 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalItemObject.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalItemObject.java @@ -82,4 +82,9 @@ public interface DimensionalItemObject extends NameableObject { /** Sets the query modifiers for an indicator expression. */ void setQueryMods(QueryModifiers queryMods); + + /** Gets the absolute period offset regardless of whether there are query modifiers. */ + default int getPeriodOffset() { + return (getQueryMods() != null) ? getQueryMods().getPeriodOffset() : 0; + } } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObject.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObject.java index 60e02ef6bdc6..255b47837bca 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObject.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObject.java @@ -42,6 +42,7 @@ import org.hisp.dhis.option.OptionSet; import org.hisp.dhis.organisationunit.OrganisationUnitGroup; import org.hisp.dhis.organisationunit.OrganisationUnitGroupSet; +import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramStage; /** @@ -177,6 +178,14 @@ default boolean hasLegendSet() { /** Gets the program stage (not persisted). */ ProgramStage getProgramStage(); + /** Gets the program (not persisted). */ + Program getProgram(); + + /** Indicates whether this dimension has a program (not persisted). */ + default boolean hasProgram() { + return getProgram() != null; + } + /** Indicates whether this dimension has a program stage (not persisted). */ default boolean hasProgramStage() { return getProgramStage() != null; diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java index 98d81379bb3f..1028e6408ae5 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java @@ -27,6 +27,12 @@ */ package org.hisp.dhis.common; +import static java.util.stream.Collectors.joining; +import static lombok.AccessLevel.PRIVATE; +import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; +import static org.apache.commons.lang3.StringUtils.defaultIfBlank; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.commons.lang3.StringUtils.substringAfterLast; import static org.hisp.dhis.common.DimensionalObject.DIMENSION_NAME_SEP; import static org.hisp.dhis.common.DimensionalObject.DIMENSION_SEP; import static org.hisp.dhis.common.DimensionalObject.ITEM_SEP; @@ -46,14 +52,22 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.Stream; +import lombok.NoArgsConstructor; import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Triple; import org.hisp.dhis.common.comparator.ObjectStringValueComparator; import org.hisp.dhis.dataelement.DataElementOperand; +import org.hisp.dhis.eventvisualization.Attribute; +import org.hisp.dhis.eventvisualization.EventRepetition; +import org.hisp.dhis.program.Program; +import org.hisp.dhis.program.ProgramStage; /** * @author Lars Helge Overland */ +@NoArgsConstructor(access = PRIVATE) public class DimensionalObjectUtils { public static final String COMPOSITE_DIM_OBJECT_ESCAPED_SEP = "\\."; @@ -114,6 +128,203 @@ public static List getDimensions(Collection dimension return dims; } + /** + * This method links existing associations between objects. This is mainly needed in cases where + * attributes need to be programmatically associated to fulfill client requirements. + * + * @param eventAnalyticalObject the source object + * @param dimensionalObject where the associations will happen + * @param parent the parent attribute, where the association object should be appended to + * @return the dimensional object containing the correct associations. + */ + public static DimensionalObject linkAssociations( + EventAnalyticalObject eventAnalyticalObject, + DimensionalObject dimensionalObject, + Attribute parent) { + // Associating event repetitions. + List repetitions = eventAnalyticalObject.getEventRepetitions(); + + if (isNotEmpty(repetitions)) { + for (EventRepetition eventRepetition : repetitions) { + String dimension = dimensionalObject.getDimension(); + + if (isNotBlank(dimension)) { + associateEventRepetitionDimensions( + eventRepetition, + dimensionalObject.getProgram(), + dimensionalObject.getProgramStage(), + parent, + dimension); + + boolean associationFound = dimension.equals(eventRepetition.getDimension()); + + if (associationFound) { + ((BaseDimensionalObject) dimensionalObject).setEventRepetition(eventRepetition); + } + } + } + } + + return dimensionalObject; + } + + /** + * This method will split the given dimension into individual "uid" and create each object {@link + * Triple} based on their respective "uid". + * + * @param qualifiedDimension the dimension. It can be a simple uid like "dimUid", or a qualified + * value like "programUid.stageUid.dimUid". + * @return a {@link Triple} of {@link Program}, {@link ProgramStage} and {@link + * BaseDimensionalObject}. + */ + public static Triple asBaseObjects( + String qualifiedDimension) { + String[] uids = qualifiedDimension.split("\\."); + BaseDimensionalObject dimensionalObject = new BaseDimensionalObject(); + + if (uids.length == 1) { + dimensionalObject.setUid(qualifiedDimension); + return Triple.of(null, null, dimensionalObject); + } else if (uids.length == 2) { + dimensionalObject.setUid(uids[1]); + + Program p = new Program(); + p.setUid(uids[0]); + + return Triple.of(p, null, dimensionalObject); + } else if (uids.length == 3) { + dimensionalObject.setUid(uids[2]); + + Program p = new Program(); + p.setUid(uids[0]); + + ProgramStage ps = new ProgramStage(); + ps.setUid(uids[1]); + + return Triple.of(p, ps, dimensionalObject); + } + + return Triple.of(null, null, null); + } + + /** + * This method will associate the given objects with the given event repetition, populating the + * respective objects in the event repetition. + * + * @param eventRepetition the {@link EventRepetition}. + * @param program the {@link Program} to be associated. + * @param programStage the {@link ProgramStage} to be associated. + * @param parent the parent {@link Attribute} + * @param dimension the dimension to be associated. + */ + private static void associateEventRepetitionDimensions( + EventRepetition eventRepetition, + Program program, + ProgramStage programStage, + Attribute parent, + String dimension) { + boolean belongsToProgram = + program != null && program.getUid().equals(eventRepetition.getProgram()); + boolean belongsToProgramStage = + programStage != null && programStage.getUid().equals(eventRepetition.getProgramStage()); + boolean belongsToDimension = + dimension != null + && dimension.equals(eventRepetition.getDimension()) + && parent == eventRepetition.getParent(); + boolean hasNoProgramOrStage = program == null && programStage == null; + + if (belongsToDimension) { + eventRepetition.setParent(parent); + + if (hasNoProgramOrStage) { + eventRepetition.setDimension(dimension); + } + + if (belongsToProgram) { + eventRepetition.setProgram(program.getUid()); + } + + if (belongsToProgramStage) { + eventRepetition.setProgramStage(programStage.getUid()); + } + } + } + + /** + * This method will iterate through each {@link DimensionalObject} in the given list and join the + * "program" + "stage" + "dimension" present in the respective {@link DimensionalObject}. This + * will result in a list of qualified dimensions in the format: + * "programUid.stageUid.dimensionUid". + * + * @param dimensionalObjects the list of {@link DimensionalObject}. + * @return the list of qualified dimensions. + */ + public static List getQualifiedDimensions(List dimensionalObjects) { + List dims = new ArrayList<>(); + + if (dimensionalObjects != null) { + for (DimensionalObject dimension : dimensionalObjects) { + dims.add(getQualifiedDimension(dimension)); + } + } + + return dims; + } + + /** + * This method takes a {@link DimensionalObject} and join the "program" + "stage" + "dimension" + * present in the {@link DimensionalObject}. This will result in a list of qualified dimensions in + * the format: "programUid.stageUid.dimensionUid" or "programUid.dimensionUid". If there is no + * program and no stage, a simple dimension is returned, ie: "dimensionUid". + * + * @param dimensionalObject the {@link DimensionalObject}. + * @return the qualified dimension. + */ + private static String getQualifiedDimension(DimensionalObject dimensionalObject) { + String programUid = + dimensionalObject.getProgram() != null ? dimensionalObject.getProgram().getUid() : null; + String programStageUid = null; + + if (dimensionalObject.hasProgramStage()) { + programStageUid = dimensionalObject.getProgramStage().getUid(); + + if (dimensionalObject.getProgramStage().getProgram() != null) { + programUid = dimensionalObject.getProgramStage().getProgram().getUid(); + } + } + + return asQualifiedDimension(dimensionalObject.getDimension(), programUid, programStageUid); + } + + /** + * Based on the given input dimensions, this method will simply join them together using "." as + * separator. ie: "programUid.programStageUid.dimensionUid" or "programUid.dimensionUid". If there + * is no program and no stage, a simple dimension is returned, ie: "dimensionUid". + * + * @param dimension the representation of a dimension uid. + * @param program the representation of a program uid. + * @param programStage the representation of a program stage uid. + * @return the qualified dimension. + */ + public static String asQualifiedDimension(String dimension, String program, String programStage) { + return Stream.of(program, programStage, dimension) + .filter(StringUtils::isNotBlank) + .collect(joining(COMPOSITE_DIM_OBJECT_PLAIN_SEP)); + } + + /** + * Simply removes any prefix from the qualified dimension, and returns only the actual dimension, + * represented by the last "dimensionUid" of the argument. + * + * @param qualifiedDimension the full dimension, ie. "programUid.programStageUid.dimensionUid" + * @return the uid of the actual dimension, ie: "dimensionUid", or itself if the given argument is + * blank/null/empty. + */ + public static String asActualDimension(String qualifiedDimension) { + return defaultIfBlank( + substringAfterLast(qualifiedDimension, COMPOSITE_DIM_OBJECT_PLAIN_SEP), qualifiedDimension); + } + /** * Creates a two-dimensional array of dimension items based on the list of DimensionalObjects. * I.e. the list of items of each DimensionalObject is converted to an array and inserted into the @@ -600,7 +811,7 @@ public static String sortKey(String key) { public static String getKey(List objects) { return objects.stream() .map(DimensionalItemObject::getShortName) - .collect(Collectors.joining(NAME_SEP)) + .collect(joining(NAME_SEP)) .replaceAll(" ", NAME_SEP) .toLowerCase(); } @@ -612,9 +823,7 @@ public static String getKey(List objects) { * @return a column name string. */ public static String getName(List objects) { - return objects.stream() - .map(DimensionalItemObject::getShortName) - .collect(Collectors.joining(COL_SEP)); + return objects.stream().map(DimensionalItemObject::getShortName).collect(joining(COL_SEP)); } /** diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/EventDataQueryRequest.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/EventDataQueryRequest.java index 452a0e808501..8e207f42871e 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/EventDataQueryRequest.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/EventDataQueryRequest.java @@ -30,11 +30,13 @@ import static org.hisp.dhis.common.CustomDateHelper.getCustomDateFilters; import static org.hisp.dhis.common.CustomDateHelper.getDimensionsWithRefactoredPeDimension; import static org.hisp.dhis.common.CustomDateHelper.isPeDimension; +import static org.hisp.dhis.util.OrganisationUnitCriteriaUtils.getAnalyticsQueryCriteria; import java.util.Arrays; import java.util.Date; import java.util.HashSet; import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -42,6 +44,7 @@ import lombok.Builder; import lombok.Getter; import org.hisp.dhis.analytics.AggregationType; +import org.hisp.dhis.analytics.AnalyticsMetaDataKey; import org.hisp.dhis.analytics.EventOutputType; import org.hisp.dhis.analytics.SortOrder; import org.hisp.dhis.common.RequestTypeAware.EndpointAction; @@ -109,6 +112,8 @@ public class EventDataQueryRequest { private String userOrgUnit; + protected List userOrganisationUnitsCriteria; + private DhisApiVersion apiVersion; private OrganisationUnitSelectionMode ouMode; @@ -146,6 +151,8 @@ public class EventDataQueryRequest { /** flag to enable row context in grid response */ private boolean rowContext; + protected String userOrganisationUnitCriteria; + /** * Copies all properties of this request onto the given request. * @@ -197,6 +204,7 @@ public T copyTo(T request) { queryRequest.enhancedConditions = this.enhancedConditions; queryRequest.outputIdScheme = outputIdScheme; queryRequest.rowContext = rowContext; + queryRequest.userOrganisationUnitCriteria = userOrganisationUnitCriteria; return request; } @@ -256,6 +264,7 @@ public EventDataQueryRequestBuilder fromCriteria(EventsAnalyticsQueryCriteria cr .endpointItem(criteria.getEndpointItem()) .endpointAction(criteria.getEndpointAction()) .enhancedConditions(criteria.isEnhancedConditions()) + .userOrganisationUnitCriteria(getAnalyticsQueryCriteria(criteria.getDimension())) .rowContext(criteria.isRowContext()); if (criteria.getDimension() == null) { @@ -332,6 +341,7 @@ public EventDataQueryRequestBuilder fromCriteria(EnrollmentAnalyticsQueryCriteri .endpointItem(criteria.getEndpointItem()) .endpointAction(criteria.getEndpointAction()) .enhancedConditions(criteria.isEnhancedConditions()) + .userOrganisationUnitCriteria(getAnalyticsQueryCriteria(criteria.getDimension())) .rowContext(criteria.isRowContext()); if (criteria.getDimension() == null) { diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/eventvisualization/EventRepetition.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/eventvisualization/EventRepetition.java index ecea353f7ac4..b7189308b120 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/eventvisualization/EventRepetition.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/eventvisualization/EventRepetition.java @@ -62,6 +62,16 @@ public class EventRepetition implements Serializable, EmbeddedObject { @Nonnull private String dimension; + /** The program associated with the event repetition. */ + @JsonProperty + @JacksonXmlProperty(namespace = DXF_2_0) + private String program; + + /** The program stage associated with the event repetition. */ + @JsonProperty + @JacksonXmlProperty(namespace = DXF_2_0) + private String programStage; + /** * Represents the list of event indexes to be queried. It holds a list of integers that are * interpreted as follows: diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/eventvisualization/EventVisualization.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/eventvisualization/EventVisualization.java index a020aab345c9..82c5004b11b6 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/eventvisualization/EventVisualization.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/eventvisualization/EventVisualization.java @@ -28,7 +28,7 @@ package org.hisp.dhis.eventvisualization; import static java.util.stream.Collectors.toList; -import static org.apache.commons.lang3.ArrayUtils.contains; +import static org.apache.commons.lang3.StringUtils.containsAny; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.join; import static org.hisp.dhis.common.AnalyticsType.EVENT; @@ -80,6 +80,7 @@ import org.hisp.dhis.schema.annotation.Property; import org.hisp.dhis.schema.annotation.PropertyRange; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; +import org.hisp.dhis.trackedentity.TrackedEntityType; import org.hisp.dhis.translation.Translatable; import org.hisp.dhis.user.User; @@ -108,6 +109,9 @@ public class EventVisualization extends BaseAnalyticalObject /** Program stage. */ private ProgramStage programStage; + /** Tracked entity type associated. */ + private TrackedEntityType trackedEntityType; + /** Data element value dimension. */ private DataElement dataElementValueDimension; @@ -365,7 +369,7 @@ public void setSorting(List sorting) { /** * Some EventVisualization's may not have columnDimensions. * - *

PIE, GAUGE and others don't not have rowDimensions. + *

PIE, GAUGE and others do not have rowDimensions. */ @Override public void populateAnalyticalProperties() { @@ -415,12 +419,17 @@ public void validateSortingState() { s -> { if (isBlank(s.getDimension()) || s.getDirection() == null) { throw new IllegalArgumentException("Sorting is not valid"); - } else if (columns.stream().noneMatch(c -> contains(s.getDimension().split("\\."), c))) { + } else if (columns.stream() + .noneMatch(c -> containsAny(s.getDimension(), c.split("\\.")))) { throw new IllegalStateException(s.getDimension()); } }); } + public boolean isMultiProgram() { + return trackedEntityType != null; + } + public AnalyticsType getAnalyticsType() { return EVENT; } @@ -453,6 +462,17 @@ public void setProgramStage(ProgramStage programStage) { this.programStage = programStage; } + @JsonProperty + @JsonSerialize(as = BaseIdentifiableObject.class) + @JacksonXmlProperty(namespace = DXF_2_0) + public TrackedEntityType getTrackedEntityType() { + return trackedEntityType; + } + + public void setTrackedEntityType(TrackedEntityType trackedEntityType) { + this.trackedEntityType = trackedEntityType; + } + @JsonProperty @JsonSerialize(as = BaseIdentifiableObject.class) @JacksonXmlProperty(namespace = DXF_2_0) diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/eventvisualization/SimpleDimension.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/eventvisualization/SimpleDimension.java index cb7ca5b1dfd7..5a08f2d47599 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/eventvisualization/SimpleDimension.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/eventvisualization/SimpleDimension.java @@ -33,6 +33,7 @@ import static org.hisp.dhis.common.DxfNamespaces.DXF_2_0; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import java.io.Serializable; @@ -42,6 +43,7 @@ import javax.annotation.Nonnull; import lombok.Data; import org.hisp.dhis.common.DimensionType; +import org.hisp.dhis.common.DimensionalObjectUtils; /** * This is used to represents dimensions that are needed by clients but do not actually exists in @@ -84,7 +86,7 @@ public DimensionType getParentType() { return parentType; } - public static boolean contains(final String dimension) { + public static boolean contains(String dimension) { return stream(Type.values()).anyMatch(t -> t.getDimension().equals(dimension)); } @@ -106,26 +108,48 @@ public static Type from(final String dimension) { @Nonnull private String dimension; + @JsonProperty + @JacksonXmlProperty(namespace = DXF_2_0) + private String program; + + @JsonProperty + @JacksonXmlProperty(namespace = DXF_2_0) + private String programStage; + @JsonProperty @JacksonXmlProperty(namespace = DXF_2_0) @Nonnull private List values; public SimpleDimension(@Nonnull Attribute parent, @Nonnull String dimension) { - this(parent, dimension, new ArrayList<>()); + this(parent, dimension, null, null, new ArrayList<>()); + } + + public SimpleDimension( + @Nonnull Attribute parent, @Nonnull String dimension, String program, String programStage) { + this(parent, dimension, program, programStage, new ArrayList<>()); } @JsonCreator public SimpleDimension( @JsonProperty("parent") @Nonnull Attribute parent, @JsonProperty("dimension") @Nonnull String dimension, + @JsonProperty("program") String program, + @JsonProperty("programStage") String programStage, @JsonProperty("values") @Nonnull List values) { this.parent = parent; this.dimension = dimension; + this.program = program; + this.programStage = programStage; this.values = values; } - boolean belongsTo(final Attribute parent) { + boolean belongsTo(Attribute parent) { return this.parent == parent; } + + @JsonIgnore + public String asQualifiedDimension() { + return DimensionalObjectUtils.asQualifiedDimension(dimension, program, programStage); + } } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/eventvisualization/SimpleDimensionHandler.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/eventvisualization/SimpleDimensionHandler.java index 85def6fb5f4f..3dad75b55e6f 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/eventvisualization/SimpleDimensionHandler.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/eventvisualization/SimpleDimensionHandler.java @@ -29,6 +29,7 @@ import static java.util.stream.Collectors.toList; import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; +import static org.hisp.dhis.common.DimensionalObjectUtils.asActualDimension; import static org.hisp.dhis.eventvisualization.Attribute.COLUMN; import static org.hisp.dhis.eventvisualization.Attribute.FILTER; import static org.hisp.dhis.eventvisualization.Attribute.ROW; @@ -68,15 +69,20 @@ public SimpleDimensionHandler(final EventAnalyticalObject eventAnalyticalObject) * representing the associated DimensionalObject. It actually returns an instance of * BaseDimensionalObject. * - * @param dimension the dimension, ie: dx, pe, eventDate - * @param parent the parent attribute - * @return the respective dimensional object + * @param qualifiedDimension the dimension, ie: dx, pe, eventDate, program.eventDate, + * program.stage.dimension. + * @param parent the parent attribute. + * @return the respective dimensional object. * @throws IllegalArgumentException if the dimension does not exist in {@link - * SimpleDimension.Type} + * SimpleDimension.Type}. */ - public DimensionalObject getDimensionalObject(final String dimension, final Attribute parent) { + public DimensionalObject getDimensionalObject(String qualifiedDimension, Attribute parent) { + String actualDim = asActualDimension(qualifiedDimension); + return new BaseDimensionalObject( - dimension, from(dimension).getParentType(), loadDimensionalItems(dimension, parent)); + qualifiedDimension, + from(actualDim).getParentType(), + getSimpleDimensionalItems(qualifiedDimension, parent)); } /** @@ -93,25 +99,41 @@ public void associateDimensions() { associateDimensionalObjects(eventAnalyticalObject.getFilters(), FILTER); } + /** + * Makes the correct internal associations where applicable, for each one of the given {@link + * DimensionalObject} in the list. + * + * @param dimensionalObjects the list of {@link DimensionalObject}. + * @param parent the parent {@link Attribute} of the possible association. + */ private void associateDimensionalObjects( - final List dimensionalObjects, final Attribute attribute) { + List dimensionalObjects, Attribute parent) { if (isNotEmpty(dimensionalObjects)) { - for (final DimensionalObject object : dimensionalObjects) { + for (DimensionalObject object : dimensionalObjects) { if (object != null && contains(object.getUid())) { eventAnalyticalObject .getSimpleDimensions() - .add(createSimpleEventDimensionFor(object, attribute)); + .add(createSimpleEventDimensionFor(object, parent)); } } } } - private List loadDimensionalItems( - final String dimension, final Attribute parent) { - final List items = new ArrayList<>(); + /** + * Returns a list of {@link BaseDimensionalItemObject} based on the internal list of {@link + * SimpleDimension} objects, and also based on the given dimension. The items returned must have + * the same qualified dimension and same parent. + * + * @param qualifiedDimension the qualified dimension. + * @param parent the parent {@link Attribute} of the association. + * @return the list of {@link BaseDimensionalItemObject}. + */ + private List getSimpleDimensionalItems( + String qualifiedDimension, Attribute parent) { + List items = new ArrayList<>(); - for (final SimpleDimension simpleDimension : eventAnalyticalObject.getSimpleDimensions()) { - final boolean hasSameDimension = simpleDimension.getDimension().equals(dimension); + for (SimpleDimension simpleDimension : eventAnalyticalObject.getSimpleDimensions()) { + boolean hasSameDimension = simpleDimension.asQualifiedDimension().equals(qualifiedDimension); if (simpleDimension.belongsTo(parent) && hasSameDimension) { items.addAll( @@ -124,16 +146,29 @@ private List loadDimensionalItems( return items; } + /** + * Simply creates a new {@link SimpleDimension} object based on the given arguments. + * + * @param dimensionalObject the {@link DimensionalObject}. + * @param parent the parent {@link Attribute} of the association. + * @return an instance of {@link SimpleDimension}. + */ private SimpleDimension createSimpleEventDimensionFor( - final DimensionalObject dimensionalObject, final Attribute attribute) { - final SimpleDimension simpleDimension = - new SimpleDimension(attribute, dimensionalObject.getUid()); + DimensionalObject dimensionalObject, Attribute parent) { + String programUid = + dimensionalObject.getProgram() != null ? dimensionalObject.getProgram().getUid() : null; + String programStageUid = + dimensionalObject.getProgramStage() != null + ? dimensionalObject.getProgramStage().getUid() + : null; + String actualDim = asActualDimension(dimensionalObject.getUid()); + + SimpleDimension simpleDimension = + new SimpleDimension(parent, actualDim, programUid, programStageUid); if (isNotEmpty(dimensionalObject.getItems())) { simpleDimension.setValues( - dimensionalObject.getItems().stream() - .map(DimensionalItemObject::getUid) - .collect(toList())); + dimensionalObject.getItems().stream().map(DimensionalItemObject::getUid).toList()); } return simpleDimension; diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java index fb385fc6d489..a77105997294 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java @@ -410,7 +410,7 @@ public enum ErrorCode { E7129("Program is specified but does not exist: `{0}`"), E7130("Program stage is specified but does not exist: `{0}`"), E7131("Query failed, likely because the query timed out"), - E7132("An indicator expression caused division by zero operation"), + E7132("Expression violation. Maybe an indicator caused division by zero?"), E7133("Query cannot be executed, possibly because of invalid types or invalid operation"), E7134("Cannot retrieve total value for data elements with skip total category combination"), E7135("Date time is not parsable: `{0}`"), diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/mapping/MapService.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/mapping/MapService.java index 365dd7c422b5..ac5bc7db8c58 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/mapping/MapService.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/mapping/MapService.java @@ -34,5 +34,7 @@ public enum MapService { WMS, TMS, XYZ, - VECTOR_STYLE + VECTOR_STYLE, + GEOJSON_URL, + ARCGIS_FEATURE } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/relationship/Relationship.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/relationship/Relationship.java index e4db27f313ea..fcd68d599662 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/relationship/Relationship.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/relationship/Relationship.java @@ -33,6 +33,7 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; import java.io.Serializable; +import java.util.Date; import org.hisp.dhis.audit.AuditAttribute; import org.hisp.dhis.audit.AuditScope; import org.hisp.dhis.audit.Auditable; @@ -51,6 +52,8 @@ public class Relationship extends SoftDeletableObject implements Serializable { /** Determines if a de-serialized file is compatible with this class. */ private static final long serialVersionUID = 3818815755138507997L; + private Date createdAtClient; + @AuditAttribute private RelationshipType relationshipType; @AuditAttribute private RelationshipItem from; @@ -85,6 +88,15 @@ public Relationship() {} // Getters and setters // ------------------------------------------------------------------------- + @JsonProperty + public Date getCreatedAtClient() { + return createdAtClient; + } + + public void setCreatedAtClient(Date createdAtClient) { + this.createdAtClient = createdAtClient; + } + /** * @return the relationshipType */ diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfiguration.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfiguration.java index df2669a9f172..842c0b13a2f0 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfiguration.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfiguration.java @@ -55,6 +55,7 @@ import org.hisp.dhis.scheduling.parameters.DataSynchronizationJobParameters; import org.hisp.dhis.scheduling.parameters.DisableInactiveUsersJobParameters; import org.hisp.dhis.scheduling.parameters.EventProgramsDataSynchronizationJobParameters; +import org.hisp.dhis.scheduling.parameters.GeoJsonImportJobParams; import org.hisp.dhis.scheduling.parameters.LockExceptionCleanupJobParameters; import org.hisp.dhis.scheduling.parameters.MetadataSyncJobParameters; import org.hisp.dhis.scheduling.parameters.MonitoringJobParameters; @@ -290,7 +291,8 @@ public boolean hasCronExpression() { @JsonSubTypes.Type(value = TestJobParameters.class, name = "TEST"), @JsonSubTypes.Type( value = ImportOptions.class, - name = "COMPLETE_DATA_SET_REGISTRATION_IMPORT") + name = "COMPLETE_DATA_SET_REGISTRATION_IMPORT"), + @JsonSubTypes.Type(value = GeoJsonImportJobParams.class, name = "GEO_JSON_IMPORT") }) public JobParameters getJobParameters() { return jobParameters; diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfigurationService.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfigurationService.java index 57a8a6060b0b..49aef39a283e 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfigurationService.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfigurationService.java @@ -150,14 +150,11 @@ String create(JobConfiguration config, MimeType contentType, InputStream content * Get all job configurations that should start within the next n seconds. * * @param dueInNextSeconds number of seconds from now the job should start - * @param limitToNext1 true, to only return a single config per {@link JobType}, false to return - * all due jobs * @param includeWaiting true to also list jobs that cannot run because another job of the same * type is already running * @return only jobs that should start soon within the given number of seconds */ - List getDueJobConfigurations( - int dueInNextSeconds, boolean limitToNext1, boolean includeWaiting); + List getDueJobConfigurations(int dueInNextSeconds, boolean includeWaiting); /** * Finds stale jobs. diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java index 154ff45e376c..6a48216c8bde 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java @@ -46,6 +46,7 @@ import org.hisp.dhis.scheduling.parameters.DataSynchronizationJobParameters; import org.hisp.dhis.scheduling.parameters.DisableInactiveUsersJobParameters; import org.hisp.dhis.scheduling.parameters.EventProgramsDataSynchronizationJobParameters; +import org.hisp.dhis.scheduling.parameters.GeoJsonImportJobParams; import org.hisp.dhis.scheduling.parameters.LockExceptionCleanupJobParameters; import org.hisp.dhis.scheduling.parameters.MetadataSyncJobParameters; import org.hisp.dhis.scheduling.parameters.MockJobParameters; @@ -102,7 +103,7 @@ public enum JobType { DATAVALUE_IMPORT_INTERNAL(), METADATA_IMPORT(), DATAVALUE_IMPORT(ImportOptions.class), - GEOJSON_IMPORT(), + GEOJSON_IMPORT(GeoJsonImportJobParams.class), EVENT_IMPORT(), ENROLLMENT_IMPORT(), TEI_IMPORT(), @@ -182,6 +183,10 @@ static Defaults dailyRandomBetween3and5(String uid, String name) { this.defaults = defaults; } + /** + * @return true, if {@link JobProgress} events should be forwarded to the {@link + * org.eclipse.emf.common.notify.Notifier} API, otherwise false + */ public boolean isUsingNotifications() { return this == RESOURCE_TABLE || this == SEND_SCHEDULED_MESSAGE @@ -199,9 +204,14 @@ public boolean isUsingNotifications() { || this == PREDICTOR || this == DATAVALUE_IMPORT || this == COMPLETE_DATA_SET_REGISTRATION_IMPORT - || this == METADATA_IMPORT; + || this == METADATA_IMPORT + || this == GEOJSON_IMPORT; } + /** + * @return true, when an error notification should be sent by email in case the job execution + * fails, otherwise false + */ public boolean isUsingErrorNotification() { return this == ANALYTICS_TABLE || this == VALIDATION_RESULTS_NOTIFICATION @@ -214,6 +224,15 @@ public boolean isUsingErrorNotification() { || this == METADATA_IMPORT; } + /** + * @return true, if jobs of this type should try to run as soon as possible by having job + * scheduler workers execute all known ready jobs of the type, when false only the oldest of + * the ready jobs per type is attempted to start in a single loop cycle + */ + public boolean isUsingContinuousExecution() { + return this == METADATA_IMPORT; + } + public boolean hasJobParameters() { return jobParameters != null; } diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/geojson/GeoJsonImportParams.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/GeoJsonImportJobParams.java similarity index 72% rename from dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/geojson/GeoJsonImportParams.java rename to dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/GeoJsonImportJobParams.java index 4c055303208c..3b5cae782763 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/geojson/GeoJsonImportParams.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/GeoJsonImportJobParams.java @@ -25,34 +25,45 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.hisp.dhis.dxf2.geojson; +package org.hisp.dhis.scheduling.parameters; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; import org.hisp.dhis.common.IdentifiableProperty; +import org.hisp.dhis.scheduling.JobParameters; import org.hisp.dhis.user.User; @Builder(toBuilder = true) @Getter +@Setter +@NoArgsConstructor @AllArgsConstructor(access = AccessLevel.PRIVATE) -public class GeoJsonImportParams { +public class GeoJsonImportJobParams implements JobParameters { /** * If true the import is validated and processed without actually modifying any organisation unit * or storing GeoJSON data. */ - private final boolean dryRun; + @JsonProperty private boolean dryRun; - private final String orgUnitIdProperty; + @JsonProperty private String orgUnitIdProperty; - private final IdentifiableProperty idType; + @JsonProperty private IdentifiableProperty idType; /** * Optional UID that refers to an {@link org.hisp.dhis.attribute.Attribute} for which the geometry * is stored. */ - private final String attributeId; + @JsonProperty private String attributeId; - private final User user; + /** + * no `@JsonProperty` added to {@link User} here as the User property is not needed in the JSONB + * object when job config is being persisted (job params are persisted as JSONB). The {@link User} + * property is used in the non-async import so is still required here. + */ + private User user; } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/subexpression/SubexpressionDimensionItem.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/subexpression/SubexpressionDimensionItem.java index 98c892b65314..b319b05f1143 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/subexpression/SubexpressionDimensionItem.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/subexpression/SubexpressionDimensionItem.java @@ -73,6 +73,10 @@ public SubexpressionDimensionItem( // Logic // ------------------------------------------------------------------------- + public boolean hasPeriodOffsets() { + return getItems().stream().anyMatch(i -> i.getPeriodOffset() != 0); + } + /** * Gets a quoted SQL column name for a data element uid and optionally a category option combo * uid. These column names are generated in the SQL fragment for each item and referenced in the @@ -90,9 +94,26 @@ public static String getItemColumnName( String aoc = isEmpty(aocUid) ? "" : "_" + aocUid; - String aggregationName = - (mods != null && mods.getAggregationType() != null) ? mods.getAggregationType().name() : ""; + String periodOffsetMod = + (mods != null && mods.getPeriodOffset() != 0) + ? formatOffsetValue(mods.getPeriodOffset()) + : ""; + + String aggregationMod = + (mods != null && mods.getAggregationType() != null) + ? "_agg_" + mods.getAggregationType().name() + : ""; + + return "\"" + deUid + separator + coc + aoc + periodOffsetMod + aggregationMod + "\""; + } + + // ------------------------------------------------------------------------- + // Supportive methods + // ------------------------------------------------------------------------- - return "\"" + deUid + separator + coc + aoc + aggregationName + "\""; + private static String formatOffsetValue(int offset) { + return (offset < 0) + ? "_minus_" + Integer.toString(-offset) + : "_plus_" + Integer.toString(offset); } } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/OrganisationUnitCriteriaUtils.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/OrganisationUnitCriteriaUtils.java new file mode 100644 index 000000000000..e9ad4f67c066 --- /dev/null +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/OrganisationUnitCriteriaUtils.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.util; + +import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; +import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.hisp.dhis.analytics.AnalyticsMetaDataKey; + +/** Utilities for organisation unit criteria of incoming analytics request. */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class OrganisationUnitCriteriaUtils { + /** + * Converts the given user organisation unit criteria into a list of {@link AnalyticsMetaDataKey} + * (MetaData). + * + * @param userOrganisationUnitsCriteria {@link String} ("ou:USER_ORGUNIT;USER_ORGUNIT_CHILDREN"). + * @return a list of {@link AnalyticsMetaDataKeys} or an empty list. + */ + public static List getAnalyticsMetaDataKeys( + String userOrganisationUnitsCriteria) { + List keys = new ArrayList<>(); + + if (userOrganisationUnitsCriteria == null + || !userOrganisationUnitsCriteria.contains(ORGUNIT_DIM_ID + ":")) { + return keys; + } + + userOrganisationUnitsCriteria = + userOrganisationUnitsCriteria.replace(ORGUNIT_DIM_ID + ":", StringUtils.EMPTY); + List criteria = Arrays.stream(userOrganisationUnitsCriteria.split(";")).toList(); + return criteria.stream() + .filter( + c -> + c.equalsIgnoreCase(AnalyticsMetaDataKey.USER_ORGUNIT.getKey()) + || c.equalsIgnoreCase(AnalyticsMetaDataKey.USER_ORGUNIT_CHILDREN.getKey()) + || c.equalsIgnoreCase(AnalyticsMetaDataKey.USER_ORGUNIT_GRANDCHILDREN.getKey())) + .map(AnalyticsMetaDataKey::valueOf) + .toList(); + } + + /** + * Transform request criteria into a comma separated string + * (USER_ORG_UNIT,USER_ORGUNIT_CHILDREN,USER_ORGUNIT_GRANDCHILDREN) + * + * @param dimensions, set of the requested dimensions + * @return string of the criteria, or empty. + */ + public static String getAnalyticsQueryCriteria(Set dimensions) { + return isNotEmpty(dimensions) + ? dimensions.stream() + .filter(d -> d.contains(ORGUNIT_DIM_ID)) + .collect(Collectors.joining(",")) + : StringUtils.EMPTY; + } +} diff --git a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/BaseDimensionalObjectTest.java b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/BaseDimensionalObjectTest.java index e35a110a1324..b77acf6b4327 100644 --- a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/BaseDimensionalObjectTest.java +++ b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/BaseDimensionalObjectTest.java @@ -32,6 +32,10 @@ import static org.hamcrest.Matchers.hasProperty; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -96,6 +100,47 @@ void verifyInstanceCloneObject() { hasSize(target.getDimensionItemKeywords().getKeywords().size())); } + @Test + void testConstructorWithFullQualifiedDimension() { + // When + BaseDimensionalObject baseDimensionalObject = + new BaseDimensionalObject("programUid.programStageUid.dimensionUid"); + + // Then + assertTrue(baseDimensionalObject.hasProgram()); + assertTrue(baseDimensionalObject.hasProgramStage()); + assertEquals("programUid", baseDimensionalObject.getProgram().getUid()); + assertEquals("programStageUid", baseDimensionalObject.getProgramStage().getUid()); + assertEquals("dimensionUid", baseDimensionalObject.getUid()); + } + + @Test + void testConstructorQualifiedDimensionNoStage() { + // When + BaseDimensionalObject baseDimensionalObject = + new BaseDimensionalObject("programUid.dimensionUid"); + + // Then + assertTrue(baseDimensionalObject.hasProgram()); + assertFalse(baseDimensionalObject.hasProgramStage()); + assertEquals("programUid", baseDimensionalObject.getProgram().getUid()); + assertNull(baseDimensionalObject.getProgramStage()); + assertEquals("dimensionUid", baseDimensionalObject.getUid()); + } + + @Test + void testConstructorQualifiedDimensionOnlyDimensionItem() { + // When + BaseDimensionalObject baseDimensionalObject = new BaseDimensionalObject("dimensionUid"); + + // Then + assertFalse(baseDimensionalObject.hasProgram()); + assertFalse(baseDimensionalObject.hasProgramStage()); + assertNull(baseDimensionalObject.getProgram()); + assertNull(baseDimensionalObject.getProgramStage()); + assertEquals("dimensionUid", baseDimensionalObject.getUid()); + } + private DimensionalItemObject buildDimensionalItemObject() { return rnd.nextObject(BaseDimensionalItemObject.class); } diff --git a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/DimensionalObjectUtilsTest.java b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/DimensionalObjectUtilsTest.java index 5b07c155479a..bd08c8826418 100644 --- a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/DimensionalObjectUtilsTest.java +++ b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/DimensionalObjectUtilsTest.java @@ -27,6 +27,13 @@ */ package org.hisp.dhis.common; +import static org.hisp.dhis.common.DimensionalObjectUtils.asActualDimension; +import static org.hisp.dhis.common.DimensionalObjectUtils.asBaseObjects; +import static org.hisp.dhis.common.DimensionalObjectUtils.asQualifiedDimension; +import static org.hisp.dhis.common.DimensionalObjectUtils.getQualifiedDimensions; +import static org.hisp.dhis.common.DimensionalObjectUtils.linkAssociations; +import static org.hisp.dhis.eventvisualization.Attribute.COLUMN; +import static org.hisp.dhis.eventvisualization.EventVisualizationType.PIVOT_TABLE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; @@ -38,14 +45,18 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.commons.lang3.tuple.Triple; import org.hisp.dhis.attribute.Attribute; import org.hisp.dhis.attribute.AttributeValue; import org.hisp.dhis.category.CategoryOptionCombo; import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.dataelement.DataElementGroup; import org.hisp.dhis.dataelement.DataElementOperand; +import org.hisp.dhis.eventvisualization.EventRepetition; +import org.hisp.dhis.eventvisualization.EventVisualization; import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramDataElementDimensionItem; +import org.hisp.dhis.program.ProgramStage; import org.junit.jupiter.api.Test; /** @@ -271,11 +282,301 @@ void testConvertToDimItemValueMap() { new DimensionItemObjectValue(deA, 10D), new DimensionItemObjectValue(deB, 20D), new DimensionItemObjectValue(deC, 30D)); - final Map asMap = + Map asMap = DimensionalObjectUtils.convertToDimItemValueMap(list); assertEquals(asMap.size(), 3); - assertEquals(((Double) asMap.get(deA)).intValue(), 10); - assertEquals(((Double) asMap.get(deB)).intValue(), 20); - assertEquals(((Double) asMap.get(deC)).intValue(), 30); + assertEquals(10, ((Double) asMap.get(deA)).intValue()); + assertEquals(20, ((Double) asMap.get(deB)).intValue()); + assertEquals(30, ((Double) asMap.get(deC)).intValue()); + } + + @Test + void testLinkAssociationsSuccessfully() { + // Given + EventAnalyticalObject eventAnalyticalObject = stubEventAnalyticalObject(); + BaseDimensionalObject dimensionalObject = stubDimensionalObject(); + org.hisp.dhis.eventvisualization.Attribute parent = COLUMN; + + // When + DimensionalObject result = linkAssociations(eventAnalyticalObject, dimensionalObject, parent); + + // Then + assertEquals(eventAnalyticalObject.getEventRepetitions().get(0), result.getEventRepetition()); + } + + @Test + void testLinkAssociationsDoesNotFindValidAssociation() { + // Given + EventAnalyticalObject eventAnalyticalObject = stubEventAnalyticalObject(); + BaseDimensionalObject dimensionalObject = stubDimensionalObject(); + dimensionalObject.setUid("nonLinkableUid"); + org.hisp.dhis.eventvisualization.Attribute parent = COLUMN; + + // When + DimensionalObject result = linkAssociations(eventAnalyticalObject, dimensionalObject, parent); + + // Then + assertNull(result.getEventRepetition()); + } + + @Test + void testLinkAssociationsWithProgramAndStage() { + // Given + EventAnalyticalObject eventAnalyticalObject = stubEventAnalyticalObject(); + BaseDimensionalObject dimensionalObject = stubDimensionalObject(); + org.hisp.dhis.eventvisualization.Attribute parent = COLUMN; + + // When + DimensionalObject result = linkAssociations(eventAnalyticalObject, dimensionalObject, parent); + + // Then + assertEquals(eventAnalyticalObject.getEventRepetitions().get(0), result.getEventRepetition()); + assertEquals( + eventAnalyticalObject.getEventRepetitions().get(0).getParent(), + result.getEventRepetition().getParent()); + assertEquals(dimensionalObject.getProgram(), result.getProgram()); + assertEquals(dimensionalObject.getProgramStage(), result.getProgramStage()); + } + + @Test + void testLinkAssociationsWithProgramOnly() { + // Given + EventAnalyticalObject eventAnalyticalObject = stubEventAnalyticalObject(); + BaseDimensionalObject dimensionalObject = stubDimensionalObject(); + dimensionalObject.setProgramStage(null); + org.hisp.dhis.eventvisualization.Attribute parent = COLUMN; + + // When + DimensionalObject result = linkAssociations(eventAnalyticalObject, dimensionalObject, parent); + + // Then + assertEquals(eventAnalyticalObject.getEventRepetitions().get(0), result.getEventRepetition()); + assertEquals( + eventAnalyticalObject.getEventRepetitions().get(0).getParent(), + result.getEventRepetition().getParent()); + assertEquals(dimensionalObject.getProgram(), result.getProgram()); + assertNull(result.getProgramStage()); + } + + @Test + void testLinkAssociationsWithProgramStageOnly() { + // Given + EventAnalyticalObject eventAnalyticalObject = stubEventAnalyticalObject(); + BaseDimensionalObject dimensionalObject = stubDimensionalObject(); + dimensionalObject.setProgram(null); + org.hisp.dhis.eventvisualization.Attribute parent = COLUMN; + + // When + DimensionalObject result = linkAssociations(eventAnalyticalObject, dimensionalObject, parent); + + // Then + assertEquals(eventAnalyticalObject.getEventRepetitions().get(0), result.getEventRepetition()); + assertEquals( + eventAnalyticalObject.getEventRepetitions().get(0).getParent(), + result.getEventRepetition().getParent()); + assertNull(result.getProgram()); + assertEquals(dimensionalObject.getProgramStage(), result.getProgramStage()); + } + + @Test + void testGetQualifiedDimensionsWithFullValue() { + // Given + List dimensionalObjects = List.of(stubDimensionalObject()); + + // When + List results = getQualifiedDimensions(dimensionalObjects); + + // Then + assertEquals("programUid.programStageUid.dimensionUid", results.get(0)); + } + + @Test + void testGetQualifiedDimensionsOnlyProgram() { + // Given + BaseDimensionalObject dimensionalObject = stubDimensionalObject(); + dimensionalObject.setProgramStage(null); + List dimensionalObjects = List.of(dimensionalObject); + + // When + List results = getQualifiedDimensions(dimensionalObjects); + + // Then + assertEquals("programUid.dimensionUid", results.get(0)); + } + + @Test + void testGetQualifiedDimensionsOnlyDimensionItem() { + // Given + BaseDimensionalObject dimensionalObject = stubDimensionalObject(); + dimensionalObject.setProgram(null); + dimensionalObject.setProgramStage(null); + List dimensionalObjects = List.of(dimensionalObject); + + // When + List results = getQualifiedDimensions(dimensionalObjects); + + // Then + assertEquals("dimensionUid", results.get(0)); + } + + @Test + void testAsQualifiedDimensionUsingAll() { + // Given + String programUid = "programUid"; + String programStageUid = "programStageUid"; + String dimensionUid = "dimensionUid"; + + // When + String result = asQualifiedDimension(dimensionUid, programUid, programStageUid); + + // Then + assertEquals("programUid.programStageUid.dimensionUid", result); + } + + @Test + void testAsQualifiedDimensionNoProgramStage() { + // Given + String programUid = "programUid"; + String programStageUid = null; + String dimensionUid = "dimensionUid"; + + // When + String result = asQualifiedDimension(dimensionUid, programUid, programStageUid); + + // Then + assertEquals("programUid.dimensionUid", result); + } + + @Test + void testAsQualifiedDimensionOnlyDimension() { + // Given + String programUid = null; + String programStageUid = null; + String dimensionUid = "dimensionUid"; + + // When + String result = asQualifiedDimension(dimensionUid, programUid, programStageUid); + + // Then + assertEquals("dimensionUid", result); + } + + @Test + void testAsBaseObjects() { + // Given + String qualifiedDim = "programUid.programStageUid.dimensionUid"; + + // When + Triple result = asBaseObjects(qualifiedDim); + + // Then + assertEquals("programUid", result.getLeft().getUid()); + assertEquals("programStageUid", result.getMiddle().getUid()); + assertEquals("dimensionUid", result.getRight().getUid()); + } + + @Test + void testBaseObjectsNoProgramStage() { + // Given + String qualifiedDim = "programUid.dimensionUid"; + + // When + Triple result = asBaseObjects(qualifiedDim); + + // Then + assertEquals("programUid", result.getLeft().getUid()); + assertNull(result.getMiddle()); + assertEquals("dimensionUid", result.getRight().getUid()); + } + + @Test + void testBaseObjectsOnlyDimensionItem() { + // Given + String qualifiedDim = "dimensionUid"; + + // When + Triple result = asBaseObjects(qualifiedDim); + + // Then + assertNull(result.getLeft()); + assertNull(result.getMiddle()); + assertEquals("dimensionUid", result.getRight().getUid()); + } + + @Test + void testAsActualDimension() { + // Given + String qualifiedDim = "programUid.programStageUid.dimensionUid"; + + // When + String result = asActualDimension(qualifiedDim); + + // Then + assertEquals("dimensionUid", result); + } + + @Test + void testAsActualDimensionNoProgramStage() { + // Given + String qualifiedDim = "programUid.dimensionUid"; + + // When + String result = asActualDimension(qualifiedDim); + + // Then + assertEquals("dimensionUid", result); + } + + @Test + void testAsActualDimensionOnlyDimensionItem() { + // Given + String qualifiedDim = "dimensionUid"; + + // When + String result = asActualDimension(qualifiedDim); + + // Then + assertEquals("dimensionUid", result); + } + + private BaseDimensionalObject stubDimensionalObject() { + BaseDimensionalObject baseDimensionalObject = new BaseDimensionalObject(); + baseDimensionalObject.setDimension("dimensionUid"); + baseDimensionalObject.setProgram(stubProgram()); + baseDimensionalObject.setProgramStage(stubProgramStage()); + + return baseDimensionalObject; + } + + private EventAnalyticalObject stubEventAnalyticalObject() { + EventVisualization eventVisualization = new EventVisualization(); + eventVisualization.setType(PIVOT_TABLE); + eventVisualization.setEventRepetitions(List.of(stubEventRepetition())); + + return eventVisualization; + } + + private EventRepetition stubEventRepetition() { + EventRepetition eventRepetition = + new EventRepetition( + COLUMN, "dimensionUid", "programUid", "programStageUid", List.of(-1, 2)); + eventRepetition.setProgram(stubProgram().getUid()); + eventRepetition.setProgramStage(stubProgramStage().getUid()); + + return eventRepetition; + } + + private Program stubProgram() { + Program program = new Program(); + program.setUid("programUid"); + + return program; + } + + private ProgramStage stubProgramStage() { + ProgramStage programStage = new ProgramStage(); + programStage.setUid("programStageUid"); + + return programStage; } } diff --git a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/subexpression/SubexpressionDimenstionItemTest.java b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/subexpression/SubexpressionDimenstionItemTest.java index b5b58f6e468f..c0c054e0f5bd 100644 --- a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/subexpression/SubexpressionDimenstionItemTest.java +++ b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/subexpression/SubexpressionDimenstionItemTest.java @@ -28,7 +28,9 @@ package org.hisp.dhis.subexpression; import static org.hisp.dhis.analytics.AggregationType.AVERAGE; +import static org.hisp.dhis.analytics.AggregationType.COUNT; import static org.hisp.dhis.analytics.AggregationType.MAX; +import static org.hisp.dhis.analytics.AggregationType.SUM; import static org.hisp.dhis.common.DimensionItemType.SUBEXPRESSION_DIMENSION_ITEM; import static org.hisp.dhis.subexpression.SubexpressionDimensionItem.getItemColumnName; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -70,13 +72,49 @@ void testGetItemColumnName() { assertEquals("\"de_co\"", getItemColumnName("de", "co", null, null)); assertEquals("\"de_co_ao\"", getItemColumnName("de", "co", "ao", null)); assertEquals("\"de__ao\"", getItemColumnName("de", null, "ao", null)); + } + @Test + void testGetItemColumnNameWithAggregationType() { QueryModifiers mods = QueryModifiers.builder().aggregationType(MAX).build(); - // Test for coc and aoc = empty string when missing - assertEquals("\"deMAX\"", getItemColumnName("de", "", "", mods)); - assertEquals("\"de_coMAX\"", getItemColumnName("de", "co", "", mods)); - assertEquals("\"de_co_aoMAX\"", getItemColumnName("de", "co", "ao", mods)); - assertEquals("\"de__aoMAX\"", getItemColumnName("de", "", "ao", mods)); + assertEquals("\"de_agg_MAX\"", getItemColumnName("de", "", "", mods)); + assertEquals("\"de_co_agg_MAX\"", getItemColumnName("de", "co", "", mods)); + assertEquals("\"de_co_ao_agg_MAX\"", getItemColumnName("de", "co", "ao", mods)); + assertEquals("\"de__ao_agg_MAX\"", getItemColumnName("de", "", "ao", mods)); + } + + @Test + void testGetItemColumnNameWithPeriodOffset() { + QueryModifiers mods = QueryModifiers.builder().periodOffset(-2).build(); + + assertEquals("\"de_minus_2\"", getItemColumnName("de", "", "", mods)); + assertEquals("\"de_co_minus_2\"", getItemColumnName("de", "co", "", mods)); + assertEquals("\"de_co_ao_minus_2\"", getItemColumnName("de", "co", "ao", mods)); + assertEquals("\"de__ao_minus_2\"", getItemColumnName("de", "", "ao", mods)); + + mods = QueryModifiers.builder().periodOffset(3).build(); + + assertEquals("\"de_plus_3\"", getItemColumnName("de", "", "", mods)); + assertEquals("\"de_co_plus_3\"", getItemColumnName("de", "co", "", mods)); + assertEquals("\"de_co_ao_plus_3\"", getItemColumnName("de", "co", "ao", mods)); + assertEquals("\"de__ao_plus_3\"", getItemColumnName("de", "", "ao", mods)); + } + + @Test + void testGetItemColumnNameWithPeriodOffsetAndAggregationType() { + QueryModifiers mods = QueryModifiers.builder().periodOffset(-1).aggregationType(SUM).build(); + + assertEquals("\"de_minus_1_agg_SUM\"", getItemColumnName("de", "", "", mods)); + assertEquals("\"de_co_minus_1_agg_SUM\"", getItemColumnName("de", "co", "", mods)); + assertEquals("\"de_co_ao_minus_1_agg_SUM\"", getItemColumnName("de", "co", "ao", mods)); + assertEquals("\"de__ao_minus_1_agg_SUM\"", getItemColumnName("de", "", "ao", mods)); + + mods = QueryModifiers.builder().periodOffset(1).aggregationType(COUNT).build(); + + assertEquals("\"de_plus_1_agg_COUNT\"", getItemColumnName("de", "", "", mods)); + assertEquals("\"de_co_plus_1_agg_COUNT\"", getItemColumnName("de", "co", "", mods)); + assertEquals("\"de_co_ao_plus_1_agg_COUNT\"", getItemColumnName("de", "co", "ao", mods)); + assertEquals("\"de__ao_plus_1_agg_COUNT\"", getItemColumnName("de", "", "ao", mods)); } } diff --git a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/util/OrganisationUnitCriteriaUtilsTest.java b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/util/OrganisationUnitCriteriaUtilsTest.java new file mode 100644 index 000000000000..d0e3afae686c --- /dev/null +++ b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/util/OrganisationUnitCriteriaUtilsTest.java @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.util; + +import static org.hisp.dhis.analytics.AnalyticsMetaDataKey.USER_ORGUNIT; +import static org.hisp.dhis.analytics.AnalyticsMetaDataKey.USER_ORGUNIT_CHILDREN; +import static org.hisp.dhis.analytics.AnalyticsMetaDataKey.USER_ORGUNIT_GRANDCHILDREN; +import static org.hisp.dhis.util.OrganisationUnitCriteriaUtils.getAnalyticsMetaDataKeys; +import static org.hisp.dhis.util.OrganisationUnitCriteriaUtils.getAnalyticsQueryCriteria; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; +import org.hisp.dhis.analytics.AnalyticsMetaDataKey; +import org.hisp.dhis.common.AggregateAnalyticsQueryCriteria; +import org.hisp.dhis.common.EnrollmentAnalyticsQueryCriteria; +import org.hisp.dhis.common.EventsAnalyticsQueryCriteria; +import org.junit.jupiter.api.Test; + +class OrganisationUnitCriteriaUtilsTest { + private static final String validOuDimensions = + "ou:USER_ORGUNIT;USER_ORGUNIT_CHILDREN;USER_ORGUNIT_GRANDCHILDREN"; + private static final String invalidOuDimensions = + "USER_ORGUNIT;USER_ORGUNIT_CHILDREN;USER_ORGUNIT_GRANDCHILDREN"; + + @Test + void testGetAnalyticsMetaDataKeys_All() { + // given + // when + List keys = getAnalyticsMetaDataKeys(validOuDimensions); + // then + assertEquals(3, keys.size()); + assertEquals(USER_ORGUNIT.getKey(), keys.get(0).getKey()); + assertEquals(USER_ORGUNIT_CHILDREN.getKey(), keys.get(1).getKey()); + assertEquals(USER_ORGUNIT_GRANDCHILDREN.getKey(), keys.get(2).getKey()); + } + + @Test + void testGetAnalyticsMetaDataKeys_Unsupported() { + // given + // when + List keys = getAnalyticsMetaDataKeys(invalidOuDimensions); + + // then + assertEquals(0, keys.size()); + } + + @Test + void testGetAnalyticsQueryCriteria_Enrollment() { + // given + EnrollmentAnalyticsQueryCriteria enrollmentAnalyticsQueryCriteria = + new EnrollmentAnalyticsQueryCriteria(); + enrollmentAnalyticsQueryCriteria.setDimension(Set.of(validOuDimensions)); + + // when + String analyticsQueryCriteria = + getAnalyticsQueryCriteria(enrollmentAnalyticsQueryCriteria.getDimension()); + + // then + assertEquals(validOuDimensions, analyticsQueryCriteria); + } + + @Test + void testGetAnalyticsQueryCriteria_Event() { + // given + EventsAnalyticsQueryCriteria eventsAnalyticsQueryCriteria = new EventsAnalyticsQueryCriteria(); + eventsAnalyticsQueryCriteria.setDimension(Set.of(validOuDimensions)); + + // when + String analyticsQueryCriteria = + getAnalyticsQueryCriteria(eventsAnalyticsQueryCriteria.getDimension()); + + // then + assertEquals(validOuDimensions, analyticsQueryCriteria); + } + + @Test + void testGetAnalyticsQueryCriteria_Aggregate() { + // given + AggregateAnalyticsQueryCriteria aggregateAnalyticsQueryCriteria = + new AggregateAnalyticsQueryCriteria(); + aggregateAnalyticsQueryCriteria.setDimension(Set.of(validOuDimensions)); + + // when + String analyticsQueryCriteria = + getAnalyticsQueryCriteria(aggregateAnalyticsQueryCriteria.getDimension()); + + // then + assertEquals(validOuDimensions, analyticsQueryCriteria); + } + + @Test + void testGetAnalyticsQueryCriteria_Enrollment_No_Dimension() { + // given + EnrollmentAnalyticsQueryCriteria enrollmentAnalyticsQueryCriteria = + new EnrollmentAnalyticsQueryCriteria(); + enrollmentAnalyticsQueryCriteria.setDimension(Set.of(invalidOuDimensions)); + + // when + String analyticsQueryCriteria = + getAnalyticsQueryCriteria(enrollmentAnalyticsQueryCriteria.getDimension()); + + // then + assertEquals(StringUtils.EMPTY, analyticsQueryCriteria); + } + + @Test + void testGetAnalyticsQueryCriteria_Event_No_Dimension() { + // given + EventsAnalyticsQueryCriteria eventsAnalyticsQueryCriteria = new EventsAnalyticsQueryCriteria(); + eventsAnalyticsQueryCriteria.setDimension(Set.of(invalidOuDimensions)); + + // when + String analyticsQueryCriteria = + getAnalyticsQueryCriteria(eventsAnalyticsQueryCriteria.getDimension()); + + // then + assertEquals(StringUtils.EMPTY, analyticsQueryCriteria); + } + + @Test + void testGetAnalyticsQueryCriteria_Aggregate_No_Dimension() { + // given + AggregateAnalyticsQueryCriteria aggregateAnalyticsQueryCriteria = + new AggregateAnalyticsQueryCriteria(); + aggregateAnalyticsQueryCriteria.setDimension(Set.of(invalidOuDimensions)); + + // when + String analyticsQueryCriteria = + getAnalyticsQueryCriteria(aggregateAnalyticsQueryCriteria.getDimension()); + + // then + assertEquals(StringUtils.EMPTY, analyticsQueryCriteria); + } +} diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java index a791430c7554..7f93d311f06c 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java @@ -30,6 +30,8 @@ import static java.time.temporal.ChronoUnit.YEARS; import static java.util.Comparator.reverseOrder; import static java.util.stream.Collectors.toList; +import static org.hisp.dhis.period.PeriodDataProvider.DataSource.DATABASE; +import static org.hisp.dhis.period.PeriodDataProvider.DataSource.SYSTEM_DEFINED; import static org.hisp.dhis.scheduling.JobProgress.FailurePolicy.SKIP_ITEM; import com.google.common.collect.Lists; @@ -184,7 +186,9 @@ public void generateDataElementTable() { @Override @Transactional public void generateDatePeriodTable() { - List availableYears = periodDataProvider.getAvailableYears(); + List availableYears = + periodDataProvider.getAvailableYears( + analyticsExportSettings.getMaxPeriodYearsOffset() == null ? SYSTEM_DEFINED : DATABASE); checkYearsOffset(availableYears); resourceTableStore.generateResourceTable( @@ -203,30 +207,33 @@ public void generateDatePeriodTable() { * @param yearsToCheck the list of years to be checked. */ private void checkYearsOffset(List yearsToCheck) { - int maxYearsOffset = analyticsExportSettings.getMaxPeriodYearsOffset(); - int minRangeAllowed = Year.now().minus(maxYearsOffset, YEARS).getValue(); - int maxRangeAllowed = Year.now().plus(maxYearsOffset, YEARS).getValue(); - - boolean yearsOutOfRange = - yearsToCheck.stream().anyMatch(year -> year < minRangeAllowed || year > maxRangeAllowed); - - if (yearsOutOfRange) { - String errorMessage = "Your database contains years out of the allowed offset."; - errorMessage += - "\n Range of years allowed (based on your system settings and existing data): " - + yearsToCheck.stream() - .filter(year -> year >= minRangeAllowed && year <= maxRangeAllowed) - .toList() - + "."; - errorMessage += - "\n Years out of range found: " - + yearsToCheck.stream() - .filter(year -> year < minRangeAllowed || year > maxRangeAllowed) - .toList() - + "."; - - log.warn(errorMessage); - throw new RuntimeException(errorMessage); + Integer maxYearsOffset = analyticsExportSettings.getMaxPeriodYearsOffset(); + + if (maxYearsOffset != null) { + int minRangeAllowed = Year.now().minus(maxYearsOffset, YEARS).getValue(); + int maxRangeAllowed = Year.now().plus(maxYearsOffset, YEARS).getValue(); + + boolean yearsOutOfRange = + yearsToCheck.stream().anyMatch(year -> year < minRangeAllowed || year > maxRangeAllowed); + + if (yearsOutOfRange) { + String errorMessage = "Your database contains years out of the allowed offset."; + errorMessage += + "\n Range of years allowed (based on your system settings and existing data): " + + yearsToCheck.stream() + .filter(year -> year >= minRangeAllowed && year <= maxRangeAllowed) + .toList() + + "."; + errorMessage += + "\n Years out of range found: " + + yearsToCheck.stream() + .filter(year -> year < minRangeAllowed || year > maxRangeAllowed) + .toList() + + "."; + + log.warn(errorMessage); + throw new RuntimeException(errorMessage); + } } } diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/resourcetable/table/CategoryResourceTable.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/resourcetable/table/CategoryResourceTable.java index 3eb2a31936a8..b6663500400d 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/resourcetable/table/CategoryResourceTable.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/resourcetable/table/CategoryResourceTable.java @@ -98,7 +98,7 @@ public Optional getPopulateTempTableStatement() { sql += "(" + "select co.name from categoryoptioncombos_categoryoptions cocco " - + "inner join dataelementcategoryoption co on cocco.categoryoptionid = co.categoryoptionid " + + "inner join categoryoption co on cocco.categoryoptionid = co.categoryoptionid " + "inner join categories_categoryoptions cco on co.categoryoptionid = cco.categoryoptionid " + "where coc.categoryoptioncomboid = cocco.categoryoptioncomboid " + "and cco.categoryid = " @@ -111,7 +111,7 @@ public Optional getPopulateTempTableStatement() { sql += "(" + "select co.uid from categoryoptioncombos_categoryoptions cocco " - + "inner join dataelementcategoryoption co on cocco.categoryoptionid = co.categoryoptionid " + + "inner join categoryoption co on cocco.categoryoptionid = co.categoryoptionid " + "inner join categories_categoryoptions cco on co.categoryoptionid = cco.categoryoptionid " + "where coc.categoryoptioncomboid = cocco.categoryoptioncomboid " + "and cco.categoryid = " diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_no_options.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_no_options.yaml index 2a5960ad6148..f3ea6e66da76 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_no_options.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_no_options.yaml @@ -31,11 +31,11 @@ section: Categories summary_sql: >- select COUNT(*) as value, - 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM dataelementcategory), 0) as percent - from dataelementcategory where categoryid + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM category), 0) as percent + from category where categoryid not in (select distinct categoryid from categories_categoryoptions); details_sql: >- - SELECT uid,name from dataelementcategory + SELECT uid,name from category where categoryid not in (select distinct categoryid from categories_categoryoptions) ORDER BY name; diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category.yaml index a8ea841e03c3..ef102c86dd4e 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category.yaml @@ -29,10 +29,10 @@ name: categories_one_default_category description: Only one "default" category should exist section: Categories summary_sql: >- - SELECT count(*) AS count FROM dataelementcategory + SELECT count(*) AS count FROM category WHERE name = 'default' AND uid != 'GLevLNI9wkl'; details_sql: >- - SELECT uid, name FROM dataelementcategory + SELECT uid, name FROM category WHERE name = 'default' AND uid != 'GLevLNI9wkl' ORDER BY categoryid; details_id_type: categories diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option.yaml index 72465253f354..d2cb23209543 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option.yaml @@ -29,10 +29,10 @@ name: categories_one_default_category_option description: Only one "default" category option should exist section: Categories summary_sql: >- - SELECT count(*) AS count FROM dataelementcategoryoption + SELECT count(*) AS count FROM categoryoption WHERE name = 'default' AND uid != 'xYerKDKCefk'; details_sql: >- - SELECT uid, name FROM dataelementcategoryoption + SELECT uid, name FROM categoryoption WHERE name = 'default' AND uid != 'xYerKDKCefk' ORDER BY categoryoptionid; details_id_type: categoryOptions diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_same_category_options.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_same_category_options.yaml index 09bcc3ab03ac..d922a2acf9b5 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_same_category_options.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_same_category_options.yaml @@ -39,10 +39,10 @@ ) SELECT COUNT(*) as value, 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) - FROM dataelementcategory),0 ) percent + FROM category),0 ) percent FROM duplicative_categories; details_sql: >- - SELECT x.uid,'(' || b.rank || ') ' || x.name as name from dataelementcategory x + SELECT x.uid,'(' || b.rank || ') ' || x.name as name from category x INNER JOIN ( SELECT categoryid, array_agg(categoryoptionid ORDER BY categoryoptionid) as catoptions from categories_categoryoptions GROUP BY categoryid diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_shared_category_options_in_combo.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_shared_category_options_in_combo.yaml index 229002d24706..82a97bb0285b 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_shared_category_options_in_combo.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_shared_category_options_in_combo.yaml @@ -34,7 +34,7 @@ summary_sql: >- select cc.name as cc_name, co.categoryoptionid, co.name as co_name from categorycombo cc inner join categorycombos_categories ccc on cc.categorycomboid=ccc.categorycomboid inner join categories_categoryoptions cco on ccc.categoryid=cco.categoryid - inner join dataelementcategoryoption co on cco.categoryoptionid=co.categoryoptionid + inner join categoryoption co on cco.categoryoptionid=co.categoryoptionid group by cc_name, co.categoryoptionid, co_name having count(*) > 1 ) SELECT COUNT(*)as value, @@ -45,7 +45,7 @@ details_sql: >- select cc.uid, cc.name as cc_name, co.categoryoptionid, co.name as co_name from categorycombo cc inner join categorycombos_categories ccc on cc.categorycomboid=ccc.categorycomboid inner join categories_categoryoptions cco on ccc.categoryid=cco.categoryid - inner join dataelementcategoryoption co on cco.categoryoptionid=co.categoryoptionid + inner join categoryoption co on cco.categoryoptionid=co.categoryoptionid group by cc.uid, cc_name, co.categoryoptionid, co_name having count(*) > 1 ) SELECT uid, cc_name as name, diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_excess_members.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_excess_members.yaml index 7a865aab41e8..67c7d28fbff9 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_excess_members.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_excess_members.yaml @@ -48,13 +48,13 @@ summary_sql: >- INNER JOIN categoryoptiongroupsetmembers b on a.categoryoptiongroupid = b.categoryoptiongroupid ) c GROUP BY c.categoryoptionid,c.categoryoptiongroupsetid HAVING array_length(array_agg(c.categoryoptionid), 1) > 1 ) d ) e - INNER JOIN dataelementcategoryoption co on e.categoryoptionid = co.categoryoptionid + INNER JOIN categoryoption co on e.categoryoptionid = co.categoryoptionid INNER JOIN categoryoptiongroupset cogs on e.categoryoptiongroupsetid = cogs.categoryoptiongroupsetid INNER JOIN categoryoptiongroup cog on e.categoryoptiongroupid = cog.categoryoptiongroupid ) x GROUP BY x.uid,x.name,x.cogs_name ) SELECT COUNT(*), - 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM dataelementcategoryoption), 0) as percent + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM categoryoption), 0) as percent FROM cos_multiple_groups; details_sql: >- SELECT x.uid,x.name, x.cogs_name || ' :{'|| @@ -74,7 +74,7 @@ details_sql: >- INNER JOIN categoryoptiongroupsetmembers b on a.categoryoptiongroupid = b.categoryoptiongroupid ) c GROUP BY c.categoryoptionid,c.categoryoptiongroupsetid HAVING array_length(array_agg(c.categoryoptionid), 1) > 1 ) d ) e - INNER JOIN dataelementcategoryoption co on e.categoryoptionid = co.categoryoptionid + INNER JOIN categoryoption co on e.categoryoptionid = co.categoryoptionid INNER JOIN categoryoptiongroupset cogs on e.categoryoptiongroupsetid = cogs.categoryoptiongroupsetid INNER JOIN categoryoptiongroup cog on e.categoryoptiongroupid = cog.categoryoptiongroupid ) x GROUP BY x.uid,x.name,x.cogs_name diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_sets_incomplete.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_sets_incomplete.yaml index abc7b2eeb11c..3707569f30ee 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_sets_incomplete.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_sets_incomplete.yaml @@ -53,8 +53,8 @@ summary_sql: >- (SELECT categoryid,array_agg(categoryoptionid) as wants from categories_categoryoptions GROUP BY categoryid) as z USING (categoryid) ) as cat_option_group_check ) f) g INNER JOIN categoryoptiongroupset cogs USING(categoryoptiongroupsetid) - INNER JOIN dataelementcategory cats USING(categoryid) - INNER JOIN dataelementcategoryoption opt USING(categoryoptionid) + INNER JOIN category cats USING(categoryid) + INNER JOIN categoryoption opt USING(categoryoptionid) ORDER BY cogs.uid, cats.name) SELECT COUNT(*) as value, 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) from categoryoptiongroupmembers @@ -83,8 +83,8 @@ details_sql: >- (SELECT categoryid,array_agg(categoryoptionid) as wants from categories_categoryoptions GROUP BY categoryid) as z USING (categoryid) ) as cat_option_group_check ) f) g INNER JOIN categoryoptiongroupset cogs USING(categoryoptiongroupsetid) - INNER JOIN dataelementcategory cats USING(categoryid) - INNER JOIN dataelementcategoryoption opt USING(categoryoptionid) + INNER JOIN category cats USING(categoryid) + INNER JOIN categoryoption opt USING(categoryoptionid) ORDER BY cogs.uid, cats.name; details_id_type: categoryOptionGroupSets severity: SEVERE diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_options_no_categories.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_options_no_categories.yaml index 26f1ecb871b0..ca31c72bc66f 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_options_no_categories.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_options_no_categories.yaml @@ -31,17 +31,17 @@ section_order: 3 summary_sql: >- WITH category_options_no_categories AS ( - SELECT uid,name FROM dataelementcategoryoption + SELECT uid,name FROM categoryoption WHERE categoryoptionid NOT IN (SELECT DISTINCT categoryoptionid FROM categories_categoryoptions)) SELECT COUNT(*) as value, 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) - FROM dataelementcategoryoption), 0 ) as percent + FROM categoryoption), 0 ) as percent FROM category_options_no_categories; details_sql: >- - SELECT uid,name FROM dataelementcategoryoption + SELECT uid,name FROM categoryoption WHERE categoryoptionid NOT IN (SELECT DISTINCT categoryoptionid diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java index ac04a8479b32..b34c775dee60 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java @@ -102,7 +102,7 @@ void testReadDataIntegrityYaml() { .getIssues() .get(0) .getComment() - .startsWith("SELECT uid,name from dataelementcategory")); + .startsWith("SELECT uid,name from category")); } @Test diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/resourcetable/DefaultResourceTableServiceTest.java b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/resourcetable/DefaultResourceTableServiceTest.java index 752eb08cf7df..899d990e6051 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/resourcetable/DefaultResourceTableServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/resourcetable/DefaultResourceTableServiceTest.java @@ -28,6 +28,7 @@ package org.hisp.dhis.resourcetable; import static java.time.temporal.ChronoUnit.YEARS; +import static org.hisp.dhis.period.PeriodDataProvider.DataSource.DATABASE; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -63,7 +64,7 @@ void generateDatePeriodTableWhenYearIsOutOfRange() { int defaultOffset = 22; // When - when(periodDataProvider.getAvailableYears()).thenReturn(yearsToCheck); + when(periodDataProvider.getAvailableYears(DATABASE)).thenReturn(yearsToCheck); when(analyticsExportSettings.getMaxPeriodYearsOffset()).thenReturn(defaultOffset); // Then @@ -82,7 +83,7 @@ void generateDatePeriodTableWhenOffsetIsZeroWithPreviousYears() { int zeroOffset = 0; // When - when(periodDataProvider.getAvailableYears()).thenReturn(yearsToCheck); + when(periodDataProvider.getAvailableYears(DATABASE)).thenReturn(yearsToCheck); when(analyticsExportSettings.getMaxPeriodYearsOffset()).thenReturn(zeroOffset); // Then @@ -101,7 +102,7 @@ void generateDatePeriodTableWhenOffsetIsZeroWithCurrentYear() { int zeroOffset = 0; // When - when(periodDataProvider.getAvailableYears()).thenReturn(yearsToCheck); + when(periodDataProvider.getAvailableYears(DATABASE)).thenReturn(yearsToCheck); when(analyticsExportSettings.getMaxPeriodYearsOffset()).thenReturn(zeroOffset); doNothing().when(resourceTableStore).generateResourceTable(any()); @@ -120,7 +121,7 @@ void generateDatePeriodTableWhenYearsAreInExpectedRange() { int defaultOffset = 2; // When - when(periodDataProvider.getAvailableYears()).thenReturn(yearsToCheck); + when(periodDataProvider.getAvailableYears(DATABASE)).thenReturn(yearsToCheck); when(analyticsExportSettings.getMaxPeriodYearsOffset()).thenReturn(defaultOffset); doNothing().when(resourceTableStore).generateResourceTable(any()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/pom.xml b/dhis-2/dhis-services/dhis-service-analytics/pom.xml index 9072720af2bc..d8c74619e504 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/pom.xml +++ b/dhis-2/dhis-services/dhis-service-analytics/pom.xml @@ -39,6 +39,10 @@ org.hisp.dhis dhis-service-setting + + org.hisp.dhis + dhis-support-expression-parser + org.hisp.dhis dhis-support-commons diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java index 1cfd312f0705..ca39a84faee0 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java @@ -117,6 +117,7 @@ import org.hisp.dhis.user.User; import org.hisp.dhis.util.DateUtils; import org.hisp.dhis.util.ObjectUtils; +import org.hisp.dhis.util.OrganisationUnitCriteriaUtils; import org.springframework.util.Assert; /** @@ -420,6 +421,8 @@ public class DataQueryParams { /** Used to set the type of OrgUnit from the current user to the {@see DataQueryParams} object */ protected UserOrgUnitType userOrgUnitType; + protected List userOrganisationUnitsCriteria; + /** Mapping of organisation unit sub-hierarchy roots and lowest available data approval levels. */ protected transient Map dataApprovalLevels = new HashMap<>(); @@ -538,10 +541,15 @@ public T copyTo(T params) { params.explainOrderId = this.explainOrderId; params.serverBaseUrl = this.serverBaseUrl; params.download = this.download; + params.userOrganisationUnitsCriteria = this.userOrganisationUnitsCriteria; return params; } + public List getUserOrganisationUnitsCriteria() { + return userOrganisationUnitsCriteria; + } + public String getExplainOrderId() { return explainOrderId; } @@ -3019,6 +3027,13 @@ public Builder withUserOrgUnitType(UserOrgUnitType userOrgUnitType) { return this; } + public Builder withUserOrganisationUnitsCriteria(String userOrganisationUnitsCriteria) { + + this.params.userOrganisationUnitsCriteria = + OrganisationUnitCriteriaUtils.getAnalyticsMetaDataKeys(userOrganisationUnitsCriteria); + return this; + } + public Builder withAnalyzeOrderId() { this.params.explainOrderId = UUID.randomUUID().toString(); return this; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/analyze/RequestExecutionPlanStore.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/analyze/RequestExecutionPlanStore.java index 901ec6cfe0f6..2062b3dd85b5 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/analyze/RequestExecutionPlanStore.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/analyze/RequestExecutionPlanStore.java @@ -64,7 +64,9 @@ public class RequestExecutionPlanStore implements ExecutionPlanStore { @Nonnull private final JdbcTemplate jdbcTemplate; - @Nonnull private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; + @Nonnull + @Qualifier("analyticsNamedParameterJdbcTemplate") + private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; @Nonnull private final ScheduledExecutorService executorService; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/SqlQueryExecutor.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/SqlQueryExecutor.java index a820750e322d..8444c1488744 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/SqlQueryExecutor.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/SqlQueryExecutor.java @@ -46,7 +46,7 @@ public class SqlQueryExecutor implements QueryExecutor { @Nonnull private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; - public SqlQueryExecutor(@Qualifier("readOnlyJdbcTemplate") JdbcTemplate jdbcTemplate) { + public SqlQueryExecutor(@Qualifier("analyticsReadOnlyJdbcTemplate") JdbcTemplate jdbcTemplate) { this.namedParameterJdbcTemplate = new NamedParameterJdbcTemplate(jdbcTemplate); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/TableInfoReader.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/TableInfoReader.java index a201dc2578e6..299d86a6c9c3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/TableInfoReader.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/TableInfoReader.java @@ -34,6 +34,7 @@ import javax.annotation.Nonnull; import lombok.AllArgsConstructor; import lombok.Data; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; @@ -46,6 +47,8 @@ @Component @AllArgsConstructor public class TableInfoReader { + + @Qualifier("analyticsJdbcTemplate") private final JdbcTemplate jdbcTemplate; /** diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/processing/MetadataParamsHandler.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/processing/MetadataParamsHandler.java index b45ce476d8af..22696c9589f5 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/processing/MetadataParamsHandler.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/processing/MetadataParamsHandler.java @@ -35,12 +35,16 @@ import static org.hisp.dhis.analytics.AnalyticsMetaDataKey.ORG_UNIT_NAME_HIERARCHY; import static org.hisp.dhis.analytics.AnalyticsMetaDataKey.PAGER; import static org.hisp.dhis.analytics.orgunit.OrgUnitHelper.getActiveOrganisationUnits; +import static org.hisp.dhis.analytics.util.AnalyticsOrganisationUnitUtils.getUserOrganisationUnitItems; import static org.hisp.dhis.common.DimensionalObjectUtils.asTypedList; import static org.hisp.dhis.organisationunit.OrganisationUnit.getParentGraphMap; import static org.hisp.dhis.organisationunit.OrganisationUnit.getParentNameGraphMap; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; +import org.hisp.dhis.analytics.AnalyticsMetaDataKey; import org.hisp.dhis.analytics.common.MetadataInfo; import org.hisp.dhis.analytics.common.params.AnalyticsPagingParams; import org.hisp.dhis.analytics.common.params.CommonParams; @@ -63,6 +67,7 @@ @Component public class MetadataParamsHandler { private static final String DOT = "."; + private static final String ORG_UNIT_DIM = "ou"; /** * Appends the metadata to the given {@link Grid} based on the given arguments. @@ -73,10 +78,16 @@ public class MetadataParamsHandler { */ public void handle(Grid grid, CommonParams commonParams, User user, long rowsCount) { if (!commonParams.isSkipMeta()) { - MetadataInfo metadataInfo = new MetadataInfo(); // Dimensions. - metadataInfo.put(ITEMS.getKey(), new MetadataItemsHandler().handle(grid, commonParams)); + List userOrgUnitMetaDataKeys = + getUserOrgUnitsMetadataKeys(commonParams); + Map items = + new HashMap<>(new MetadataItemsHandler().handle(grid, commonParams)); + getUserOrganisationUnitItems(user, userOrgUnitMetaDataKeys).forEach(items::putAll); + MetadataInfo metadataInfo = new MetadataInfo(); + metadataInfo.put(ITEMS.getKey(), items); + metadataInfo.put( DIMENSIONS.getKey(), new MetadataDimensionsHandler().handle(grid, commonParams)); @@ -129,6 +140,35 @@ private List getActiveOrgUnits(Grid grid, CommonParams commonP return getActiveOrganisationUnits(grid, organisationUnits); } + /** + * Retrieve the analytics metadata keys belong to user organisation unit dimension group + * + * @param commonParams the {@link CommonParams}. + * @return list of the {@link AnalyticsMetaDataKey} + */ + private static List getUserOrgUnitsMetadataKeys(CommonParams commonParams) { + return commonParams.getDimensionIdentifiers().stream() + .filter(dimensionIdentifier -> dimensionIdentifier.toString().equals(ORG_UNIT_DIM)) + .flatMap(dimensionIdentifier -> dimensionIdentifier.getDimension().getItems().stream()) + .flatMap(item -> item.getValues().stream()) + .filter( + item -> + item.equals(AnalyticsMetaDataKey.USER_ORGUNIT.getKey()) + || item.equals(AnalyticsMetaDataKey.USER_ORGUNIT_CHILDREN.getKey()) + || item.equals(AnalyticsMetaDataKey.USER_ORGUNIT_GRANDCHILDREN.getKey())) + .map( + item -> { + if (item.equals(AnalyticsMetaDataKey.USER_ORGUNIT.getKey())) { + return AnalyticsMetaDataKey.USER_ORGUNIT; + } + if (item.equals(AnalyticsMetaDataKey.USER_ORGUNIT_CHILDREN.getKey())) { + return AnalyticsMetaDataKey.USER_ORGUNIT_CHILDREN; + } + return AnalyticsMetaDataKey.USER_ORGUNIT_GRANDCHILDREN; + }) + .toList(); + } + /** * Returns the query {@link QueryItem} identifier. It may be prefixed with its program stage * identifier (if one exists). diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java index d7aebf9e1355..7750367f270e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java @@ -166,6 +166,7 @@ public DataQueryParams getFromRequest(DataQueryRequest request) { .withDuplicatesOnly(request.isDuplicatesOnly()) .withApprovalLevel(request.getApprovalLevel()) .withUserOrgUnitType(request.getUserOrgUnitType()) + .withUserOrganisationUnitsCriteria(request.getUserOrganisationUnitCriteria()) .withApiVersion(request.getApiVersion()) .withLocale(locale) .withOutputFormat(ANALYTICS) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java index b7b70e71c2f1..bd3c2a0f9b57 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java @@ -40,17 +40,21 @@ import static org.hisp.dhis.analytics.DataQueryParams.LEVEL_PREFIX; import static org.hisp.dhis.analytics.DataQueryParams.VALUE_ID; import static org.hisp.dhis.analytics.DataType.TEXT; +import static org.hisp.dhis.analytics.data.SubexpressionPeriodOffsetUtils.getParamsWithOffsetPeriods; import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.ANALYTICS_TBL_ALIAS; import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quote; import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quoteAlias; import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quoteAliasCommaSeparate; import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quoteWithFunction; import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quotedListOf; +import static org.hisp.dhis.analytics.util.AnalyticsUtils.ERR_MSG_SILENT_FALLBACK; import static org.hisp.dhis.analytics.util.AnalyticsUtils.throwIllegalQueryEx; +import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; import static org.hisp.dhis.common.DimensionalObject.DIMENSION_SEP; import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; import static org.hisp.dhis.commons.collection.CollectionUtils.concat; import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString; +import static org.hisp.dhis.dxf2.webmessage.WebMessageUtils.relationDoesNotExist; import static org.hisp.dhis.util.DateUtils.getMediumDateString; import com.google.common.collect.Lists; @@ -73,6 +77,7 @@ import org.hisp.dhis.analytics.DataQueryParams; import org.hisp.dhis.analytics.DataType; import org.hisp.dhis.analytics.MeasureFilter; +import org.hisp.dhis.analytics.Partitions; import org.hisp.dhis.analytics.QueryPlanner; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; import org.hisp.dhis.analytics.table.PartitionUtils; @@ -152,7 +157,7 @@ public class JdbcAnalyticsManager implements AnalyticsManager { private final QueryPlanner queryPlanner; - @Qualifier("readOnlyJdbcTemplate") + @Qualifier("analyticsReadOnlyJdbcTemplate") private final JdbcTemplate jdbcTemplate; private final ExecutionPlanStore executionPlanStore; @@ -180,21 +185,33 @@ public Future> getAggregatedDataValues( params = queryPlanner.assignPartitionsFromQueryPeriods(params, tableType); } + if (params.hasSubexpressions() && params.getSubexpression().hasPeriodOffsets()) { + params = getParamsWithOffsetPartitions(params, tableType); + } + String sql = getSql(params, tableType); log.debug(sql); + final DataQueryParams immutableParams = DataQueryParams.newBuilder(params).build(); + if (params.analyzeOnly()) { - executionPlanStore.addExecutionPlan(params.getExplainOrderId(), sql); + withExceptionHandling( + () -> executionPlanStore.addExecutionPlan(immutableParams.getExplainOrderId(), sql)); return new AsyncResult<>(Maps.newHashMap()); } Map map; try { - map = getKeyValueMap(params, sql, maxLimit); + map = + withExceptionHandling(() -> getKeyValueMap(immutableParams, sql, maxLimit)) + .orElse(Map.of()); } catch (BadSqlGrammarException ex) { - log.info(AnalyticsUtils.ERR_MSG_TABLE_NOT_EXISTING, ex); + if (relationDoesNotExist(ex.getSQLException())) { + throw ex; + } + log.warn(ERR_MSG_SILENT_FALLBACK, ex); return new AsyncResult<>(Maps.newHashMap()); } @@ -202,7 +219,6 @@ public Future> getAggregatedDataValues( return new AsyncResult<>(map); } catch (DataAccessResourceFailureException ex) { - log.warn(ErrorCode.E7131.getMessage(), ex); throw new QueryRuntimeException(ErrorCode.E7131); } catch (RuntimeException ex) { log.error(DebugUtils.getStackTrace(ex)); @@ -272,6 +288,23 @@ public void replaceDataPeriodsWithAggregationPeriods( // Supportive methods // ------------------------------------------------------------------------- + /** + * For params with subexpression period offsets, inserts the partitions we will need to fetch the + * offset data from the database. + * + *

Note that the params query periods are not changed because these are still the reporting + * periods that we need to know when constructing the subexpression sub-query. + */ + private DataQueryParams getParamsWithOffsetPartitions( + DataQueryParams params, AnalyticsTableType tableType) { + + DataQueryParams paramsWithOffsetPeriods = getParamsWithOffsetPeriods(params); + DataQueryParams paramsWithOffsetPartitions = + queryPlanner.assignPartitionsFromQueryPeriods(paramsWithOffsetPeriods, tableType); + Partitions offsetParitions = paramsWithOffsetPartitions.getPartitions(); + return DataQueryParams.newBuilder(params).withPartitions(offsetParitions).build(); + } + /** * Generates the query SQL. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcRawAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcRawAnalyticsManager.java index 54bb61659ecc..9823235ef473 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcRawAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcRawAnalyticsManager.java @@ -73,7 +73,7 @@ public class JdbcRawAnalyticsManager implements RawAnalyticsManager { private static final String DIM_NAME_OU = "ou.path"; - @Qualifier("readOnlyJdbcTemplate") + @Qualifier("analyticsReadOnlyJdbcTemplate") private final JdbcTemplate jdbcTemplate; // ------------------------------------------------------------------------- diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcSubexpressionQueryGenerator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcSubexpressionQueryGenerator.java index 2ab51f95a6c9..88eafc13747b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcSubexpressionQueryGenerator.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcSubexpressionQueryGenerator.java @@ -32,16 +32,26 @@ import static org.apache.commons.lang3.StringUtils.isEmpty; import static org.hisp.dhis.analytics.AggregationType.MAX; import static org.hisp.dhis.analytics.AnalyticsAggregationType.fromAggregationType; +import static org.hisp.dhis.analytics.DataType.BOOLEAN; +import static org.hisp.dhis.analytics.DataType.NUMERIC; +import static org.hisp.dhis.analytics.DataType.fromValueType; import static org.hisp.dhis.analytics.data.JdbcAnalyticsManager.AO; import static org.hisp.dhis.analytics.data.JdbcAnalyticsManager.CO; import static org.hisp.dhis.analytics.data.JdbcAnalyticsManager.DX; import static org.hisp.dhis.analytics.data.JdbcAnalyticsManager.OU; import static org.hisp.dhis.analytics.data.JdbcAnalyticsManager.VALUE; +import static org.hisp.dhis.analytics.data.SubexpressionPeriodOffsetUtils.DELTA; +import static org.hisp.dhis.analytics.data.SubexpressionPeriodOffsetUtils.REPORTPERIOD; +import static org.hisp.dhis.analytics.data.SubexpressionPeriodOffsetUtils.SHIFT; +import static org.hisp.dhis.analytics.data.SubexpressionPeriodOffsetUtils.getParamsWithOffsetPeriodsWithoutData; +import static org.hisp.dhis.analytics.data.SubexpressionPeriodOffsetUtils.joinPeriodOffsetValues; import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.ANALYTICS_TBL_ALIAS; import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.encode; import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quote; import static org.hisp.dhis.common.DimensionalObject.DATA_X_DIM_ID; +import static org.hisp.dhis.common.DimensionalObject.PERIOD_DIM_ID; import static org.hisp.dhis.commons.collection.CollectionUtils.addUnique; +import static org.hisp.dhis.parser.expression.ParserUtils.castSql; import static org.hisp.dhis.subexpression.SubexpressionDimensionItem.getItemColumnName; import java.util.List; @@ -49,7 +59,11 @@ import org.hisp.dhis.analytics.AnalyticsAggregationType; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.DataQueryParams; +import org.hisp.dhis.analytics.DataType; +import org.hisp.dhis.common.BaseDimensionalObject; +import org.hisp.dhis.common.DimensionType; import org.hisp.dhis.common.DimensionalItemObject; +import org.hisp.dhis.common.DimensionalObject; import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.dataelement.DataElementOperand; import org.hisp.dhis.subexpression.SubexpressionDimensionItem; @@ -84,6 +98,36 @@ * group by uidlevel2, monthly; * * + * If there are data elements (or data element operands) inside the subexpresion having a + * .periodOffset query modifier (with a non-zero value), then an inline table is joined which allows + * for mapping between data periods (the period in the database) and reporting periods (the period + * for which the data is reported, after applying the period offset). For example, consider the + * subexpression: + * + * 

+ *       subExpression( #{A2VfEfPflHV} + #{A2VfEfPflHV}.periodOffset(-1) )
+ *
+ * + *

This would generate the following query logic (again, simplified here) when evaluated for the + * two months 202309 and 202310: + * + * 

+ * select uidlevel2, monthly, 'subExpreUid' as dx, sum("A2VfEfPflHV" + "A2VfEfPflHV_minus_1") as value
+ * from (select uidlevel2,
+ *              shift.reportperiod as monthly,
+ *              sum(case when dx = 'A2VfEfPflHV' and shift.delta = 0 then value else null end) as "A2VfEfPflHV",
+ *              sum(case when dx = 'A2VfEfPflHV' and shift.delta = -1 then value else null end) as "A2VfEfPflHV_minus_1"
+ *       from analytics
+ *       join (values(-1,'202309','202308'),(-1,'202310','202309'),
+ *                   (0,'202309','202309'),(0,'202310','202310'))
+ *              as shift (delta, reportperiod, dataperiod) on dataperiod = monthly
+ *       where monthly in ('202308', '202309', '202310')
+ *       and dx in ('A2VfEfPflHV') // (greatly improves performance)
+ *       group by uidlevel2, monthly, ou) as ax
+ * where "A2VfEfPflHV" is not null
+ * group by uidlevel2, monthly;
+ *
+ * * @author Jim Grace */ public class JdbcSubexpressionQueryGenerator { @@ -102,6 +146,9 @@ public class JdbcSubexpressionQueryGenerator { /** The subexpression being processed, from the parameters. */ private final SubexpressionDimensionItem subex; + /** Whether this subexpression has any period offsets. */ + private final boolean hasPeriodOffsets; + public JdbcSubexpressionQueryGenerator( JdbcAnalyticsManager jam, DataQueryParams params, AnalyticsTableType tableType) { this.jam = jam; @@ -110,6 +157,7 @@ public JdbcSubexpressionQueryGenerator( this.paramsWithoutData = DataQueryParams.newBuilder(params).removeDimension(DATA_X_DIM_ID).build(); this.subex = params.getSubexpression(); + this.hasPeriodOffsets = subex.hasPeriodOffsets(); } /** @@ -163,6 +211,8 @@ private String getFrom() { String fromSub = "from " + jam.getFromSourceClause(params) + " as " + ANALYTICS_TBL_ALIAS + " "; + String joinSub = (hasPeriodOffsets) ? joinPeriodOffsetValues(params) : ""; + String whereSub = getWhereSubquery(); String groupBySub = getGroupBySubquery(); @@ -170,6 +220,7 @@ private String getFrom() { return "from (" + selectSub + fromSub + + joinSub + whereSub + groupBySub + ") as " @@ -180,7 +231,9 @@ private String getFrom() { /** Gets the subquery select clause. */ private String getSelectSubquery() { String dimensionColumns = - jam.getCommaDelimitedQuotedDimensionColumns(paramsWithoutData.getDimensions()); + (hasPeriodOffsets) + ? getPeriodOffsetSelectDimensionColumns() + : jam.getCommaDelimitedQuotedDimensionColumns(paramsWithoutData.getDimensions()); String subexItemColumns = subex.getItems().stream().map(this::getItemSql).distinct().collect(joining(",")); @@ -188,9 +241,25 @@ private String getSelectSubquery() { return "select " + dimensionColumns + ", " + subexItemColumns + " "; } + private String getPeriodOffsetSelectDimensionColumns() { + String nonPeriodDimensions = + jam.getCommaDelimitedQuotedDimensionColumns(paramsWithoutData.getNonPeriodDimensions()); + return SHIFT + + "." + + REPORTPERIOD + + " as " + + paramsWithoutData.getPeriodType().toLowerCase() + + (nonPeriodDimensions.isEmpty() ? "" : "," + nonPeriodDimensions); + } + /** Gets the subquery where clause. */ private String getWhereSubquery() { - String sql = jam.getWhereClause(paramsWithoutData, tableType); + DataQueryParams whereSubQueryParams = + hasPeriodOffsets ? getParamsWithOffsetPeriodsWithoutData(params) : paramsWithoutData; + + whereSubQueryParams = getParamsWithPeriodType(whereSubQueryParams); + + String sql = jam.getWhereClause(whereSubQueryParams, tableType); if (!sql.isEmpty()) { sql += "and "; @@ -203,13 +272,41 @@ private String getWhereSubquery() { /** Gets the subquery group by clause. */ private String getGroupBySubquery() { - List cols = jam.getQuotedDimensionColumns(paramsWithoutData.getDimensions()); + DataQueryParams groupByParams = + hasPeriodOffsets + ? DataQueryParams.newBuilder(paramsWithoutData).removeDimension(PERIOD_DIM_ID).build() + : getParamsWithPeriodType(paramsWithoutData); + + List cols = jam.getQuotedDimensionColumns(groupByParams.getDimensions()); addUnique(cols, quote(ANALYTICS_TBL_ALIAS, OU)); + if (hasPeriodOffsets) { + cols.add(SHIFT + "." + REPORTPERIOD); + } + return " group by " + join(",", cols); } + /** + * Gets parameters where the period type will be the selected column for the period (so a query + * for this column will also data from enclosed, shorter periods). + */ + private DataQueryParams getParamsWithPeriodType(DataQueryParams query) { + String periodType = query.getPeriodType(); + if (periodType != null) { + BaseDimensionalObject periodDim = + new BaseDimensionalObject( + DimensionalObject.PERIOD_DIM_ID, + DimensionType.PERIOD, + periodType.toLowerCase(), + null, + query.getPeriods()); + query = DataQueryParams.newBuilder(query).replaceDimension(periodDim).build(); + } + return query; + } + /** Gets a comma-separated list of the quoted UIDs of the data elements in the subexpression. */ private String getSubexpressionDataElementList() { return subex.getItems().stream() @@ -255,15 +352,18 @@ private String getItemSql(DimensionalItemObject item) { : " and " + quote(ANALYTICS_TBL_ALIAS, CO) + "='" + cocUid + "'") + (isEmpty(aocUid) ? "" - : " and " + quote(ANALYTICS_TBL_ALIAS, AO) + "='" + aocUid + "'"); + : " and " + quote(ANALYTICS_TBL_ALIAS, AO) + "='" + aocUid + "'") + + (hasPeriodOffsets + ? " and " + SHIFT + "." + DELTA + " = " + item.getPeriodOffset() + : ""); String value = quote(dataElement.getValueColumn()); - String cast = - (dataElement.getValueType().isBoolean() - && dataElement.getAggregationType().allowsNonnumeric()) - ? "::int::bool" - : ""; + DataType dataType = fromValueType(dataElement.getValueType()); + if (dataType == BOOLEAN) { + dataType = NUMERIC; // Booleans are always aggregated as numeric in subex. + } + String cast = castSql("", dataType); String column = getItemColumnName(deUid, cocUid, aocUid, dataElement.getQueryMods()); @@ -272,8 +372,8 @@ private String getItemSql(DimensionalItemObject item) { + conditional + " then " + value - + " else null end)" + cast + + " else null end)" + " as " + column; } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/SubexpressionPeriodOffsetUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/SubexpressionPeriodOffsetUtils.java new file mode 100644 index 000000000000..c062eb239b2c --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/SubexpressionPeriodOffsetUtils.java @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.data; + +import static java.lang.String.format; +import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quote; +import static org.hisp.dhis.analytics.util.PeriodOffsetUtils.shiftPeriod; +import static org.hisp.dhis.common.DimensionalObject.DATA_X_DIM_ID; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.hisp.dhis.analytics.DataQueryParams; +import org.hisp.dhis.common.BaseDimensionalObject; +import org.hisp.dhis.common.DimensionType; +import org.hisp.dhis.common.DimensionalItemObject; +import org.hisp.dhis.common.DimensionalObject; +import org.hisp.dhis.period.Period; + +/** + * Utility methods for generating a subExpression query containing periodOffsets. + * + *

These methods help in the translation between report periods (the periods reported to the + * user) and data periods (the database periods in which data is fetched). For example a + * periodOffset of -1 for a reporting period of '202310' (Oct 2023) means that the data comes from + * data period '202309' (Sep 2023). + * + * @author Jim Grace + */ +public class SubexpressionPeriodOffsetUtils { + private SubexpressionPeriodOffsetUtils() { + throw new UnsupportedOperationException("util"); + } + + static final String SHIFT = "shift"; + + static final String DELTA = quote("delta"); + + static final String REPORTPERIOD = quote("reportperiod"); + + private static final String DATAPERIOD = quote("dataperiod"); + + /** + * For {@link DataQueryParams} containing a subexpression with periodOffsets, joins an inline + * value table that maps the reporting periods to data periods, for each distinct periodOffset + * value. + * + *

For example, for periods '202309' and '202310' and periods offsets -1 and 0, this would + * generate: + * + * 

+   *       join (values(-1,'202309','202308'),(-1,'202310','202309'),
+   *                   (0,'202309','202309'),(0,'202310','202310'))
+   *       as shift (delta, reportperiod, dataperiod) on dataperiod = "monthly"
+   *
+ * + * @param params parameters with reporting periods + * @return a join clause to an inline table to map reporting periods to data periods + */ + protected static String joinPeriodOffsetValues(DataQueryParams params) { + List reportPeriods = getReportPeriods(params); + List periodOffsets = getPeriodOffsets(params); + + StringBuilder sb = new StringBuilder(" join (values"); + + for (Integer delta : periodOffsets) { + for (Period reportPeriod : reportPeriods) { + Period dataPeriod = shiftPeriod(reportPeriod, delta); + sb.append( + format("(%s,'%s','%s'),", delta, reportPeriod.getIsoDate(), dataPeriod.getIsoDate())); + } + } + sb.setLength(sb.length() - 1); // Remove final "," + + sb.append( + format( + ") as %s (%s, %s, %s) on %s = %s", + SHIFT, + DELTA, + REPORTPERIOD, + DATAPERIOD, + DATAPERIOD, + quote(params.getPeriodType().toLowerCase()))); + + return sb.toString(); + } + + /** + * For {@link DataQueryParams} containing a subexpression with periodOffsets, returns the + * parameters with data periods and without data dimension items. + * + * @param params parameters with reporting periods + * @return parameters with data periods and without data dimension items + */ + protected static DataQueryParams getParamsWithOffsetPeriodsWithoutData(DataQueryParams params) { + return DataQueryParams.newBuilder(getParamsWithOffsetPeriods(params)) + .removeDimension(DATA_X_DIM_ID) + .build(); + } + + /** + * For {@link DataQueryParams} containing a subexpression with periodOffsets, replaces the + * reporting periods in the parameters (the periods to return to the user) with data periods (the + * periods to fetch data from the database, according to the periodOffsets). + * + * @param params parameters with reporting periods + * @return parameters with data periods + */ + protected static DataQueryParams getParamsWithOffsetPeriods(DataQueryParams params) { + List reportPeriods = getReportPeriods(params); + List periodOffsets = getPeriodOffsets(params); + + Set shiftedPeriods = new HashSet<>(); + for (Period reportPeriod : reportPeriods) { + for (Integer periodOffset : periodOffsets) { + shiftedPeriods.add(shiftPeriod(reportPeriod, periodOffset)); + } + } + + List dataPeriods = + shiftedPeriods.stream() + .sorted() // Useful for testing, troubleshooting, etc. + .map(DimensionalItemObject.class::cast) + .toList(); + + DimensionalObject periodDimension = + new BaseDimensionalObject( + DimensionalObject.PERIOD_DIM_ID, DimensionType.PERIOD, dataPeriods); + + return DataQueryParams.newBuilder(params).replaceDimension(periodDimension).build(); + } + + // ------------------------------------------------------------------------- + // Supportive methods + // ------------------------------------------------------------------------- + + /** Gets the report periods from paramms as a list of Periods. */ + private static List getReportPeriods(DataQueryParams params) { + return params.getPeriods().stream().map(Period.class::cast).toList(); + } + + /** Gets the sorted, distinct period offsets from items within the subexpression. */ + private static List getPeriodOffsets(DataQueryParams params) { + if (!params.hasSubexpressions()) { + params.getSubexpression(); + } + if (params.getSubexpression().getItems().isEmpty()) { + params.getSubexpression(); + } + return params.getSubexpression().getItems().stream() + .map(DimensionalItemObject::getPeriodOffset) + .distinct() + .sorted() + .toList(); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java index 6899bef6beca..821ae3dff6f1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java @@ -62,6 +62,7 @@ import static org.hisp.dhis.analytics.util.AnalyticsUtils.getRoundedValueObject; import static org.hisp.dhis.analytics.util.AnalyticsUtils.hasPeriod; import static org.hisp.dhis.analytics.util.AnalyticsUtils.isPeriodInPeriods; +import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; import static org.hisp.dhis.analytics.util.PeriodOffsetUtils.buildYearToDateRows; import static org.hisp.dhis.analytics.util.PeriodOffsetUtils.getPeriodOffsetRow; import static org.hisp.dhis.analytics.util.PeriodOffsetUtils.isYearToDate; @@ -547,7 +548,8 @@ public void addRawData(DataQueryParams params, Grid grid) { params = queryPlanner.withTableNameAndPartitions(params, plannerParams); - rawAnalyticsManager.getRawDataValues(params, grid); + final DataQueryParams immutableParams = DataQueryParams.newBuilder(params).build(); + withExceptionHandling(() -> rawAnalyticsManager.getRawDataValues(immutableParams, grid)); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/MetadataHandler.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/MetadataHandler.java index fb0fb81e1707..5e3208c47c52 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/MetadataHandler.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/MetadataHandler.java @@ -56,11 +56,13 @@ import org.hisp.dhis.analytics.DataQueryParams; import org.hisp.dhis.analytics.DataQueryService; import org.hisp.dhis.analytics.orgunit.OrgUnitHelper; +import org.hisp.dhis.analytics.util.AnalyticsOrganisationUnitUtils; import org.hisp.dhis.calendar.Calendar; import org.hisp.dhis.common.DimensionalObject; import org.hisp.dhis.common.Grid; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.period.PeriodType; +import org.hisp.dhis.user.CurrentUserService; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; @@ -72,6 +74,8 @@ public class MetadataHandler { private final SchemeIdResponseMapper schemeIdResponseMapper; + private final CurrentUserService currentUserService; + /** * Adds meta data values to the given grid based on the given data query parameters. * @@ -88,13 +92,18 @@ public void addMetaData(DataQueryParams params, Grid grid) { // Items / names element // ----------------------------------------------------------------- - Map cocNameMap = getCocNameMap(params); + Map items = new HashMap<>(getDimensionMetadataItemMap(params, grid)); - metaData.put(ITEMS.getKey(), getDimensionMetadataItemMap(params, grid)); + AnalyticsOrganisationUnitUtils.getUserOrganisationUnitItems( + currentUserService.getCurrentUser(), params.getUserOrganisationUnitsCriteria()) + .forEach(items::putAll); + + metaData.put(ITEMS.getKey(), items); // ----------------------------------------------------------------- // Item order elements // ----------------------------------------------------------------- + Map cocNameMap = getCocNameMap(params); Map dimensionItems = new HashMap<>(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EnrollmentAnalyticsManager.java index 31a5460709f5..1313198dc408 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EnrollmentAnalyticsManager.java @@ -54,7 +54,7 @@ public interface EnrollmentAnalyticsManager { void getEnrollments(EventQueryParams params, Grid grid, int maxLimit); /** - * Retreives count of enrollments based on params. + * Retrieves count of enrollments based on params. * * @param params the qyery to count enrollments for, * @return number of enrollments macting the parameter criteria. diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java index 60adf60a0f33..b339f31f9248 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java @@ -94,6 +94,7 @@ import org.hisp.dhis.program.ProgramStatus; import org.hisp.dhis.program.ProgramTrackedEntityAttributeDimensionItem; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; +import org.hisp.dhis.util.OrganisationUnitCriteriaUtils; /** * Class representing query parameters for retrieving event data from the event analytics service. @@ -232,6 +233,8 @@ public class EventQueryParams extends DataQueryParams { @Getter protected EndpointAction endpointAction; + @Getter protected boolean multipleQueries = false; + // ------------------------------------------------------------------------- // Constructors // ------------------------------------------------------------------------- @@ -300,6 +303,8 @@ protected EventQueryParams instance() { params.endpointItem = this.endpointItem; params.endpointAction = this.endpointAction; params.rowContext = this.rowContext; + params.multipleQueries = this.multipleQueries; + params.userOrganisationUnitsCriteria = this.userOrganisationUnitsCriteria; return params; } @@ -1253,6 +1258,13 @@ public Builder withOrgUnitField(OrgUnitField orgUnitField) { return this; } + public Builder withUserOrganisationUnitsCriteria(String userOrganisationUnitsCriteria) { + + this.params.userOrganisationUnitsCriteria = + OrganisationUnitCriteriaUtils.getAnalyticsMetaDataKeys(userOrganisationUnitsCriteria); + return this; + } + public Builder withClusterSize(Long clusterSize) { this.params.clusterSize = clusterSize; return this; @@ -1343,5 +1355,10 @@ public Builder withRowContext(boolean rowContext) { this.params.rowContext = rowContext; return this; } + + public Builder withMultipleQueries(boolean multipleQueries) { + this.params.multipleQueries = multipleQueries; + return this; + } } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractAnalyticsService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractAnalyticsService.java index 1548565ec9d3..071e76fc4b69 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractAnalyticsService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractAnalyticsService.java @@ -68,6 +68,7 @@ import org.hisp.dhis.analytics.event.EventQueryParams; import org.hisp.dhis.analytics.event.EventQueryValidator; import org.hisp.dhis.analytics.orgunit.OrgUnitHelper; +import org.hisp.dhis.analytics.util.AnalyticsOrganisationUnitUtils; import org.hisp.dhis.analytics.util.AnalyticsUtils; import org.hisp.dhis.calendar.Calendar; import org.hisp.dhis.common.DimensionItemKeywords; @@ -88,6 +89,7 @@ import org.hisp.dhis.option.Option; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.period.PeriodType; +import org.hisp.dhis.user.CurrentUserService; import org.hisp.dhis.user.User; /** @@ -101,6 +103,8 @@ public abstract class AbstractAnalyticsService { protected final SchemeIdResponseMapper schemeIdResponseMapper; + private final CurrentUserService currentUserService; + /** * Returns a grid based on the given query. * @@ -383,20 +387,28 @@ protected void addMetadata( if (hasResults) { optionItems.addAll( - optionsPresentInGrid.values().stream() - .flatMap(Collection::stream) - .distinct() - .collect(toList())); + optionsPresentInGrid.values().stream().flatMap(Collection::stream).distinct().toList()); } else { optionItems.addAll(getItemOptionsAsFilter(params.getItemOptions(), params.getItems())); } + Map items = new HashMap<>(); + AnalyticsOrganisationUnitUtils.getUserOrganisationUnitItems( + currentUserService.getCurrentUser(), params.getUserOrganisationUnitsCriteria()) + .forEach(items::putAll); + + if (params.isComingFromQuery()) { + items.putAll(getMetadataItems(params, periodKeywords, optionItems, grid)); + } else { + items.putAll(getMetadataItems(params)); + } + + metadata.put(ITEMS.getKey(), items); + if (params.isComingFromQuery()) { - metadata.put(ITEMS.getKey(), getMetadataItems(params, periodKeywords, optionItems, grid)); metadata.put( DIMENSIONS.getKey(), getDimensionItems(params, Optional.of(optionsPresentInGrid))); } else { - metadata.put(ITEMS.getKey(), getMetadataItems(params)); metadata.put(DIMENSIONS.getKey(), getDimensionItems(params, empty())); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 762859f518bc..aec16d20e0c1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -54,6 +54,7 @@ import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quote; import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quoteAlias; import static org.hisp.dhis.analytics.util.AnalyticsUtils.throwIllegalQueryEx; +import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; import static org.hisp.dhis.common.DimensionItemType.PROGRAM_INDICATOR; import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; @@ -63,6 +64,7 @@ import static org.hisp.dhis.common.RequestTypeAware.EndpointItem.ENROLLMENT; import static org.hisp.dhis.commons.util.TextUtils.getCommaDelimitedString; import static org.hisp.dhis.system.util.MathUtils.getRounded; +import static org.springframework.transaction.annotation.Propagation.REQUIRES_NEW; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -106,7 +108,6 @@ import org.hisp.dhis.common.OrganisationUnitSelectionMode; import org.hisp.dhis.common.QueryFilter; import org.hisp.dhis.common.QueryItem; -import org.hisp.dhis.common.QueryRuntimeException; import org.hisp.dhis.common.Reference; import org.hisp.dhis.common.RepeatableStageParams; import org.hisp.dhis.common.ValueType; @@ -122,10 +123,9 @@ import org.hisp.dhis.program.ProgramIndicatorService; import org.hisp.dhis.system.util.MathUtils; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.dao.DataAccessResourceFailureException; -import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.support.rowset.SqlRowSet; +import org.springframework.transaction.annotation.Transactional; /** * @author Markus Bekken @@ -155,7 +155,7 @@ public abstract class AbstractJdbcEventAnalyticsManager { private static final Collector AND_JOINER = joining(AND); - @Qualifier("readOnlyJdbcTemplate") + @Qualifier("analyticsReadOnlyJdbcTemplate") protected final JdbcTemplate jdbcTemplate; protected final ProgramIndicatorService programIndicatorService; @@ -510,6 +510,7 @@ protected Optional getAlias(QueryItem queryItem) { .map(RepeatableStageParams::getDimension); } + @Transactional(readOnly = true, propagation = REQUIRES_NEW) public Grid getAggregatedEventData(EventQueryParams params, Grid grid, int maxLimit) { String aggregateClause = getAggregateClause(params); @@ -553,18 +554,14 @@ public Grid getAggregatedEventData(EventQueryParams params, Grid grid, int maxLi // Grid // --------------------------------------------------------------------- - try { - if (params.analyzeOnly()) { - executionPlanStore.addExecutionPlan(params.getExplainOrderId(), sql); - } else { - getAggregatedEventData(grid, params, sql); - } - } catch (BadSqlGrammarException ex) { - log.info(AnalyticsUtils.ERR_MSG_TABLE_NOT_EXISTING, ex); - throw ex; - } catch (DataAccessResourceFailureException ex) { - log.warn(ErrorCode.E7131.getMessage(), ex); - throw new QueryRuntimeException(ErrorCode.E7131); + final String finalSqlValue = sql; + + if (params.analyzeOnly()) { + withExceptionHandling( + () -> executionPlanStore.addExecutionPlan(params.getExplainOrderId(), finalSqlValue)); + } else { + withExceptionHandling( + () -> getAggregatedEventData(grid, params, finalSqlValue), params.isMultipleQueries()); } return grid; @@ -978,8 +975,6 @@ protected String getAggregatedEnrollmentsSql(List headers, EventQuer OUTER_SQL_ALIAS + "." + ((Period) it).getPeriodType().getPeriodTypeEnum().getName()) - .toList() - .stream() .distinct()) .collect(joining(",")); @@ -1005,24 +1000,6 @@ protected String getAggregatedEnrollmentsSql(List headers, EventQuer return sql; } - /** - * Wraps the provided interface around a common exception handling strategy. - * - * @param runnable the {@link Runnable} containing the code block to execute and wrap around the - * exception handling. - */ - protected void withExceptionHandling(Runnable runnable) { - try { - runnable.run(); - } catch (BadSqlGrammarException ex) { - log.info(AnalyticsUtils.ERR_MSG_TABLE_NOT_EXISTING, ex); - throw ex; - } catch (DataAccessResourceFailureException ex) { - log.warn(ErrorCode.E7131.getMessage(), ex); - throw new QueryRuntimeException(ErrorCode.E7131); - } - } - /** * Adds a value from the given row set to the grid. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEnrollmentAnalyticsService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEnrollmentAnalyticsService.java index 3982454c6f09..994b492a0221 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEnrollmentAnalyticsService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEnrollmentAnalyticsService.java @@ -49,6 +49,7 @@ import org.hisp.dhis.common.GridHeader; import org.hisp.dhis.common.RequestTypeAware; import org.hisp.dhis.system.grid.ListGrid; +import org.hisp.dhis.user.CurrentUserService; import org.hisp.dhis.util.Timer; import org.springframework.stereotype.Service; @@ -97,8 +98,9 @@ public DefaultEnrollmentAnalyticsService( AnalyticsSecurityManager securityManager, EventQueryPlanner queryPlanner, EventQueryValidator queryValidator, - SchemeIdResponseMapper schemeIdResponseMapper) { - super(securityManager, queryValidator, schemeIdResponseMapper); + SchemeIdResponseMapper schemeIdResponseMapper, + CurrentUserService currentUserService) { + super(securityManager, queryValidator, schemeIdResponseMapper, currentUserService); checkNotNull(enrollmentAnalyticsManager); checkNotNull(queryPlanner); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java index d23584396e31..9743044c660b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java @@ -96,6 +96,7 @@ import org.hisp.dhis.system.database.DatabaseInfo; import org.hisp.dhis.system.grid.ListGrid; import org.hisp.dhis.trackedentity.TrackedEntityAttributeService; +import org.hisp.dhis.user.CurrentUserService; import org.hisp.dhis.util.Timer; import org.springframework.stereotype.Service; @@ -197,8 +198,9 @@ public DefaultEventAnalyticsService( DatabaseInfo databaseInfo, AnalyticsCache analyticsCache, EnrollmentAnalyticsManager enrollmentAnalyticsManager, - SchemeIdResponseMapper schemeIdResponseMapper) { - super(securityManager, queryValidator, schemeIdResponseMapper); + SchemeIdResponseMapper schemeIdResponseMapper, + CurrentUserService currentUserService) { + super(securityManager, queryValidator, schemeIdResponseMapper, currentUserService); checkNotNull(dataElementService); checkNotNull(trackedEntityAttributeService); @@ -784,12 +786,14 @@ protected long addData(Grid grid, EventQueryParams params) { timer.getSplitTime("Planned event query, got partitions: " + params.getPartitions()); long count = 0; + EventQueryParams immutableParams = new EventQueryParams.Builder(params).build(); if (params.getPartitions().hasAny() || params.isSkipPartitioning()) { - eventAnalyticsManager.getEvents(params, grid, queryValidator.getMaxLimit()); + + eventAnalyticsManager.getEvents(immutableParams, grid, queryValidator.getMaxLimit()); if (params.isPaging() && params.isTotalPages()) { - count = eventAnalyticsManager.getEventCount(params); + count = eventAnalyticsManager.getEventCount(immutableParams); } timer.getTime("Got events " + grid.getHeight()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryService.java index 82ee4887f299..c6926134e144 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryService.java @@ -226,6 +226,7 @@ public EventQueryParams getFromRequest(EventDataQueryRequest request, boolean an .withEnhancedConditions(request.isEnhancedConditions()) .withEndpointItem(request.getEndpointItem()) .withEndpointAction(request.getEndpointAction()) + .withUserOrganisationUnitsCriteria(request.getUserOrganisationUnitCriteria()) .withRowContext(request.isRowContext()); if (analyzeOnly) { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java index 67ce35b87f59..0e5ec99fec0d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java @@ -144,10 +144,30 @@ private EventQueryParams withTableNameAndPartitions(EventQueryParams params) { */ private List withTableNameAndPartitions(List queries) { List list = new ArrayList<>(); - queries.forEach(query -> list.add(withTableNameAndPartitions(query))); + + boolean isMultipleQueries = queries.size() > 1; + + queries.forEach( + query -> + list.add(withMultipleQueries(isMultipleQueries, withTableNameAndPartitions(query)))); + return list; } + /** + * Sets the "multipleQueries" flag in EventParams and builds it + * + * @param isMultipleQueries flag to detect if multiple queries are to be run + * @param eventQueryParams the eventQueryParams template + * @return an eventQueryParams with proper "multipleQueries" flag set + */ + private EventQueryParams withMultipleQueries( + boolean isMultipleQueries, EventQueryParams eventQueryParams) { + return new EventQueryParams.Builder(eventQueryParams) + .withMultipleQueries(isMultipleQueries) + .build(); + } + /** * Groups by organisation unit level. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 64a36dcb9f6c..672bfdf83ce6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -36,6 +36,7 @@ import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.getCoalesce; import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quote; import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quoteAlias; +import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; @@ -55,7 +56,6 @@ import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; import org.hisp.dhis.analytics.event.EnrollmentAnalyticsManager; import org.hisp.dhis.analytics.event.EventQueryParams; -import org.hisp.dhis.analytics.util.AnalyticsUtils; import org.hisp.dhis.common.DimensionType; import org.hisp.dhis.common.DimensionalItemObject; import org.hisp.dhis.common.DimensionalObject; @@ -63,21 +63,18 @@ import org.hisp.dhis.common.Grid; import org.hisp.dhis.common.OrganisationUnitSelectionMode; import org.hisp.dhis.common.QueryItem; -import org.hisp.dhis.common.QueryRuntimeException; import org.hisp.dhis.common.ValueStatus; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.commons.collection.ListUtils; import org.hisp.dhis.commons.util.ExpressionUtils; import org.hisp.dhis.commons.util.SqlHelper; -import org.hisp.dhis.feedback.ErrorCode; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.program.AnalyticsType; import org.hisp.dhis.program.ProgramIndicatorService; import org.hisp.dhis.system.util.SqlUtils; import org.hisp.dhis.util.DateUtils; import org.locationtech.jts.util.Assert; -import org.springframework.dao.DataAccessResourceFailureException; -import org.springframework.jdbc.BadSqlGrammarException; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.InvalidResultSetAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.support.rowset.SqlRowSet; @@ -119,7 +116,7 @@ public class JdbcEnrollmentAnalyticsManager extends AbstractJdbcEventAnalyticsMa "enrollmentstatus"); public JdbcEnrollmentAnalyticsManager( - JdbcTemplate jdbcTemplate, + @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, ProgramIndicatorService programIndicatorService, ProgramIndicatorSubqueryBuilder programIndicatorSubqueryBuilder, EnrollmentTimeFieldSqlRenderer timeFieldSqlRenderer, @@ -137,9 +134,11 @@ public void getEnrollments(EventQueryParams params, Grid grid, int maxLimit) { : getAggregatedEnrollmentsSql(params, maxLimit); if (params.analyzeOnly()) { - executionPlanStore.addExecutionPlan(params.getExplainOrderId(), sql); + withExceptionHandling( + () -> executionPlanStore.addExecutionPlan(params.getExplainOrderId(), sql)); } else { - withExceptionHandling(() -> getEnrollments(params, grid, sql, maxLimit == 0)); + withExceptionHandling( + () -> getEnrollments(params, grid, sql, maxLimit == 0), params.isMultipleQueries()); } } @@ -252,20 +251,19 @@ public long getEnrollmentCount(EventQueryParams params) { long count = 0; - try { - log.debug("Analytics enrollment count SQL: " + sql); + log.debug("Analytics enrollment count SQL: " + sql); - if (params.analyzeOnly()) { - executionPlanStore.addExecutionPlan(params.getExplainOrderId(), sql); - } else { - count = jdbcTemplate.queryForObject(sql, Long.class); - } - } catch (BadSqlGrammarException ex) { - log.info(AnalyticsUtils.ERR_MSG_TABLE_NOT_EXISTING, ex); - throw ex; - } catch (DataAccessResourceFailureException ex) { - log.warn(ErrorCode.E7131.getMessage(), ex); - throw new QueryRuntimeException(ErrorCode.E7131); + final String finalSqlValue = sql; + + if (params.analyzeOnly()) { + withExceptionHandling( + () -> executionPlanStore.addExecutionPlan(params.getExplainOrderId(), finalSqlValue)); + } else { + count = + withExceptionHandling( + () -> jdbcTemplate.queryForObject(finalSqlValue, Long.class), + params.isMultipleQueries()) + .orElse(0l); } return count; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java index 4d8ddcc28ec0..710eab35adf6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java @@ -43,6 +43,7 @@ import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quote; import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quoteAlias; import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quoteAliasCommaSeparate; +import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString; @@ -72,7 +73,6 @@ import org.hisp.dhis.analytics.event.EventAnalyticsManager; import org.hisp.dhis.analytics.event.EventQueryParams; import org.hisp.dhis.analytics.util.AnalyticsSqlUtils; -import org.hisp.dhis.analytics.util.AnalyticsUtils; import org.hisp.dhis.common.DimensionType; import org.hisp.dhis.common.DimensionalItemObject; import org.hisp.dhis.common.DimensionalObject; @@ -91,9 +91,9 @@ import org.hisp.dhis.program.AnalyticsType; import org.hisp.dhis.program.ProgramIndicatorService; import org.postgresql.util.PSQLException; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessResourceFailureException; import org.springframework.dao.DataIntegrityViolationException; -import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.support.rowset.SqlRowSet; import org.springframework.stereotype.Service; @@ -114,7 +114,7 @@ public class JdbcEventAnalyticsManager extends AbstractJdbcEventAnalyticsManager private final EventTimeFieldSqlRenderer timeFieldSqlRenderer; public JdbcEventAnalyticsManager( - JdbcTemplate jdbcTemplate, + @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, ProgramIndicatorService programIndicatorService, ProgramIndicatorSubqueryBuilder programIndicatorSubqueryBuilder, EventTimeFieldSqlRenderer timeFieldSqlRenderer, @@ -129,9 +129,11 @@ public Grid getEvents(EventQueryParams params, Grid grid, int maxLimit) { String sql = getAggregatedEnrollmentsSql(params, maxLimit); if (params.analyzeOnly()) { - executionPlanStore.addExecutionPlan(params.getExplainOrderId(), sql); + withExceptionHandling( + () -> executionPlanStore.addExecutionPlan(params.getExplainOrderId(), sql)); } else { - withExceptionHandling(() -> getEvents(params, grid, sql, maxLimit == 0)); + withExceptionHandling( + () -> getEvents(params, grid, sql, maxLimit == 0), params.isMultipleQueries()); } return grid; @@ -239,23 +241,18 @@ public long getEventCount(EventQueryParams params) { long count = 0; - try { - log.debug("Analytics event count SQL: '{}'", sql); + log.debug("Analytics event count SQL: '{}'", sql); - if (params.analyzeOnly()) { - executionPlanStore.addExecutionPlan(params.getExplainOrderId(), sql); - } else { - count = jdbcTemplate.queryForObject(sql, Long.class); - } - } catch (BadSqlGrammarException ex) { - log.info(AnalyticsUtils.ERR_MSG_TABLE_NOT_EXISTING, ex); - throw ex; - } catch (DataAccessResourceFailureException ex) { - log.warn(E7131.getMessage(), ex); - throw new QueryRuntimeException(E7131); - } catch (DataIntegrityViolationException ex) { - log.warn(E7132.getMessage(), ex); - throw new QueryRuntimeException(E7132); + final String finalSqlValue = sql; + + if (params.analyzeOnly()) { + withExceptionHandling( + () -> executionPlanStore.addExecutionPlan(params.getExplainOrderId(), finalSqlValue), + params.isMultipleQueries()); + } else { + count = + withExceptionHandling(() -> jdbcTemplate.queryForObject(finalSqlValue, Long.class)) + .orElse(0l); } return count; @@ -281,7 +278,9 @@ public Rectangle getRectangle(EventQueryParams params) { Rectangle rectangle = new Rectangle(); - SqlRowSet rowSet = queryForRows(sql); + final String finalSqlValue = sql; + + SqlRowSet rowSet = withExceptionHandling(() -> queryForRows(finalSqlValue)).get(); if (rowSet.next()) { Object extent = rowSet.getObject(COL_EXTENT); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/orgunit/data/JdbcOrgUnitAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/orgunit/data/JdbcOrgUnitAnalyticsManager.java index 988b03d3a152..75c59d0e7ec6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/orgunit/data/JdbcOrgUnitAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/orgunit/data/JdbcOrgUnitAnalyticsManager.java @@ -49,6 +49,7 @@ import org.hisp.dhis.feedback.ErrorMessage; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.organisationunit.OrganisationUnitGroupSet; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.support.rowset.SqlRowSet; import org.springframework.stereotype.Service; @@ -60,6 +61,8 @@ @RequiredArgsConstructor public class JdbcOrgUnitAnalyticsManager implements OrgUnitAnalyticsManager { private final TableInfoReader tableInfoReader; + + @Qualifier("analyticsJdbcTemplate") private final JdbcTemplate jdbcTemplate; @Override diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/partition/JdbcPartitionManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/partition/JdbcPartitionManager.java index a91114ac6b83..b196dc84b3f9 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/partition/JdbcPartitionManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/partition/JdbcPartitionManager.java @@ -38,6 +38,7 @@ import org.hisp.dhis.analytics.Partitions; import org.hisp.dhis.analytics.table.PartitionUtils; import org.hisp.dhis.common.event.ApplicationCacheClearedEvent; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.event.EventListener; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; @@ -51,6 +52,7 @@ public class JdbcPartitionManager implements PartitionManager { private Map> analyticsPartitions = new HashMap<>(); + @Qualifier("analyticsJdbcTemplate") private final JdbcTemplate jdbcTemplate; @Override diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java index c3e467ae8ee8..40752f7591db 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java @@ -80,6 +80,7 @@ import org.hisp.dhis.system.util.MathUtils; import org.hisp.dhis.util.DateUtils; import org.hisp.dhis.util.ObjectUtils; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -128,7 +129,7 @@ public JdbcAnalyticsTableManager( StatementBuilder statementBuilder, PartitionManager partitionManager, DatabaseInfo databaseInfo, - JdbcTemplate jdbcTemplate, + @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, AnalyticsExportSettings analyticsExportSettings, PeriodDataProvider periodDataProvider) { super( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java index c3c261cd1791..bd282659a920 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java @@ -67,6 +67,7 @@ import org.hisp.dhis.setting.SystemSettingManager; import org.hisp.dhis.system.database.DatabaseInfo; import org.hisp.dhis.util.DateUtils; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -87,7 +88,7 @@ public JdbcCompletenessTableManager( StatementBuilder statementBuilder, PartitionManager partitionManager, DatabaseInfo databaseInfo, - JdbcTemplate jdbcTemplate, + @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, AnalyticsExportSettings analyticsExportSettings, PeriodDataProvider periodDataProvider) { super( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java index edbbcfaae5d3..0e73ebeb91cc 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java @@ -62,6 +62,7 @@ import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingManager; import org.hisp.dhis.system.database.DatabaseInfo; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -91,7 +92,7 @@ public JdbcCompletenessTargetTableManager( StatementBuilder statementBuilder, PartitionManager partitionManager, DatabaseInfo databaseInfo, - JdbcTemplate jdbcTemplate, + @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, AnalyticsExportSettings analyticsExportSettings, PeriodDataProvider periodDataProvider) { super( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index 06162c6c459b..380890094e29 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -63,6 +63,7 @@ import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingManager; import org.hisp.dhis.system.database.DatabaseInfo; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -83,7 +84,7 @@ public JdbcEnrollmentAnalyticsTableManager( StatementBuilder statementBuilder, PartitionManager partitionManager, DatabaseInfo databaseInfo, - JdbcTemplate jdbcTemplate, + @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, AnalyticsExportSettings analyticsExportSettings, PeriodDataProvider periodDataProvider) { super( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index f62deb7d4d6d..9afd7d577b8b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -43,6 +43,8 @@ import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quote; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.analytics.util.DisplayNameUtils.getDisplayName; +import static org.hisp.dhis.period.PeriodDataProvider.DataSource.DATABASE; +import static org.hisp.dhis.period.PeriodDataProvider.DataSource.SYSTEM_DEFINED; import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; import static org.hisp.dhis.util.DateUtils.getLongDateString; @@ -83,6 +85,7 @@ import org.hisp.dhis.system.database.DatabaseInfo; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; import org.hisp.dhis.util.DateUtils; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -111,7 +114,7 @@ public JdbcEventAnalyticsTableManager( StatementBuilder statementBuilder, PartitionManager partitionManager, DatabaseInfo databaseInfo, - JdbcTemplate jdbcTemplate, + @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, AnalyticsExportSettings analyticsExportSettings, PeriodDataProvider periodDataProvider) { super( @@ -241,7 +244,9 @@ public List getAnalyticsTables(AnalyticsTableUpdateParams params "Get tables using earliest: %s, spatial support: %b", params.getFromDate(), databaseInfo.isSpatialSupport())); - List availableDataYears = periodDataProvider.getAvailableYears(); + List availableDataYears = + periodDataProvider.getAvailableYears( + analyticsExportSettings.getMaxPeriodYearsOffset() == null ? SYSTEM_DEFINED : DATABASE); return params.isLatestUpdate() ? getLatestAnalyticsTables(params) @@ -439,7 +444,9 @@ protected List getPartitionChecks(AnalyticsTablePartition partition) { @Override protected void populateTable( AnalyticsTableUpdateParams params, AnalyticsTablePartition partition) { - List availableDataYears = periodDataProvider.getAvailableYears(); + List availableDataYears = + periodDataProvider.getAvailableYears( + analyticsExportSettings.getMaxPeriodYearsOffset() == null ? SYSTEM_DEFINED : DATABASE); Integer firstDataYear = availableDataYears.get(0); Integer latestDataYear = availableDataYears.get(availableDataYears.size() - 1); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java index 425c658afcf7..7a876be2fa0d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java @@ -57,6 +57,7 @@ import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingManager; import org.hisp.dhis.system.database.DatabaseInfo; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -77,7 +78,7 @@ public JdbcOrgUnitTargetTableManager( StatementBuilder statementBuilder, PartitionManager partitionManager, DatabaseInfo databaseInfo, - JdbcTemplate jdbcTemplate, + @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, AnalyticsExportSettings analyticsExportSettings, PeriodDataProvider periodDataProvider) { super( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java index 57ff84252f30..be2333f65c91 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java @@ -66,6 +66,7 @@ import org.hisp.dhis.setting.SystemSettingManager; import org.hisp.dhis.system.database.DatabaseInfo; import org.hisp.quick.JdbcConfiguration; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -97,7 +98,7 @@ public JdbcOwnershipAnalyticsTableManager( StatementBuilder statementBuilder, PartitionManager partitionManager, DatabaseInfo databaseInfo, - JdbcTemplate jdbcTemplate, + @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, JdbcConfiguration jdbcConfiguration, AnalyticsExportSettings analyticsExportSettings, PeriodDataProvider periodDataProvider) { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiAnalyticsTableManager.java index 0d7705595da3..10cd0dea7a88 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiAnalyticsTableManager.java @@ -82,6 +82,7 @@ import org.hisp.dhis.trackedentity.TrackedEntityAttributeService; import org.hisp.dhis.trackedentity.TrackedEntityType; import org.hisp.dhis.trackedentity.TrackedEntityTypeService; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; @@ -107,7 +108,7 @@ public JdbcTeiAnalyticsTableManager( StatementBuilder statementBuilder, PartitionManager partitionManager, DatabaseInfo databaseInfo, - JdbcTemplate jdbcTemplate, + @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, TrackedEntityTypeService trackedEntityTypeService, TrackedEntityAttributeService trackedEntityAttributeService, AnalyticsExportSettings settings, diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEnrollmentsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEnrollmentsAnalyticsTableManager.java index 8774b2a633c6..6edfaef05b46 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEnrollmentsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEnrollmentsAnalyticsTableManager.java @@ -70,6 +70,7 @@ import org.hisp.dhis.setting.SystemSettingManager; import org.hisp.dhis.system.database.DatabaseInfo; import org.hisp.dhis.trackedentity.TrackedEntityTypeService; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; @@ -89,7 +90,7 @@ public JdbcTeiEnrollmentsAnalyticsTableManager( StatementBuilder statementBuilder, PartitionManager partitionManager, DatabaseInfo databaseInfo, - JdbcTemplate jdbcTemplate, + @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, TrackedEntityTypeService trackedEntityTypeService, AnalyticsExportSettings settings, PeriodDataProvider periodDataProvider) { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEventsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEventsAnalyticsTableManager.java index 5deca091540b..58d36e1d063e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEventsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEventsAnalyticsTableManager.java @@ -49,6 +49,8 @@ import static org.hisp.dhis.analytics.table.PartitionUtils.getStartDate; import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quote; import static org.hisp.dhis.commons.util.TextUtils.removeLastComma; +import static org.hisp.dhis.period.PeriodDataProvider.DataSource.DATABASE; +import static org.hisp.dhis.period.PeriodDataProvider.DataSource.SYSTEM_DEFINED; import static org.hisp.dhis.util.DateUtils.getLongDateString; import static org.hisp.dhis.util.DateUtils.getMediumDateString; import static org.springframework.util.Assert.notNull; @@ -79,6 +81,7 @@ import org.hisp.dhis.system.database.DatabaseInfo; import org.hisp.dhis.trackedentity.TrackedEntityType; import org.hisp.dhis.trackedentity.TrackedEntityTypeService; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; @@ -100,7 +103,7 @@ public JdbcTeiEventsAnalyticsTableManager( StatementBuilder statementBuilder, PartitionManager partitionManager, DatabaseInfo databaseInfo, - JdbcTemplate jdbcTemplate, + @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, TrackedEntityTypeService trackedEntityTypeService, AnalyticsExportSettings settings, PeriodDataProvider periodDataProvider) { @@ -228,7 +231,9 @@ private List getDataYears(AnalyticsTableUpdateParams params, TrackedEnt + "'"); } - List availableDataYears = periodDataProvider.getAvailableYears(); + List availableDataYears = + periodDataProvider.getAvailableYears( + analyticsExportSettings.getMaxPeriodYearsOffset() == null ? SYSTEM_DEFINED : DATABASE); Integer firstDataYear = availableDataYears.get(0); Integer latestDataYear = availableDataYears.get(availableDataYears.size() - 1); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java index 77828fd5cf27..b7a3a1aeec12 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java @@ -65,6 +65,7 @@ import org.hisp.dhis.setting.SystemSettingManager; import org.hisp.dhis.system.database.DatabaseInfo; import org.hisp.dhis.util.DateUtils; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; @@ -91,7 +92,7 @@ public JdbcValidationResultTableManager( StatementBuilder statementBuilder, PartitionManager partitionManager, DatabaseInfo databaseInfo, - JdbcTemplate jdbcTemplate, + @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, AnalyticsExportSettings analyticsExportSettings, PeriodDataProvider periodDataProvider) { super( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/TeiAnalyticsQueryService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/TeiAnalyticsQueryService.java index babdb159a63c..98368d797fd4 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/TeiAnalyticsQueryService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/TeiAnalyticsQueryService.java @@ -29,8 +29,7 @@ import static java.util.Collections.singleton; import static java.util.UUID.randomUUID; -import static org.hisp.dhis.analytics.util.AnalyticsUtils.ERR_MSG_TABLE_NOT_EXISTING; -import static org.hisp.dhis.feedback.ErrorCode.E7131; +import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; import static org.springframework.util.Assert.notNull; import java.util.List; @@ -47,11 +46,9 @@ import org.hisp.dhis.analytics.common.query.Field; import org.hisp.dhis.analytics.tei.query.context.sql.SqlQueryCreator; import org.hisp.dhis.analytics.tei.query.context.sql.SqlQueryCreatorService; +import org.hisp.dhis.common.ExecutionPlan; import org.hisp.dhis.common.Grid; -import org.hisp.dhis.common.QueryRuntimeException; import org.hisp.dhis.system.grid.ListGrid; -import org.springframework.dao.DataAccessResourceFailureException; -import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.stereotype.Service; /** @@ -92,23 +89,17 @@ public Grid getGrid(@Nonnull TeiQueryParams queryParams) { SqlQueryCreator queryCreator = sqlQueryCreatorService.getSqlQueryCreator(queryParams); - Optional result = Optional.empty(); - long rowsCount = 0; + Optional result = + withExceptionHandling(() -> queryExecutor.find(queryCreator.createForSelect())); - try { - result = Optional.of(queryExecutor.find(queryCreator.createForSelect())); + long rowsCount = 0; - AnalyticsPagingParams pagingParams = queryParams.getCommonParams().getPagingParams(); + AnalyticsPagingParams pagingParams = queryParams.getCommonParams().getPagingParams(); - if (pagingParams.showTotalPages()) { - rowsCount = queryExecutor.count(queryCreator.createForCount()); - } - } catch (BadSqlGrammarException ex) { - log.info(ERR_MSG_TABLE_NOT_EXISTING, ex); - throw ex; - } catch (DataAccessResourceFailureException ex) { - log.warn(E7131.getMessage(), ex); - throw new QueryRuntimeException(E7131); + if (pagingParams.showTotalPages()) { + rowsCount = + withExceptionHandling(() -> queryExecutor.count(queryCreator.createForCount())) + .orElse(0l); } List fields = queryCreator.getRenderableSqlQuery().getSelectFields(); @@ -129,28 +120,24 @@ public Grid getGridExplain(@Nonnull TeiQueryParams queryParams) { Grid grid = new ListGrid(); - try { - String explainId = randomUUID().toString(); - - SqlQueryCreator sqlQueryCreator = sqlQueryCreatorService.getSqlQueryCreator(queryParams); + String explainId = randomUUID().toString(); - executionPlanStore.addExecutionPlan(explainId, sqlQueryCreator.createForSelect()); + SqlQueryCreator sqlQueryCreator = sqlQueryCreatorService.getSqlQueryCreator(queryParams); - AnalyticsPagingParams pagingParams = queryParams.getCommonParams().getPagingParams(); + withExceptionHandling( + () -> executionPlanStore.addExecutionPlan(explainId, sqlQueryCreator.createForSelect())); - if (pagingParams.showTotalPages()) { - executionPlanStore.addExecutionPlan(explainId, sqlQueryCreator.createForCount()); - } + AnalyticsPagingParams pagingParams = queryParams.getCommonParams().getPagingParams(); - grid.addPerformanceMetrics(executionPlanStore.getExecutionPlans(explainId)); - } catch (BadSqlGrammarException ex) { - log.info(ERR_MSG_TABLE_NOT_EXISTING, ex); - throw ex; - } catch (DataAccessResourceFailureException ex) { - log.warn(E7131.getMessage(), ex); - throw new QueryRuntimeException(E7131); + if (pagingParams.showTotalPages()) { + withExceptionHandling( + () -> executionPlanStore.addExecutionPlan(explainId, sqlQueryCreator.createForCount())); } + List executionPlans = executionPlanStore.getExecutionPlans(explainId); + + grid.addPerformanceMetrics(executionPlans); + return grid; } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsOrganisationUnitUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsOrganisationUnitUtils.java new file mode 100644 index 000000000000..a85492c0c23f --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsOrganisationUnitUtils.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.util; + +import static org.hisp.dhis.analytics.AnalyticsMetaDataKey.ORG_UNITS; +import static org.hisp.dhis.analytics.AnalyticsMetaDataKey.USER_ORGUNIT; +import static org.hisp.dhis.analytics.AnalyticsMetaDataKey.USER_ORGUNIT_CHILDREN; +import static org.hisp.dhis.analytics.AnalyticsMetaDataKey.USER_ORGUNIT_GRANDCHILDREN; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.hisp.dhis.analytics.AnalyticsMetaDataKey; +import org.hisp.dhis.common.BaseIdentifiableObject; +import org.hisp.dhis.user.User; + +/** Utilities for organisation unit criteria of outcoming analytics response. */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class AnalyticsOrganisationUnitUtils { + /** + * Retrieve collection of all uids of organisation units belongs to current user and present in + * response grid. + * + * @param currentUser the {@link org.hisp.dhis.user.CurrentUser}. + * @return intersection of requested user organisation units and all units in response grid. + */ + public static Collection> getUserOrganisationUnitItems( + User currentUser, List userOrganisationUnitsCriteria) { + List> userOrganisations = new ArrayList<>(); + + if (userOrganisationUnitsCriteria == null || userOrganisationUnitsCriteria.isEmpty()) { + return userOrganisations; + } + + if (userOrganisationUnitsCriteria.contains(USER_ORGUNIT)) { + Map userOrganisationUnits = + AnalyticsOrganisationUnitUtils.getUserOrganisationUnitUidList(USER_ORGUNIT, currentUser); + userOrganisations.add(userOrganisationUnits); + } + + if (userOrganisationUnitsCriteria.contains(USER_ORGUNIT_CHILDREN)) { + Map userChildrenOrganisationUnits = + AnalyticsOrganisationUnitUtils.getUserOrganisationUnitUidList( + USER_ORGUNIT_CHILDREN, currentUser); + userOrganisations.add(userChildrenOrganisationUnits); + } + + if (userOrganisationUnitsCriteria.contains(USER_ORGUNIT_GRANDCHILDREN)) { + Map userGrandChildrenOrganisationUnits = + AnalyticsOrganisationUnitUtils.getUserOrganisationUnitUidList( + USER_ORGUNIT_GRANDCHILDREN, currentUser); + userOrganisations.add(userGrandChildrenOrganisationUnits); + } + + return userOrganisations; + } + + private static Map getUserOrganisationUnitUidList( + AnalyticsMetaDataKey analyticsMetaDataKey, User currentUser) { + + List userOrgUnitList; + + switch (analyticsMetaDataKey) { + case USER_ORGUNIT -> userOrgUnitList = + currentUser.getOrganisationUnits().stream().map(BaseIdentifiableObject::getUid).toList(); + case USER_ORGUNIT_CHILDREN -> userOrgUnitList = + currentUser.getOrganisationUnits().stream() + .flatMap(ou -> ou.getChildren().stream()) + .map(BaseIdentifiableObject::getUid) + .toList(); + case USER_ORGUNIT_GRANDCHILDREN -> userOrgUnitList = + currentUser.getOrganisationUnits().stream() + .flatMap(ou -> ou.getGrandChildren().stream()) + .map(BaseIdentifiableObject::getUid) + .toList(); + default -> userOrgUnitList = List.of(); + } + + return Map.of(analyticsMetaDataKey.getKey(), Map.of(ORG_UNITS.getKey(), userOrgUnitList)); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java index f86c0536fd70..b32a35e43505 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java @@ -35,7 +35,10 @@ import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; import static org.hisp.dhis.common.DimensionalObject.PERIOD_DIM_ID; import static org.hisp.dhis.common.DimensionalObject.QUERY_MODS_ID_SEPARATOR; +import static org.hisp.dhis.dxf2.webmessage.WebMessageUtils.relationDoesNotExist; import static org.hisp.dhis.expression.ExpressionService.SYMBOL_WILDCARD; +import static org.hisp.dhis.feedback.ErrorCode.E7131; +import static org.hisp.dhis.feedback.ErrorCode.E7132; import static org.hisp.dhis.system.util.MathUtils.getRounded; import static org.hisp.dhis.util.DateUtils.getMediumDateString; import static org.springframework.util.Assert.isTrue; @@ -50,9 +53,12 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; import java.util.regex.Pattern; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.math3.util.Precision; @@ -77,6 +83,7 @@ import org.hisp.dhis.common.MetadataItem; import org.hisp.dhis.common.NameableObjectUtils; import org.hisp.dhis.common.QueryItem; +import org.hisp.dhis.common.QueryRuntimeException; import org.hisp.dhis.common.RegexUtils; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.commons.util.TextUtils; @@ -100,11 +107,15 @@ import org.hisp.dhis.system.grid.ListGrid; import org.hisp.dhis.util.DateUtils; import org.joda.time.DateTime; +import org.springframework.dao.DataAccessResourceFailureException; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.util.Assert; /** * @author Lars Helge Overland */ +@Slf4j public class AnalyticsUtils { private static final int DECIMALS_NO_ROUNDING = 10; @@ -114,7 +125,13 @@ public class AnalyticsUtils { Pattern.compile(DataQueryParams.PREFIX_ORG_UNIT_LEVEL + "(\\d+)"); public static final String ERR_MSG_TABLE_NOT_EXISTING = - "Query failed, likely because the requested analytics table does not exist"; + "Query failed, likely because the requested analytics table does not exist: "; + + public static final String ERR_MSG_SQL_SYNTAX_ERROR = + "An error occurred during the execution of an analytics query: "; + + public static final String ERR_MSG_SILENT_FALLBACK = + "An exception occurred - silently fallback since it's multiple analytics query: "; /** * Returns an SQL statement for retrieving raw data values for an aggregate query. @@ -1041,4 +1058,82 @@ public static boolean hasPeriod(List row, int periodIndex) { && row.get(periodIndex) instanceof String && PeriodType.getPeriodFromIsoString((String) row.get(periodIndex)) != null; } + + /** + * Wraps the provided interface around a common exception handling strategy. + * + * @param runnable the {@link Runnable} containing the code block to execute and wrap around the + * exception handling. + */ + public static void withExceptionHandling(Runnable runnable) { + withExceptionHandling( + () -> { + runnable.run(); + return null; + }, + false); + } + + /** + * Wraps the provided interface around a common exception handling strategy. + * + * @param runnable the {@link Runnable} containing the code block to execute and wrap around the + * exception handling. + * @param isMultipleQueries special treatment for multiple queries should be applied (or not). + */ + public static void withExceptionHandling(Runnable runnable, boolean isMultipleQueries) { + withExceptionHandling( + () -> { + runnable.run(); + return null; + }, + isMultipleQueries); + } + + /** + * Wraps the provided interface around a common exception handling strategy. + * + * @param supplier the {@link Supplier} containing the code block to execute and wrap around the + * exception handling. + * @return the {@link Optional} wrapping th result of the supplier execution. + */ + public static Optional withExceptionHandling(Supplier supplier) { + return withExceptionHandling(supplier::get, false); + } + + /** + * Wraps the provided interface around a common exception handling strategy. + * + * @param supplier the {@link Supplier} containing the code block to execute and wrap around the + * exception handling. + * @param isMultipleQueries special treatment for multiple queries should be applied (or not). + * @return the {@link Optional} wrapping th result of the supplier execution. + */ + public static Optional withExceptionHandling( + Supplier supplier, boolean isMultipleQueries) { + try { + return Optional.ofNullable(supplier.get()); + } catch (BadSqlGrammarException ex) { + if (relationDoesNotExist(ex.getSQLException())) { + log.info(ERR_MSG_TABLE_NOT_EXISTING, ex); + throw ex; + } + if (!isMultipleQueries) { + log.error(ERR_MSG_SQL_SYNTAX_ERROR, ex); + throw ex; + } + log.info(ERR_MSG_SILENT_FALLBACK, ex); + } catch (QueryRuntimeException ex) { + log.error("Internal runtime exception", ex); + throw ex; + } catch (DataIntegrityViolationException ex) { + log.error(E7132.getMessage(), ex); + throw new QueryRuntimeException(E7132); + } catch (DataAccessResourceFailureException ex) { + log.error(E7131.getMessage(), ex); + throw new QueryRuntimeException(E7131); + } + + return Optional.empty(); + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceBaseTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceBaseTest.java index 8bbc1bd0ee3e..a2309e3bd1a3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceBaseTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceBaseTest.java @@ -53,6 +53,7 @@ import org.hisp.dhis.external.conf.DhisConfigurationProvider; import org.hisp.dhis.organisationunit.OrganisationUnitService; import org.hisp.dhis.setting.SystemSettingManager; +import org.hisp.dhis.user.CurrentUserService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -99,12 +100,15 @@ abstract class AnalyticsServiceBaseTest { @Mock private ExecutionPlanStore executionPlanStore; + @Mock private CurrentUserService currentUserService; + DataAggregator target; @BeforeEach public void baseSetUp() { HeaderHandler headerHandler = new HeaderHandler(); - MetadataHandler metadataHandler = new MetadataHandler(dataQueryService, schemeIdResponseMapper); + MetadataHandler metadataHandler = + new MetadataHandler(dataQueryService, schemeIdResponseMapper, currentUserService); DataHandler dataHandler = new DataHandler( eventAnalyticsService, diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/JdbcSubexpressionQueryGeneratorTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/JdbcSubexpressionQueryGeneratorTest.java index 24ad6e0ea286..ce2f509cd862 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/JdbcSubexpressionQueryGeneratorTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/JdbcSubexpressionQueryGeneratorTest.java @@ -121,7 +121,7 @@ void testGetSql() { String deoACol = getItemColumnName(deB.getUid(), cocA.getUid(), null, null); String deoBCol = getItemColumnName(deC.getUid(), cocB.getUid(), cocC.getUid(), null); String deoCCol = getItemColumnName(deD.getUid(), null, cocD.getUid(), null); - String deECol = getItemColumnName(deD.getUid(), null, null, queryModsMin); + String deECol = getItemColumnName(deE.getUid(), null, null, queryModsMin); String subexSql = deACol + "*(" + deoACol + "+" + deoBCol + ")+" + deoCCol + "-" + deECol; @@ -139,6 +139,7 @@ void testGetSql() { ORGUNIT_DIM_ID, DimensionType.ORGANISATION_UNIT, getList(ouA))) .addDimension( new BaseDimensionalObject(PERIOD_DIM_ID, DimensionType.PERIOD, getList(peA))) + .withPeriodType("monthly") .build(); JdbcSubexpressionQueryGenerator target = @@ -146,19 +147,19 @@ ORGUNIT_DIM_ID, DimensionType.ORGANISATION_UNIT, getList(ouA))) String expected = "select ax.\"pe\",'subexprxUID' as \"dx\"," - + "sum(\"deabcdefghA\"*(\"deabcdefghB_cuabcdefghA\"+\"deabcdefghC_cuabcdefghB_cuabcdefghC\")+\"deabcdefghD__cuabcdefghD\"-\"deabcdefghDMIN\") as \"value\" " + + "sum(\"deabcdefghA\"*(\"deabcdefghB_cuabcdefghA\"+\"deabcdefghC_cuabcdefghB_cuabcdefghC\")+\"deabcdefghD__cuabcdefghD\"-\"deabcdefghE_agg_MIN\") as \"value\" " + "from (select ax.\"pe\", " - + "sum(case when ax.\"dx\"='deabcdefghA' then \"value\" else null end) as \"deabcdefghA\"," - + "sum(case when ax.\"dx\"='deabcdefghB' and ax.\"co\"='cuabcdefghA' then \"value\" else null end) as \"deabcdefghB_cuabcdefghA\"," - + "sum(case when ax.\"dx\"='deabcdefghC' and ax.\"co\"='cuabcdefghB' and ax.\"ao\"='cuabcdefghC' then \"value\" else null end) as \"deabcdefghC_cuabcdefghB_cuabcdefghC\"," - + "sum(case when ax.\"dx\"='deabcdefghD' and ax.\"ao\"='cuabcdefghD' then \"value\" else null end) as \"deabcdefghD__cuabcdefghD\"," - + "min(case when ax.\"dx\"='deabcdefghE' then \"value\" else null end) as \"deabcdefghEMIN\" " + + "sum(case when ax.\"dx\"='deabcdefghA' then \"value\"::numeric else null end) as \"deabcdefghA\"," + + "sum(case when ax.\"dx\"='deabcdefghB' and ax.\"co\"='cuabcdefghA' then \"value\"::numeric else null end) as \"deabcdefghB_cuabcdefghA\"," + + "sum(case when ax.\"dx\"='deabcdefghC' and ax.\"co\"='cuabcdefghB' and ax.\"ao\"='cuabcdefghC' then \"value\"::numeric else null end) as \"deabcdefghC_cuabcdefghB_cuabcdefghC\"," + + "sum(case when ax.\"dx\"='deabcdefghD' and ax.\"ao\"='cuabcdefghD' then \"value\"::numeric else null end) as \"deabcdefghD__cuabcdefghD\"," + + "min(case when ax.\"dx\"='deabcdefghE' then \"value\"::numeric else null end) as \"deabcdefghE_agg_MIN\" " + "from analytics as ax " - + "where ax.\"pe\" in ('202305') " + + "where ax.\"monthly\" in ('202305') " + "and ( ax.\"ou\" in ('ouabcdefghA') ) " + "and ax.\"dx\" in ('deabcdefghA','deabcdefghB','deabcdefghC','deabcdefghD','deabcdefghE') " - + "group by ax.\"pe\",ax.\"ou\") as ax " - + "where \"deabcdefghA\"*(\"deabcdefghB_cuabcdefghA\"+\"deabcdefghC_cuabcdefghB_cuabcdefghC\")+\"deabcdefghD__cuabcdefghD\"-\"deabcdefghDMIN\" is not null " + + "group by ax.\"monthly\",ax.\"ou\") as ax " + + "where \"deabcdefghA\"*(\"deabcdefghB_cuabcdefghA\"+\"deabcdefghC_cuabcdefghB_cuabcdefghC\")+\"deabcdefghD__cuabcdefghD\"-\"deabcdefghE_agg_MIN\" is not null " + "group by ax.\"pe\" "; String actual = anonymize(target.getSql()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/SubexpressionPeriodOffsetUtilsTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/SubexpressionPeriodOffsetUtilsTest.java new file mode 100644 index 000000000000..4c4fe9184941 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/SubexpressionPeriodOffsetUtilsTest.java @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.data; + +import static java.lang.String.format; +import static org.hisp.dhis.DhisConvenienceTest.createDataElement; +import static org.hisp.dhis.DhisConvenienceTest.createPeriod; +import static org.hisp.dhis.common.DimensionalObject.DATA_X_DIM_ID; +import static org.hisp.dhis.common.DimensionalObjectUtils.getList; +import static org.hisp.dhis.utils.Assertions.assertContainsOnly; +import static org.hisp.dhis.utils.Assertions.assertIsEmpty; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Arrays; +import java.util.List; +import org.hisp.dhis.analytics.DataQueryParams; +import org.hisp.dhis.common.BaseDimensionalObject; +import org.hisp.dhis.common.DimensionType; +import org.hisp.dhis.common.DimensionalItemObject; +import org.hisp.dhis.common.QueryModifiers; +import org.hisp.dhis.dataelement.DataElement; +import org.hisp.dhis.period.Period; +import org.hisp.dhis.period.PeriodType; +import org.hisp.dhis.subexpression.SubexpressionDimensionItem; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +/** + * @author Jim Grace + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class SubexpressionPeriodOffsetUtilsTest { + private DataElement de0 = createDataElement('A'); + + private DataElement dem1 = createDataElementAWithPeriodOffset(-1); + + private DataElement dem2 = createDataElementAWithPeriodOffset(-2); + + private DataElement dep1 = createDataElementAWithPeriodOffset(1); + + private String deAUid = de0.getUid(); + + private String expression = + format( + "subExpression( #{%s} + #{%s}.periodOffset(-1) + #{%s}.periodOffset(-2) + #{%s}.periodOffset(1) )", + deAUid, deAUid, deAUid, deAUid); + + private List items = List.of(de0, dem1, dem2, dep1); + + private SubexpressionDimensionItem subex = + new SubexpressionDimensionItem(expression, items, null); + + private Period periodA = createPeriod("202309"); + + private Period periodB = createPeriod("202310"); + + private DataQueryParams params = + DataQueryParams.newBuilder() + .withPeriodType("monthly") + .withPeriods(List.of(periodA, periodB)) + .addDimension( + new BaseDimensionalObject(DATA_X_DIM_ID, DimensionType.DATA_X, getList(subex))) + .build(); + + @Test + void testJoinPeriodOffsetValues() { + String result = SubexpressionPeriodOffsetUtils.joinPeriodOffsetValues(params); + String expected = + " join (values" + + "(-2,'202309','202307'),(-2,'202310','202308')," + + "(-1,'202309','202308'),(-1,'202310','202309')," + + "(0,'202309','202309'),(0,'202310','202310')," + + "(1,'202309','202310'),(1,'202310','202311'))" + + " as shift (\"delta\", \"reportperiod\", \"dataperiod\") on \"dataperiod\" = \"monthly\""; + assertEquals(expected, result); + } + + @Test + void testGetParamsWithOffsetPeriodsWithoutData() { + DataQueryParams result = + SubexpressionPeriodOffsetUtils.getParamsWithOffsetPeriodsWithoutData(params); + + List expectedPeriods = + getPeriodList("202307", "202308", "202309", "202310", "202311"); + assertContainsOnly(expectedPeriods, result.getPeriods()); + + assertIsEmpty(result.getAllDataDimensionItems()); + } + + @Test + void testGetParamsWithDataPeriods() { + DataQueryParams result = SubexpressionPeriodOffsetUtils.getParamsWithOffsetPeriods(params); + + List expectedPeriods = + getPeriodList("202307", "202308", "202309", "202310", "202311"); + assertContainsOnly(expectedPeriods, result.getPeriods()); + + List expectedData = List.of(subex); + assertContainsOnly(expectedData, result.getAllDataDimensionItems()); + } + + // ------------------------------------------------------------------------- + // Supportive methods + // ------------------------------------------------------------------------- + + private DataElement createDataElementAWithPeriodOffset(int periodOffset) { + QueryModifiers mods = QueryModifiers.builder().periodOffset(periodOffset).build(); + DataElement de = createDataElement('A'); + de.setQueryMods(mods); + return de; + } + + private List getPeriodList(String... isoPeriods) { + return Arrays.stream(isoPeriods) + .map(PeriodType::getPeriodFromIsoString) + .map(DimensionalItemObject.class::cast) + .toList(); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractAnalyticsServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractAnalyticsServiceTest.java index 440500632d2f..a304f2a1b298 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractAnalyticsServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractAnalyticsServiceTest.java @@ -57,6 +57,7 @@ import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramStage; import org.hisp.dhis.system.grid.ListGrid; +import org.hisp.dhis.user.CurrentUserService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -90,10 +91,13 @@ class AbstractAnalyticsServiceTest { @Mock private SchemeIdResponseMapper schemeIdResponseMapper; + @Mock private CurrentUserService currentUserService; + @BeforeEach public void setUp() { dummyAnalyticsService = - new DummyAnalyticsService(securityManager, eventQueryValidator, schemeIdResponseMapper); + new DummyAnalyticsService( + securityManager, eventQueryValidator, schemeIdResponseMapper, currentUserService); peA = MonthlyPeriodType.getPeriodFromIsoString("201701"); ouA = createOrganisationUnit('A'); @@ -249,8 +253,9 @@ class DummyAnalyticsService extends AbstractAnalyticsService { public DummyAnalyticsService( AnalyticsSecurityManager securityManager, EventQueryValidator queryValidator, - SchemeIdResponseMapper schemeIdResponseMapper) { - super(securityManager, queryValidator, schemeIdResponseMapper); + SchemeIdResponseMapper schemeIdResponseMapper, + CurrentUserService currentUserService) { + super(securityManager, queryValidator, schemeIdResponseMapper, currentUserService); } @Override diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsServiceTest.java index 49dbdfdfbfce..520e4a0acf1e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsServiceTest.java @@ -55,6 +55,7 @@ import org.hisp.dhis.program.Program; import org.hisp.dhis.system.database.DatabaseInfo; import org.hisp.dhis.trackedentity.TrackedEntityAttributeService; +import org.hisp.dhis.user.CurrentUserService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -92,6 +93,8 @@ class DefaultEventAnalyticsServiceTest { @Mock private SchemeIdResponseMapper schemeIdResponseMapper; + @Mock private CurrentUserService currentUserService; + @BeforeEach public void setUp() { defaultEventAnalyticsService = @@ -106,7 +109,8 @@ public void setUp() { databaseInfo, analyticsCache, enrollmentAnalyticsManager, - schemeIdResponseMapper); + schemeIdResponseMapper, + currentUserService); } @Test diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsManagerTest.java index a2a9b64d2bca..7104b15d3f62 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsManagerTest.java @@ -43,7 +43,9 @@ import static org.hisp.dhis.common.QueryOperator.EQ; import static org.hisp.dhis.common.QueryOperator.IN; import static org.hisp.dhis.common.QueryOperator.NEQ; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.verify; @@ -85,6 +87,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; +import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.support.rowset.SqlRowSet; @@ -336,6 +339,29 @@ void verifyGetEventsWithProgramStatusParam() { assertSql(sql.getValue(), expected); } + @Test + void testBadGrammarExceptionNonMultipleQueries() { + // Given + mockEmptyRowSet(); + EventQueryParams params = createRequestParamsWithStatuses(); + when(jdbcTemplate.queryForRowSet(anyString())).thenThrow(BadSqlGrammarException.class); + + // Then + assertThrows( + BadSqlGrammarException.class, () -> subject.getEnrollments(params, new ListGrid(), 10000)); + } + + @Test + void testBadGrammarExceptionWithMultipleQueries() { + // Given + mockEmptyRowSet(); + EventQueryParams params = createRequestParamsWithMultipleQueries(); + when(jdbcTemplate.queryForRowSet(anyString())).thenThrow(BadSqlGrammarException.class); + + // Then + assertDoesNotThrow(() -> subject.getEnrollments(params, new ListGrid(), 10000)); + } + @Test void verifyWithProgramStageAndNumericDataElementAndFilter2() { EventQueryParams params = createRequestParamsWithFilter(programStage, ValueType.NUMBER); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventAnalyticsTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventAnalyticsTest.java index 911f21e6e344..3243d606b99a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventAnalyticsTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventAnalyticsTest.java @@ -183,6 +183,18 @@ protected EventQueryParams createRequestParamsWithStatuses() { return params.build(); } + protected EventQueryParams createRequestParamsWithMultipleQueries() { + OrganisationUnit ouA = createOrganisationUnit('A'); + ouA.setPath("/" + ouA.getUid()); + EventQueryParams.Builder params = new EventQueryParams.Builder(); + params.withPeriods(getList(createPeriod("2000Q1")), "quarterly"); + params.withOrganisationUnits(getList(ouA)); + params.withTableName(getTableName() + "_" + programA.getUid()); + params.withProgram(programA); + params.withMultipleQueries(true); + return params.build(); + } + protected EventQueryParams createRequestParamsWithTimeField(String timeField) { OrganisationUnit ouA = createOrganisationUnit('A'); ouA.setPath("/" + ouA.getUid()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java index 863f22bb2ad4..078af8dd771e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java @@ -50,6 +50,7 @@ import static org.hisp.dhis.analytics.ColumnDataType.INTEGER; import static org.hisp.dhis.analytics.ColumnDataType.TEXT; import static org.hisp.dhis.analytics.ColumnDataType.TIMESTAMP; +import static org.hisp.dhis.period.PeriodDataProvider.DataSource.DATABASE; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -136,7 +137,7 @@ class JdbcEventAnalyticsTableManagerTest { @Mock private PeriodDataProvider periodDataProvider; - private AnalyticsExportSettings analyticsExportSettings; + @Mock private AnalyticsExportSettings analyticsExportSettings; private JdbcEventAnalyticsTableManager subject; @@ -259,9 +260,10 @@ void verifyGetTableWithCategoryCombo() { addCategoryCombo(program, categoryCombo); when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(program)); - when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear())); + when(periodDataProvider.getAvailableYears(DATABASE)) + .thenReturn(List.of(2018, 2019, now().getYear())); - List availableDataYears = periodDataProvider.getAvailableYears(); + List availableDataYears = periodDataProvider.getAvailableYears(DATABASE); when(jdbcTemplate.queryForList( getYearQueryForCurrentYear(program, true, availableDataYears), Integer.class)) @@ -294,9 +296,10 @@ void verifyGetTableWithCategoryCombo() { void verifyClientSideTimestampsColumns() { Program program = createProgram('A'); when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(program)); - when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear())); + when(periodDataProvider.getAvailableYears(DATABASE)) + .thenReturn(List.of(2018, 2019, now().getYear())); - List availableDataYears = periodDataProvider.getAvailableYears(); + List availableDataYears = periodDataProvider.getAvailableYears(DATABASE); when(jdbcTemplate.queryForList( getYearQueryForCurrentYear(program, true, availableDataYears), Integer.class)) @@ -330,9 +333,10 @@ void verifyClientSideTimestampsColumns() { void verifyAnalyticsEventTableHasDefaultPartition() { Program program = createProgram('A'); when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(program)); - when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2021, 2022, 2023, 2024, 2025)); + when(periodDataProvider.getAvailableYears(DATABASE)) + .thenReturn(List.of(2021, 2022, 2023, 2024, 2025)); - List availableDataYears = periodDataProvider.getAvailableYears(); + List availableDataYears = periodDataProvider.getAvailableYears(DATABASE); when(jdbcTemplate.queryForList( getYearQueryForCurrentYear(program, true, availableDataYears), Integer.class)) @@ -438,9 +442,10 @@ void verifyGetTableWithDataElements() { .withToday(today) .build(); - when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear())); + when(periodDataProvider.getAvailableYears(DATABASE)) + .thenReturn(List.of(2018, 2019, now().getYear())); - List availableDataYears = periodDataProvider.getAvailableYears(); + List availableDataYears = periodDataProvider.getAvailableYears(DATABASE); when(jdbcTemplate.queryForList( getYearQueryForCurrentYear(program, true, availableDataYears), Integer.class)) @@ -506,9 +511,10 @@ void verifyGetTableWithTrackedEntityAttribute() { .withToday(today) .build(); - when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear())); + when(periodDataProvider.getAvailableYears(DATABASE)) + .thenReturn(List.of(2018, 2019, now().getYear())); - List availableDataYears = periodDataProvider.getAvailableYears(); + List availableDataYears = periodDataProvider.getAvailableYears(DATABASE); when(jdbcTemplate.queryForList( getYearQueryForCurrentYear(program, true, availableDataYears), Integer.class)) @@ -562,9 +568,10 @@ void verifyDataElementTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable( .withToday(today) .build(); - when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear())); + when(periodDataProvider.getAvailableYears(DATABASE)) + .thenReturn(List.of(2018, 2019, now().getYear())); - List availableDataYears = periodDataProvider.getAvailableYears(); + List availableDataYears = periodDataProvider.getAvailableYears(DATABASE); when(jdbcTemplate.queryForList( getYearQueryForCurrentYear(programA, true, availableDataYears), Integer.class)) @@ -603,9 +610,10 @@ void verifyTeiTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() { programA.setProgramAttributes(List.of(programTrackedEntityAttribute)); when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(programA)); - when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear())); + when(periodDataProvider.getAvailableYears(DATABASE)) + .thenReturn(List.of(2018, 2019, now().getYear())); - List availableDataYears = periodDataProvider.getAvailableYears(); + List availableDataYears = periodDataProvider.getAvailableYears(DATABASE); AnalyticsTableUpdateParams params = AnalyticsTableUpdateParams.newBuilder() @@ -650,9 +658,10 @@ void verifyOrgUnitOwnershipJoinsWhenPopulatingEventAnalyticsTable() { programA.setProgramAttributes(List.of(programTrackedEntityAttribute)); when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(programA)); - when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear())); + when(periodDataProvider.getAvailableYears(DATABASE)) + .thenReturn(List.of(2018, 2019, now().getYear())); - List availableDataYears = periodDataProvider.getAvailableYears(); + List availableDataYears = periodDataProvider.getAvailableYears(DATABASE); AnalyticsTableUpdateParams params = AnalyticsTableUpdateParams.newBuilder() @@ -688,9 +697,10 @@ void verifyGetAnalyticsTableWithOuLevels() { Program programA = rnd.nextObject(Program.class); programA.setId(0); - when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear())); + when(periodDataProvider.getAvailableYears(DATABASE)) + .thenReturn(List.of(2018, 2019, now().getYear())); - List availableDataYears = periodDataProvider.getAvailableYears(); + List availableDataYears = periodDataProvider.getAvailableYears(DATABASE); int startYear = availableDataYears.get(0); int latestYear = availableDataYears.get(availableDataYears.size() - 1); @@ -751,9 +761,10 @@ void verifyGetAnalyticsTableWithOuGroupSet() { when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(programA)); when(idObjectManager.getDataDimensionsNoAcl(OrganisationUnitGroupSet.class)) .thenReturn(ouGroupSet); - when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear())); + when(periodDataProvider.getAvailableYears(DATABASE)) + .thenReturn(List.of(2018, 2019, now().getYear())); - List availableDataYears = periodDataProvider.getAvailableYears(); + List availableDataYears = periodDataProvider.getAvailableYears(DATABASE); AnalyticsTableUpdateParams params = AnalyticsTableUpdateParams.newBuilder().withStartTime(START_TIME).build(); @@ -791,9 +802,10 @@ void verifyGetAnalyticsTableWithOptionGroupSets() { when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(programA)); when(categoryService.getAttributeCategoryOptionGroupSetsNoAcl()).thenReturn(cogs); - when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear())); + when(periodDataProvider.getAvailableYears(DATABASE)) + .thenReturn(List.of(2018, 2019, now().getYear())); - List availableDataYears = periodDataProvider.getAvailableYears(); + List availableDataYears = periodDataProvider.getAvailableYears(DATABASE); when(jdbcTemplate.queryForList( getYearQueryForCurrentYear(programA, false, availableDataYears), Integer.class)) @@ -870,9 +882,10 @@ void verifyTeaTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() { programA.setProgramAttributes(List.of(programTrackedEntityAttribute)); - when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear())); + when(periodDataProvider.getAvailableYears(DATABASE)) + .thenReturn(List.of(2018, 2019, now().getYear())); - List availableDataYears = periodDataProvider.getAvailableYears(); + List availableDataYears = periodDataProvider.getAvailableYears(DATABASE); int startYear = availableDataYears.get(0); int latestYear = availableDataYears.get(availableDataYears.size() - 1); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/util/AnalyticsOrganisationUnitUtilsTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/util/AnalyticsOrganisationUnitUtilsTest.java new file mode 100644 index 000000000000..820f2a9edb6d --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/util/AnalyticsOrganisationUnitUtilsTest.java @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.util; + +import static org.hisp.dhis.analytics.AnalyticsMetaDataKey.ORG_UNITS; +import static org.hisp.dhis.analytics.AnalyticsMetaDataKey.USER_ORGUNIT; +import static org.hisp.dhis.analytics.AnalyticsMetaDataKey.USER_ORGUNIT_CHILDREN; +import static org.hisp.dhis.analytics.AnalyticsMetaDataKey.USER_ORGUNIT_GRANDCHILDREN; +import static org.hisp.dhis.analytics.util.AnalyticsOrganisationUnitUtils.getUserOrganisationUnitItems; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.hisp.dhis.analytics.AnalyticsMetaDataKey; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.user.User; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; + +@MockitoSettings(strictness = Strictness.LENIENT) +class AnalyticsOrganisationUnitUtilsTest { + + @Mock private User user; + private final OrganisationUnit orgA = new OrganisationUnit("orgA"); + private final OrganisationUnit orgB = new OrganisationUnit("orgB"); + private final OrganisationUnit orgC = new OrganisationUnit("orgC"); + + private final String uidA = "abcdefA"; + private final String uidB = "abcdefB"; + private final String uidC = "abcdefC"; + + @BeforeEach + public void setUp() { + orgA.setUid(uidA); + orgB.setUid(uidB); + orgC.setUid(uidC); + orgA.setChildren(Set.of(orgB)); + orgB.setChildren(Set.of(orgC)); + } + + @Test + void testAnalyticsOrganisationUnitUtils_All() { + // given + List userOrganisationUnitsCriteria = + List.of(USER_ORGUNIT, USER_ORGUNIT_CHILDREN, USER_ORGUNIT_GRANDCHILDREN); + + // when + when(user.getOrganisationUnits()).thenReturn(Set.of(orgA)); + List> uidMapList = + getUserOrganisationUnitItems(user, userOrganisationUnitsCriteria).stream().toList(); + + // then + assertEquals(3, uidMapList.size()); + assertEquals( + "{" + ORG_UNITS.getKey() + "=[" + uidA + "]}", + uidMapList.get(0).get(USER_ORGUNIT.getKey()).toString()); + assertEquals( + "{" + ORG_UNITS.getKey() + "=[" + uidB + "]}", + uidMapList.get(1).get(USER_ORGUNIT_CHILDREN.getKey()).toString()); + assertEquals( + "{" + ORG_UNITS.getKey() + "=[" + uidC + "]}", + uidMapList.get(2).get(USER_ORGUNIT_GRANDCHILDREN.getKey()).toString()); + } + + @Test + void testAnalyticsOrganisationUnitUtils_Parent() { + // given + List userOrganisationUnitsCriteria = List.of(USER_ORGUNIT); + + // when + when(user.getOrganisationUnits()).thenReturn(Set.of(orgA)); + List> uidMapList = + getUserOrganisationUnitItems(user, userOrganisationUnitsCriteria).stream().toList(); + + // then + assertEquals(1, uidMapList.size()); + assertEquals( + "{" + ORG_UNITS.getKey() + "=[" + uidA + "]}", + uidMapList.get(0).get(USER_ORGUNIT.getKey()).toString()); + } + + @Test + void testAnalyticsOrganisationUnitUtils_Children() { + // given + List userOrganisationUnitsCriteria = List.of(USER_ORGUNIT_CHILDREN); + + // when + when(user.getOrganisationUnits()).thenReturn(Set.of(orgA)); + List> uidMapList = + getUserOrganisationUnitItems(user, userOrganisationUnitsCriteria).stream().toList(); + + // then + assertEquals(1, uidMapList.size()); + assertEquals( + "{" + ORG_UNITS.getKey() + "=[" + uidB + "]}", + uidMapList.get(0).get(USER_ORGUNIT_CHILDREN.getKey()).toString()); + } + + @Test + void testAnalyticsOrganisationUnitUtils_Grandchildren() { + // given + List userOrganisationUnitsCriteria = List.of(USER_ORGUNIT_GRANDCHILDREN); + + // when + when(user.getOrganisationUnits()).thenReturn(Set.of(orgA)); + List> uidMapList = + getUserOrganisationUnitItems(user, userOrganisationUnitsCriteria).stream().toList(); + + // then + assertEquals(1, uidMapList.size()); + assertEquals( + "{" + ORG_UNITS.getKey() + "=[" + uidC + "]}", + uidMapList.get(0).get(USER_ORGUNIT_GRANDCHILDREN.getKey()).toString()); + } + + @Test + void testAnalyticsOrganisationUnitUtils_All_Empty() { + // given + List userOrganisationUnitsCriteria = + List.of(USER_ORGUNIT, USER_ORGUNIT_CHILDREN, USER_ORGUNIT_GRANDCHILDREN); + + // when + when(user.getOrganisationUnits()).thenReturn(Set.of()); + List> uidMapList = + getUserOrganisationUnitItems(user, userOrganisationUnitsCriteria).stream().toList(); + + // then + assertEquals(3, uidMapList.size()); + assertEquals( + "{" + ORG_UNITS.getKey() + "=[]}", uidMapList.get(0).get(USER_ORGUNIT.getKey()).toString()); + assertEquals( + "{" + ORG_UNITS.getKey() + "=[]}", + uidMapList.get(1).get(USER_ORGUNIT_CHILDREN.getKey()).toString()); + assertEquals( + "{" + ORG_UNITS.getKey() + "=[]}", + uidMapList.get(2).get(USER_ORGUNIT_GRANDCHILDREN.getKey()).toString()); + } + + @Test + void testAnalyticsOrganisationUnitUtils_Parent_Empty() { + // given + List userOrganisationUnitsCriteria = List.of(USER_ORGUNIT); + + // when + when(user.getOrganisationUnits()).thenReturn(Set.of()); + List> uidMapList = + getUserOrganisationUnitItems(user, userOrganisationUnitsCriteria).stream().toList(); + + // then + assertEquals(1, uidMapList.size()); + assertEquals( + "{" + ORG_UNITS.getKey() + "=[]}", uidMapList.get(0).get(USER_ORGUNIT.getKey()).toString()); + } + + @Test + void testAnalyticsOrganisationUnitUtils_Children_Empty() { + // given + List userOrganisationUnitsCriteria = List.of(USER_ORGUNIT_CHILDREN); + + // when + when(user.getOrganisationUnits()).thenReturn(Set.of()); + List> uidMapList = + getUserOrganisationUnitItems(user, userOrganisationUnitsCriteria).stream().toList(); + + // then + assertEquals(1, uidMapList.size()); + assertEquals( + "{" + ORG_UNITS.getKey() + "=[]}", + uidMapList.get(0).get(USER_ORGUNIT_CHILDREN.getKey()).toString()); + } + + @Test + void testAnalyticsOrganisationUnitUtils_Grandchildren_Empty() { + // given + List userOrganisationUnitsCriteria = List.of(USER_ORGUNIT_GRANDCHILDREN); + + // when + when(user.getOrganisationUnits()).thenReturn(Set.of()); + List> uidMapList = + getUserOrganisationUnitItems(user, userOrganisationUnitsCriteria).stream().toList(); + + // then + assertEquals(1, uidMapList.size()); + assertEquals( + "{" + ORG_UNITS.getKey() + "=[]}", + uidMapList.get(0).get(USER_ORGUNIT_GRANDCHILDREN.getKey()).toString()); + } +} diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/analytics/AnalyticsExportSettings.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/analytics/AnalyticsExportSettings.java index 5db6aba72751..5d013da915da 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/analytics/AnalyticsExportSettings.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/analytics/AnalyticsExportSettings.java @@ -69,9 +69,11 @@ public String getTableType() { * Returns the years' offset defined for the period generation. See {@link * ANALYTICS_MAX_PERIOD_YEARS_OFFSET}. * - * @return the offset defined in system settings. + * @return the offset defined in system settings, or null if nothing is set. */ - public int getMaxPeriodYearsOffset() { - return systemSettingManager.getIntSetting(ANALYTICS_MAX_PERIOD_YEARS_OFFSET); + public Integer getMaxPeriodYearsOffset() { + return systemSettingManager.getIntSetting(ANALYTICS_MAX_PERIOD_YEARS_OFFSET) < 0 + ? null + : systemSettingManager.getIntSetting(ANALYTICS_MAX_PERIOD_YEARS_OFFSET); } } diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/association/CategoryOptionOrganisationUnitAssociationsQueryBuilder.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/association/CategoryOptionOrganisationUnitAssociationsQueryBuilder.java index 2778b10a2241..cda7c993cc6f 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/association/CategoryOptionOrganisationUnitAssociationsQueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/association/CategoryOptionOrganisationUnitAssociationsQueryBuilder.java @@ -45,7 +45,7 @@ public class CategoryOptionOrganisationUnitAssociationsQueryBuilder private final String joinColumnName = "categoryoptionid"; @Getter(AccessLevel.PROTECTED) - private final String baseTableName = "dataelementcategoryoption"; + private final String baseTableName = "categoryoption"; public CategoryOptionOrganisationUnitAssociationsQueryBuilder( CurrentUserService currentUserService) { diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataapproval/hibernate/HibernateDataApprovalStore.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataapproval/hibernate/HibernateDataApprovalStore.java index c874cf42c926..ccac85ed7dfa 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataapproval/hibernate/HibernateDataApprovalStore.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataapproval/hibernate/HibernateDataApprovalStore.java @@ -645,7 +645,7 @@ public List getDataApprovalStatuses( "where not exists ( " + "select 1 " + "from categoryoptioncombos_categoryoptions cocco " - + "join dataelementcategoryoption co on co.categoryoptionid = cocco.categoryoptionid " + + "join categoryoption co on co.categoryoptionid = cocco.categoryoptionid " + "where cocco.categoryoptioncomboid = coc.categoryoptioncomboid " + "and ( " + diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/datastore/DefaultDatastoreService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/datastore/DefaultDatastoreService.java index a0285a6bd188..b2efe23e71b5 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/datastore/DefaultDatastoreService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/datastore/DefaultDatastoreService.java @@ -44,7 +44,6 @@ import org.hisp.dhis.feedback.BadRequestException; import org.hisp.dhis.feedback.ConflictException; import org.hisp.dhis.jsontree.JsonNode; -import org.hisp.dhis.render.RenderService; import org.hisp.dhis.security.acl.AclService; import org.hisp.dhis.user.CurrentUserService; import org.hisp.dhis.user.User; @@ -68,8 +67,6 @@ public class DefaultDatastoreService implements DatastoreService { private final AclService aclService; - private final RenderService renderService; - @Override public void addProtection(DatastoreNamespaceProtection protection) { protectionByNamespace.put(protection.getNamespace(), protection); diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java index 8fe78139c010..9b6332826c3b 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java @@ -28,6 +28,8 @@ package org.hisp.dhis.dimension; import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.commons.lang3.StringUtils.substringBefore; import static org.hisp.dhis.common.DimensionType.CATEGORY; import static org.hisp.dhis.common.DimensionType.CATEGORY_OPTION_GROUP_SET; import static org.hisp.dhis.common.DimensionType.DATA_ELEMENT_GROUP_SET; @@ -431,13 +433,22 @@ private void populateEventRepetitions( EventAnalyticalObject object, List dimensionalObjects, Attribute parent) { if (isNotEmpty(dimensionalObjects)) { for (DimensionalObject dimensionalObject : dimensionalObjects) { - boolean hasEventRepetition = dimensionalObject.getEventRepetition() != null; - boolean hasSameDimension = hasEventRepetition && dimensionalObject.getDimension() != null; + EventRepetition eventRepetition = dimensionalObject.getEventRepetition(); + String dimension = dimensionalObject.getDimension(); + boolean associateEventRepetition = eventRepetition != null && isNotBlank(dimension); - if (hasEventRepetition && hasSameDimension) { - EventRepetition eventRepetition = dimensionalObject.getEventRepetition(); + if (associateEventRepetition) { eventRepetition.setParent(parent); - eventRepetition.setDimension(dimensionalObject.getDimension()); + eventRepetition.setDimension(dimension); + + if (dimensionalObject.hasProgramStage()) { + eventRepetition.setProgramStage(dimensionalObject.getProgramStage().getUid()); + eventRepetition.setProgram(dimensionalObject.getProgramStage().getProgram().getUid()); + } + + if (dimensionalObject.hasProgram()) { + eventRepetition.setProgram(dimensionalObject.getProgram().getUid()); + } object.getEventRepetitions().add(eventRepetition); } @@ -472,6 +483,25 @@ private DataElementOperand getDataElementOperand( return new DataElementOperand(dataElement, categoryOptionCombo, attributeOptionCombo); } + /** + * Loads a program associated with the incoming program stage, if any, so it can be used later in + * the processing flow. + * + * @param dimensionalObject + */ + private void loadProgramForStage(DimensionalObject dimensionalObject) { + if (dimensionalObject.hasProgramStage()) { + // Sometimes we may have the index of the event, so it needs to be removed. ie.: + // "stage_uid[-2]". + String stageUid = substringBefore(dimensionalObject.getProgramStage().getUid(), "["); + ProgramStage programStage = idObjectManager.get(ProgramStage.class, stageUid); + + if (programStage != null) { + dimensionalObject.getProgramStage().setProgram(programStage.getProgram()); + } + } + } + /** * Sets persistent objects for dimensional associations on the given BaseAnalyticalObject based on * the given list of transient DimensionalObjects. @@ -490,10 +520,10 @@ private void mergeDimensionalObjects( } for (DimensionalObject dimension : dimensions) { - DimensionType type = getDimensionType(dimension.getDimension()); - String dimensionId = dimension.getDimension(); + loadProgramForStage(dimension); + DimensionType type = getDimensionType(dimensionId); List items = dimension.getItems(); if (items != null) { diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/dataitem/DimItemDataElementAndOperand.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/dataitem/DimItemDataElementAndOperand.java index 4cc6679200c7..48de7722eab0 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/dataitem/DimItemDataElementAndOperand.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/dataitem/DimItemDataElementAndOperand.java @@ -28,13 +28,20 @@ package org.hisp.dhis.expression.dataitem; import static org.apache.commons.lang3.ObjectUtils.anyNotNull; +import static org.hisp.dhis.analytics.DataType.BOOLEAN; +import static org.hisp.dhis.analytics.DataType.NUMERIC; +import static org.hisp.dhis.analytics.DataType.fromValueType; import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT_OPERAND; +import static org.hisp.dhis.parser.expression.ParserUtils.castSql; +import static org.hisp.dhis.parser.expression.ParserUtils.replaceSqlNull; import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.ExprContext; import static org.hisp.dhis.subexpression.SubexpressionDimensionItem.getItemColumnName; +import org.hisp.dhis.analytics.DataType; import org.hisp.dhis.antlr.ParserExceptionWithoutContext; import org.hisp.dhis.common.DimensionalItemId; +import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.parser.expression.CommonExpressionVisitor; /** @@ -70,7 +77,13 @@ public Object getSql(ExprContext ctx, CommonExpressionVisitor visitor) { String cocUid = (ctx.uid1 == null) ? null : ctx.uid1.getText(); String aocUid = (ctx.uid2 == null) ? null : ctx.uid2.getText(); - return getItemColumnName(deUid, cocUid, aocUid, visitor.getState().getQueryMods()); + String column = getItemColumnName(deUid, cocUid, aocUid, visitor.getState().getQueryMods()); + + if (visitor.getState().isReplaceNulls()) { + column = replaceDataElementNulls(column, deUid, visitor); + } + + return column; } // ------------------------------------------------------------------------- @@ -92,4 +105,34 @@ private boolean isDataElementOperandSyntax(ExprContext ctx) { return anyNotNull(ctx.uid1, ctx.uid2); } + + /** + * Replaces null data element values with the appropriate replacement value within a subexpression + * evaluation. This is always done unless the value is within a function that tests for null, such + * as isNull() or isNotNull(). + * + *

Note that boolean data elements are always aggregated as numeric in a subexpression, so if + * they are in a context where boolean is desired (such as the first argument of an if function), + * then cast them as boolean and use a boolean replacement value. Otherwise, boolean values are + * numeric. + */ + private String replaceDataElementNulls( + String column, String deUid, CommonExpressionVisitor visitor) { + DataElement dataElement = visitor.getIdObjectManager().get(DataElement.class, deUid); + if (dataElement == null) { + throw new ParserExceptionWithoutContext( + "Data element " + deUid + " not found during SQL generation."); + } + + DataType dataType = fromValueType(dataElement.getValueType()); + if (dataType == BOOLEAN) { + if (visitor.getParams().getDataType() == BOOLEAN) { + column = castSql(column, BOOLEAN); + } else { + dataType = NUMERIC; + } + } + + return replaceSqlNull(column, dataType); + } } diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/period/PeriodDataProvider.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/period/PeriodDataProvider.java index f150d41a49b7..9d87831c4a20 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/period/PeriodDataProvider.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/period/PeriodDataProvider.java @@ -30,6 +30,8 @@ import static java.time.LocalDate.now; import static java.util.Collections.sort; import static java.util.Collections.unmodifiableList; +import static org.apache.commons.collections4.CollectionUtils.isEmpty; +import static org.hisp.dhis.period.PeriodDataProvider.DataSource.SYSTEM_DEFINED; import java.util.ArrayList; import java.util.List; @@ -46,31 +48,40 @@ @Component @RequiredArgsConstructor public class PeriodDataProvider { - private static final int BEFORE_AND_AFTER_DATA_YEARS_SUPPORTED = 5; + static final int BEFORE_AND_AFTER_DATA_YEARS_SUPPORTED = 5; + + static final int DEFAULT_FIRST_YEAR_SUPPORTED = 1975; + + static final int DEFAULT_LATEST_YEAR_SUPPORTED = now().plusYears(25).getYear(); private final JdbcTemplate jdbcTemplate; + public enum DataSource { + SYSTEM_DEFINED, + DATABASE + } + /** * Returns a distinct union of all years available in the "event" table + "period" table, both * from aggregate and tracker, with 5 years previous and future additions. * *

ie: [extra_5_previous_years, data_years, extra_5_future_year] * + * @param source the source ({@link DataSource}) where the years wil be retrieved from. * @return an unmodifiable list of distinct years and the respective additions. */ - public List getAvailableYears() { - List availableDataYears = new ArrayList<>(fetchAvailableYears()); + public List getAvailableYears(DataSource source) { + List availableDataYears = new ArrayList<>(); - if (availableDataYears.isEmpty()) { - availableDataYears.add(now().getYear()); - } - - int firstYear = availableDataYears.get(0); - int lastYear = availableDataYears.get(availableDataYears.size() - 1); - - for (int i = 0; i < BEFORE_AND_AFTER_DATA_YEARS_SUPPORTED; i++) { - availableDataYears.add(--firstYear); - availableDataYears.add(++lastYear); + if (source == SYSTEM_DEFINED) { + // Add default hard-coded years (keeps it backward compatible). + for (int year = DEFAULT_FIRST_YEAR_SUPPORTED; year <= DEFAULT_LATEST_YEAR_SUPPORTED; year++) { + availableDataYears.add(year); + } + } else { + // Add years dynamically, based on the database. + availableDataYears.addAll(fetchAvailableYears()); + addSafetyBuffer(availableDataYears, BEFORE_AND_AFTER_DATA_YEARS_SUPPORTED); } sort(availableDataYears); @@ -78,10 +89,32 @@ public List getAvailableYears() { return unmodifiableList(availableDataYears); } + /** + * Adds some extra years (based on the buffer) to the given list of years. The extra years are + * added at the end of the list. + * + *

Let's say that the given list contains [2021, 2024], and the buffer is 3. This will result + * in a list like [2021, 2024, 2020, 2025, 2019, 2026, 2018, 2027]. + * + * @param years the list of years to append new years as buffer. + * @param buffer the buffer representing the amount of years to add. + */ + void addSafetyBuffer(List years, int buffer) { + int firstYear = years.get(0); + int lastYear = years.get(years.size() - 1); + + for (int i = 0; i < buffer; i++) { + years.add(--firstYear); + years.add(++lastYear); + } + } + /** * Queries the database in order to fetch all years available in the "period" and "event" tables. + * It does a distinct union of all years available in the "event" and "period" table, both from + * aggregate and tracker. If nothing is found, the current year is returned (as default). * - * @return the list of distinct years found. + * @return the list of distinct years found in the database, or current year. */ private List fetchAvailableYears() { String dueDateOrExecutionDate = @@ -101,6 +134,12 @@ private List fetchAvailableYears() { + " is not null" + " and ev.deleted is false ) order by datayear asc"; - return jdbcTemplate.queryForList(sql, Integer.class); + List years = jdbcTemplate.queryForList(sql, Integer.class); + + if (isEmpty(years)) { + years.add(now().getYear()); + } + + return years; } } diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/ProgramExpressionItem.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/ProgramExpressionItem.java index 79dcfdb4a8cb..52cca2b89166 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/ProgramExpressionItem.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/ProgramExpressionItem.java @@ -27,8 +27,11 @@ */ package org.hisp.dhis.program; -import static org.hisp.dhis.common.ValueType.BOOLEAN; +import static org.hisp.dhis.analytics.DataType.BOOLEAN; +import static org.hisp.dhis.analytics.DataType.NUMERIC; import static org.hisp.dhis.common.ValueType.NUMBER; +import static org.hisp.dhis.parser.expression.ParserUtils.castSql; +import static org.hisp.dhis.parser.expression.ParserUtils.replaceSqlNull; import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.ExprContext; import org.hisp.dhis.analytics.DataType; @@ -54,7 +57,6 @@ * @author Jim Grace */ public abstract class ProgramExpressionItem implements ExpressionItem { - private static final String COALESCE = "coalesce("; @Override public final Object getExpressionInfo(ExprContext ctx, CommonExpressionVisitor visitor) { @@ -103,7 +105,8 @@ protected ProgramExpressionItem getProgramArgType(ExprContext ctx) { * @return the replacement value */ protected Object getNullReplacementValue(ValueType valueType) { - return ValidationUtils.getNullReplacementValue((valueType == BOOLEAN) ? NUMBER : valueType); + return ValidationUtils.getNullReplacementValue( + (valueType == ValueType.BOOLEAN) ? NUMBER : valueType); } /** @@ -115,14 +118,10 @@ protected Object getNullReplacementValue(ValueType valueType) { */ protected String replaceNullSqlValues( String column, CommonExpressionVisitor visitor, ValueType valueType) { - if (valueType.isNumeric() || valueType.isBoolean()) { - if (visitor.getParams().getDataType() == DataType.BOOLEAN) { - return COALESCE + column + "::numeric!=0,false)"; - } else { - return COALESCE + column + "::numeric,0)"; - } + DataType dataType = DataType.fromValueType(valueType); + if (dataType == NUMERIC || dataType == BOOLEAN) { + dataType = visitor.getParams().getDataType() == BOOLEAN ? BOOLEAN : NUMERIC; } - - return COALESCE + column + "::text,'')"; + return replaceSqlNull(castSql(column, dataType), dataType); } } diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobConfigurationService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobConfigurationService.java index 8834132f9ab2..45c3fb54417d 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobConfigurationService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobConfigurationService.java @@ -45,7 +45,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -234,7 +233,7 @@ public List getJobConfigurations(JobType type) { @Override @Transactional(readOnly = true) public List getDueJobConfigurations( - int dueInNextSeconds, boolean limitToNext1, boolean includeWaiting) { + int dueInNextSeconds, boolean includeWaiting) { Instant now = Instant.now(); Instant endOfWindow = now.plusSeconds(dueInNextSeconds); Duration maxCronDelay = @@ -243,16 +242,7 @@ public List getDueJobConfigurations( jobConfigurationStore .getDueJobConfigurations(includeWaiting) .filter(c -> c.isDueBetween(now, endOfWindow, maxCronDelay)); - if (!limitToNext1) return dueJobs.toList(); - Set types = EnumSet.noneOf(JobType.class); - return dueJobs - .filter( - config -> { - if (types.contains(config.getJobType())) return false; - types.add(config.getJobType()); - return true; - }) - .toList(); + return dueJobs.toList(); } @Override diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerLoopService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerLoopService.java index 82a28acedd4a..1c8f9f4fd070 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerLoopService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerLoopService.java @@ -111,7 +111,7 @@ public void assureAsLeader(int ttlSeconds) { @Override @Transactional(readOnly = true) public List getDueJobConfigurations(int dueInNextSeconds) { - return jobConfigurationService.getDueJobConfigurations(dueInNextSeconds, true, false); + return jobConfigurationService.getDueJobConfigurations(dueInNextSeconds, false); } @Override @@ -121,6 +121,13 @@ public JobConfiguration getNextInQueue(String queue, int fromPosition) { return jobConfigurationStore.getNextInQueue(queue, fromPosition); } + @CheckForNull + @Override + @Transactional(readOnly = true) + public JobConfiguration getJobConfiguration(String jobId) { + return jobConfigurationStore.getByUid(jobId); + } + @Override @Transactional public boolean tryRun(@Nonnull String jobId) { diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/JobScheduler.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/JobScheduler.java index e832f008941e..0ce4060b804d 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/JobScheduler.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/JobScheduler.java @@ -34,12 +34,8 @@ import java.time.Instant; import java.time.ZoneId; import java.time.temporal.ChronoUnit; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CancellationException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; +import java.util.*; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import lombok.RequiredArgsConstructor; @@ -92,6 +88,7 @@ public class JobScheduler implements Runnable, JobRunner { private final JobSchedulerLoopService service; private final SystemSettingManager systemSettings; private final ExecutorService workers = Executors.newCachedThreadPool(); + private final Map> continuousJobsByType = new ConcurrentHashMap<>(); public void start() { long loopTimeMs = LOOP_SECONDS * 1000L; @@ -121,7 +118,7 @@ public void run() { service.getDueJobConfigurations(LOOP_SECONDS).stream() .collect(groupingBy(JobConfiguration::getJobType)); // only attempt to start one per type per loop invocation - readyByType.forEach((type, jobs) -> runIfDue(now, jobs.get(0))); + readyByType.forEach((type, jobs) -> runIfDue(now, type, jobs)); } } catch (Exception ex) { log.error("Exceptions thrown in scheduler loop", ex); @@ -129,13 +126,59 @@ public void run() { } } + private void runIfDue(Instant now, JobType type, List jobs) { + if (!type.isUsingContinuousExecution()) { + runIfDue(now, jobs.get(0)); + return; + } + Queue jobIds = + continuousJobsByType.computeIfAbsent(type, key -> new ConcurrentLinkedQueue<>()); + // add a worker either if no worker is on it (empty new queue) or if there are many jobs + boolean spawnWorker = jobIds.isEmpty(); + // add those IDs to the queue that are not yet in it + jobs.stream() + .map(JobConfiguration::getUid) + .filter(jobId -> !jobIds.contains(jobId)) + .forEach(jobIds::add); + if (spawnWorker) { + // we want to prevent starting more than one worker per job type + // but if this does happen it is no issue as both will be pulling + // from the same queue + workers.submit(() -> runContinuous(type)); + } + } + + private void runContinuous(JobType type) { + try { + Queue jobIds = continuousJobsByType.get(type); + String jobId = jobIds.poll(); + while (jobId != null) { + JobConfiguration config = service.getJobConfiguration(jobId); + if (config != null && config.getJobStatus() == JobStatus.SCHEDULED) { + Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); + Instant dueTime = dueTime(now, config); + runDueJob(config, dueTime); + } + jobId = jobIds.poll(); + } + } finally { + // need to be done so that we never have a queue without a worker by accident + continuousJobsByType.remove(type); + } + } + private void runIfDue(Instant now, JobConfiguration config) { + Instant dueTime = dueTime(now, config); + if (dueTime != null) { + workers.submit(() -> runDueJob(config, dueTime)); + } + } + + private Instant dueTime(Instant now, JobConfiguration config) { Duration maxCronDelay = Duration.ofHours(systemSettings.getIntSetting(SettingKey.JOBS_MAX_CRON_DELAY_HOURS)); Instant dueTime = config.nextExecutionTime(now, maxCronDelay); - if (dueTime != null && !dueTime.isAfter(now)) { - workers.submit(() -> runDueJob(config, dueTime)); - } + return dueTime != null && !dueTime.isAfter(now) ? dueTime : null; } @Override diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/JobSchedulerLoopService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/JobSchedulerLoopService.java index cec1c952f251..161aabedcf5a 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/JobSchedulerLoopService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/JobSchedulerLoopService.java @@ -63,6 +63,9 @@ public interface JobSchedulerLoopService { */ List getDueJobConfigurations(int dueInNextSeconds); + @CheckForNull + JobConfiguration getJobConfiguration(String jobId); + @CheckForNull JobConfiguration getNextInQueue(String queue, int fromPosition); diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/resources/i18n_global_es.properties b/dhis-2/dhis-services/dhis-service-core/src/main/resources/i18n_global_es.properties index c3aab1dc82ab..33bbedfd3d1b 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/resources/i18n_global_es.properties +++ b/dhis-2/dhis-services/dhis-service-core/src/main/resources/i18n_global_es.properties @@ -540,7 +540,7 @@ no_of_days=No de d\u00edas variables=Variables open=Abrir normal=Normal -item=Item +item=Elemento action=Acci\u00f3n sub_total=Sub total total=Total diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/category/hibernate/Category.hbm.xml b/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/category/hibernate/Category.hbm.xml index 7227949d585a..83cbc4ca5f4c 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/category/hibernate/Category.hbm.xml +++ b/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/category/hibernate/Category.hbm.xml @@ -6,7 +6,7 @@ > - + diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/category/hibernate/CategoryOption.hbm.xml b/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/category/hibernate/CategoryOption.hbm.xml index abdcd0c31f79..d4a15caf8de1 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/category/hibernate/CategoryOption.hbm.xml +++ b/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/category/hibernate/CategoryOption.hbm.xml @@ -6,7 +6,7 @@ > - + diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/eventvisualization/hibernate/EventVisualization.hbm.xml b/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/eventvisualization/hibernate/EventVisualization.hbm.xml index 57a7f1ca081c..5b4c266b9257 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/eventvisualization/hibernate/EventVisualization.hbm.xml +++ b/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/eventvisualization/hibernate/EventVisualization.hbm.xml @@ -116,11 +116,14 @@ + column="programid" foreign-key="fk_evisualization_programid"/> + + diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/relationship/hibernate/Relationship.hbm.xml b/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/relationship/hibernate/Relationship.hbm.xml index 83193abfc8f1..20277f34bd8a 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/relationship/hibernate/Relationship.hbm.xml +++ b/dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/relationship/hibernate/Relationship.hbm.xml @@ -43,5 +43,6 @@ + diff --git a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/period/PeriodDataProviderTest.java b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/period/PeriodDataProviderTest.java index dc2204ec9c71..edd13c5cb893 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/period/PeriodDataProviderTest.java +++ b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/period/PeriodDataProviderTest.java @@ -28,6 +28,11 @@ package org.hisp.dhis.period; import static java.time.LocalDate.now; +import static org.hisp.dhis.period.PeriodDataProvider.BEFORE_AND_AFTER_DATA_YEARS_SUPPORTED; +import static org.hisp.dhis.period.PeriodDataProvider.DEFAULT_FIRST_YEAR_SUPPORTED; +import static org.hisp.dhis.period.PeriodDataProvider.DEFAULT_LATEST_YEAR_SUPPORTED; +import static org.hisp.dhis.period.PeriodDataProvider.DataSource.DATABASE; +import static org.hisp.dhis.period.PeriodDataProvider.DataSource.SYSTEM_DEFINED; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -70,7 +75,7 @@ void shouldReturnFiveExtraYearsBeforeAndAfterDataYears() { when(jdbcTemplate.queryForList(anyString(), ArgumentMatchers.>any())) .thenReturn(storedDataYears); - List dataYears = periodDataProvider.getAvailableYears(); + List dataYears = periodDataProvider.getAvailableYears(DATABASE); assertEquals(12, dataYears.size()); assertTrue(dataYears.contains((dataYears.get(storedDataYears.size() - 1) + 5))); @@ -87,11 +92,33 @@ void shouldReturnFiveExtraYearsBeforeAndAfterCurrentYearWhenNoDataExists() { when(jdbcTemplate.queryForList(anyString(), ArgumentMatchers.>any())) .thenReturn(storedDataYears); - List dataYears = periodDataProvider.getAvailableYears(); + List dataYears = periodDataProvider.getAvailableYears(DATABASE); assertEquals(11, dataYears.size()); assertTrue(dataYears.contains(currentYear + 5)); assertTrue(dataYears.contains(currentYear - 5)); assertFalse(dataYears.contains(currentYear - 6)); } + + @Test + void testGetAvailableYearsFromSystemDefinedSource() { + int currentYear = now().getYear(); + + List years = periodDataProvider.getAvailableYears(SYSTEM_DEFINED); + + assertTrue(containsAllSystemDefined(years)); + assertTrue(years.contains(currentYear + BEFORE_AND_AFTER_DATA_YEARS_SUPPORTED)); + assertTrue(years.contains(currentYear - BEFORE_AND_AFTER_DATA_YEARS_SUPPORTED)); + assertTrue(years.contains(DEFAULT_LATEST_YEAR_SUPPORTED)); + assertTrue(years.contains(DEFAULT_FIRST_YEAR_SUPPORTED)); + } + + private boolean containsAllSystemDefined(List years) { + for (int year = DEFAULT_FIRST_YEAR_SUPPORTED; year <= DEFAULT_LATEST_YEAR_SUPPORTED; year++) { + if (!years.contains(year)) { + return false; + } + } + return true; + } } diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/config/ServiceConfig.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/config/ServiceConfig.java index a4dda82dd84e..7f4e801d25ed 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/config/ServiceConfig.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/config/ServiceConfig.java @@ -130,8 +130,6 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.retry.backoff.ExponentialBackOffPolicy; import org.springframework.retry.policy.SimpleRetryPolicy; import org.springframework.retry.support.RetryTemplate; @@ -178,11 +176,6 @@ private Map, T> byClass(Collection items) { .collect(Collectors.toMap(e -> (Class) e.getClass(), Functions.identity())); } - @Bean - public NamedParameterJdbcTemplate namedParameterJdbcTemplate(JdbcTemplate jdbcTemplate) { - return new NamedParameterJdbcTemplate(jdbcTemplate); - } - @Bean("retryTemplate") public RetryTemplate retryTemplate() { ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy(); diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/SpringDataValueSetStore.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/SpringDataValueSetStore.java index be486251fa04..0b678dd9aa8c 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/SpringDataValueSetStore.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/SpringDataValueSetStore.java @@ -374,7 +374,7 @@ private String getAttributeOptionComboClause(User user) { // Get inaccessible category options "where cocco.categoryoptionid not in ( " + "select co.categoryoptionid " - + "from dataelementcategoryoption co " + + "from categoryoption co " + " where " + JpaQueryUtils.generateSQlQueryForSharingCheck( "co.sharing", user, AclService.LIKE_READ_DATA) diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/deprecated/tracker/event/JdbcEventStore.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/deprecated/tracker/event/JdbcEventStore.java index 807adbe5e002..018e5face28e 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/deprecated/tracker/event/JdbcEventStore.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/deprecated/tracker/event/JdbcEventStore.java @@ -1735,7 +1735,7 @@ private String getCategoryOptionComboQuery(User user) { + " string_agg(co.uid, ';') as co_uids, count(co.categoryoptionid) as co_count" + " from categoryoptioncombo coc " + " inner join categoryoptioncombos_categoryoptions cocco on coc.categoryoptioncomboid = cocco.categoryoptioncomboid" - + " inner join dataelementcategoryoption co on cocco.categoryoptionid = co.categoryoptionid" + + " inner join categoryoption co on cocco.categoryoptionid = co.categoryoptionid" + " group by coc.categoryoptioncomboid "; if (!isSuper(user)) { diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/context/AttributeOptionComboLoader.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/context/AttributeOptionComboLoader.java index 2b1dc921a6bf..56a4728d5f03 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/context/AttributeOptionComboLoader.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/context/AttributeOptionComboLoader.java @@ -70,7 +70,7 @@ public class AttributeOptionComboLoader { + "join categoryoptioncombos_categoryoptions coco on coc.categoryoptioncomboid = coco.categoryoptioncomboid " + "join categorycombo c on co.categorycomboid = c.categorycomboid " + "join categorycombos_categories cc on c.categorycomboid = cc.categorycomboid " - + "join dataelementcategory dec on cc.categoryid = dec.categoryid where coc." + + "join category dec on cc.categoryid = dec.categoryid where coc." + "${resolvedScheme} " + "group by coc.categoryoptioncomboid, coc.uid, coc.code, coc.ignoreapproval, coc.name, cc_uid, cc_name"; @@ -309,7 +309,7 @@ private CategoryOption loadCategoryOption(IdScheme idScheme, String id) { final String sql = "select " + key - + ", uid, code, name, startdate, enddate, sharing from dataelementcategoryoption " + + ", uid, code, name, startdate, enddate, sharing from categoryoption " + "where " + resolveId(idScheme, key, id); diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/deprecated/tracker/trackedentity/store/DefaultEventStore.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/deprecated/tracker/trackedentity/store/DefaultEventStore.java index 39c35d23420f..4dc8ef465f7c 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/deprecated/tracker/trackedentity/store/DefaultEventStore.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/deprecated/tracker/trackedentity/store/DefaultEventStore.java @@ -110,7 +110,7 @@ private String getAttributeOptionComboClause(AggregateContext ctx) { // Get inaccessible category options "where cocco.categoryoptionid not in ( " + "select co.categoryoptionid " - + "from dataelementcategoryoption co " + + "from categoryoption co " + " where " + JpaQueryUtils.generateSQlQueryForSharingCheck( "co.sharing", ctx.getUserUid(), ctx.getUserGroups(), AclService.LIKE_READ_DATA) diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/deprecated/tracker/trackedentity/store/query/EventQuery.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/deprecated/tracker/trackedentity/store/query/EventQuery.java index 19fe7cb1bf82..e7d23c56931e 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/deprecated/tracker/trackedentity/store/query/EventQuery.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/deprecated/tracker/trackedentity/store/query/EventQuery.java @@ -75,7 +75,7 @@ public enum COLUMNS { new Subselect( "( " + "SELECT string_agg(opt.uid::text, ',') " - + "FROM dataelementcategoryoption opt " + + "FROM categoryoption opt " + "join categoryoptioncombos_categoryoptions ccc " + "on opt.categoryoptionid = ccc.categoryoptionid " + "WHERE coc.categoryoptioncomboid = ccc.categoryoptioncomboid )", diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/geojson/DefaultGeoJsonService.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/geojson/DefaultGeoJsonService.java index 51a06a57b27c..d42b21b02c0d 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/geojson/DefaultGeoJsonService.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/geojson/DefaultGeoJsonService.java @@ -59,6 +59,7 @@ import org.hisp.dhis.jsontree.JsonValue; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.organisationunit.OrganisationUnitStore; +import org.hisp.dhis.scheduling.parameters.GeoJsonImportJobParams; import org.hisp.dhis.security.acl.AclService; import org.hisp.dhis.system.util.GeoUtils; import org.locationtech.jts.geom.Geometry; @@ -110,7 +111,7 @@ public GeoJsonImportReport deleteGeoData(String attributeId) { @Override @Transactional public GeoJsonImportReport importGeoData( - GeoJsonImportParams params, InputStream geoJsonFeatureCollection) { + GeoJsonImportJobParams params, InputStream geoJsonFeatureCollection) { GeoJsonImportReport report = new GeoJsonImportReport(); Attribute attribute = validateAttribute(params.getAttributeId(), report); if (report.getConflictCount() > 0) { @@ -183,7 +184,7 @@ public GeoJsonImportReport importGeoData( } private Function getGeoJsonFeatureToOrgUnitIdentifier( - GeoJsonImportParams params) { + GeoJsonImportJobParams params) { switch (params.getIdType()) { case CODE: return OrganisationUnit::getCode; @@ -195,7 +196,7 @@ private Function getGeoJsonFeatureToOrgUnitIdentifier( } private List fetchOrganisationUnits( - GeoJsonImportParams params, @Nonnull Set ouIdentifiers) { + GeoJsonImportJobParams params, @Nonnull Set ouIdentifiers) { switch (params.getIdType()) { case CODE: return organisationUnitStore.getByCode(ouIdentifiers, params.getUser()); @@ -230,7 +231,7 @@ private Attribute validateAttribute(String attributeId, GeoJsonImportReport repo } private boolean validateGeometry( - GeoJsonImportParams params, + GeoJsonImportJobParams params, OrganisationUnit target, JsonObject geometry, GeoJsonImportReport report, @@ -298,7 +299,7 @@ private Geometry validateGeometry(GeoJsonImportReport report, GeometryUpdate upd } private void updateGeometry( - GeoJsonImportParams params, + GeoJsonImportJobParams params, Attribute attribute, OrganisationUnit target, JsonObject geometry, @@ -373,7 +374,7 @@ private boolean updateGeometryProperty(GeoJsonImportReport report, GeometryUpdat } private void executeUpdate( - GeoJsonImportParams params, GeoJsonImportReport report, GeometryUpdate update) { + GeoJsonImportJobParams params, GeoJsonImportReport report, GeometryUpdate update) { try { if (update.needsUpdate() && !params.isDryRun()) { organisationUnitStore.update(update.target()); diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/geojson/GeoJsonService.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/geojson/GeoJsonService.java index 9e36bff56056..7deb56e3b648 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/geojson/GeoJsonService.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/geojson/GeoJsonService.java @@ -28,6 +28,7 @@ package org.hisp.dhis.dxf2.geojson; import java.io.InputStream; +import org.hisp.dhis.scheduling.parameters.GeoJsonImportJobParams; /** * Service for handling import/export of GeoJSON. @@ -42,7 +43,7 @@ public interface GeoJsonService { * @param geoJsonData expected to be a GeoJSON feature-collection root object * @return a report with statistics and conflicts */ - GeoJsonImportReport importGeoData(GeoJsonImportParams params, InputStream geoJsonData); + GeoJsonImportReport importGeoData(GeoJsonImportJobParams params, InputStream geoJsonData); /** * Clears all GeoJSON data from all organisation units for the given attribute. If the attribute diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/geojson/job/GeoJsonImportJob.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/geojson/job/GeoJsonImportJob.java new file mode 100644 index 000000000000..7b396cf79c12 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/geojson/job/GeoJsonImportJob.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.dxf2.geojson.job; + +import java.io.IOException; +import java.io.InputStream; +import lombok.AllArgsConstructor; +import org.hisp.dhis.dxf2.geojson.GeoJsonImportReport; +import org.hisp.dhis.dxf2.geojson.GeoJsonService; +import org.hisp.dhis.fileresource.FileResource; +import org.hisp.dhis.fileresource.FileResourceService; +import org.hisp.dhis.scheduling.Job; +import org.hisp.dhis.scheduling.JobConfiguration; +import org.hisp.dhis.scheduling.JobProgress; +import org.hisp.dhis.scheduling.JobType; +import org.hisp.dhis.scheduling.parameters.GeoJsonImportJobParams; +import org.hisp.dhis.system.notification.Notifier; +import org.springframework.stereotype.Component; + +@AllArgsConstructor +@Component +public class GeoJsonImportJob implements Job { + + private final GeoJsonService geoJsonService; + private final FileResourceService fileResourceService; + private final Notifier notifier; + + @Override + public JobType getJobType() { + return JobType.GEOJSON_IMPORT; + } + + @Override + public void execute(JobConfiguration jobConfig, JobProgress progress) { + progress.startingProcess("GeoJSON import started"); + GeoJsonImportJobParams jobParams = (GeoJsonImportJobParams) jobConfig.getJobParameters(); + + progress.startingStage("Loading file resource"); + FileResource data = + progress.runStage(() -> fileResourceService.getFileResource(jobConfig.getUid())); + progress.startingStage("Loading file content"); + + try (InputStream input = + progress.runStage(() -> fileResourceService.getFileResourceContent(data))) { + progress.startingStage("Importing GeoJSON"); + GeoJsonImportReport report = + progress.runStage(() -> geoJsonService.importGeoData(jobParams, input)); + progress.completedProcess("GeoJSON import completed : " + report.getImportCount()); + notifier.addJobSummary(jobConfig, report, GeoJsonImportReport.class); + } catch (IOException e) { + progress.failedProcess(e); + } + } +} diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/webmessage/WebMessageUtils.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/webmessage/WebMessageUtils.java index 173cb8eb2cd1..55e9842b059b 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/webmessage/WebMessageUtils.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/webmessage/WebMessageUtils.java @@ -268,11 +268,24 @@ public static WebMessage createWebMessage(SQLException ex) { } public static ErrorCode getErrorCode(SQLException ex) { - String sqlState = Optional.of(ex).map(SQLException::getSQLState).orElse(""); - - if (StringUtils.isNotBlank(sqlState) && (sqlState.equals("42P01"))) { + if (relationDoesNotExist(ex)) { return ErrorCode.E7144; } return ErrorCode.E7145; } + + /** + * Utility method to detect if the {@link SQLException} refers to a missing relation in the + * database. + * + * @param ex a {@link SQLException} to analyze + * @return true if the error is a missing relation error, false otherwise + */ + public static boolean relationDoesNotExist(SQLException ex) { + if (ex != null) { + return Optional.of(ex).map(SQLException::getSQLState).filter("42P01"::equals).isPresent(); + } + + return false; + } } diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/context/AttributeOptionComboLoaderTest.java b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/context/AttributeOptionComboLoaderTest.java index 14ac5443adab..c5a6a552f9eb 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/context/AttributeOptionComboLoaderTest.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/context/AttributeOptionComboLoaderTest.java @@ -135,7 +135,7 @@ void verifyGetAttributeOptionCombo() { when(jdbcTemplate.queryForObject( eq( - "select categoryoptionid, uid, code, name, startdate, enddate, sharing from dataelementcategoryoption where uid = 'abcdef'"), + "select categoryoptionid, uid, code, name, startdate, enddate, sharing from categoryoption where uid = 'abcdef'"), any(RowMapper.class))) .thenReturn(categoryOption); diff --git a/dhis-2/dhis-services/dhis-service-setting/src/main/java/org/hisp/dhis/setting/SettingKey.java b/dhis-2/dhis-services/dhis-service-setting/src/main/java/org/hisp/dhis/setting/SettingKey.java index 4295fa651e7d..854bae241956 100644 --- a/dhis-2/dhis-services/dhis-service-setting/src/main/java/org/hisp/dhis/setting/SettingKey.java +++ b/dhis-2/dhis-services/dhis-service-setting/src/main/java/org/hisp/dhis/setting/SettingKey.java @@ -273,7 +273,7 @@ public enum SettingKey { "keyAnalyticsCacheTtlMode", AnalyticsCacheTtlMode.FIXED, AnalyticsCacheTtlMode.class), /** The offset of years used during period generation during the analytics export process. */ - ANALYTICS_MAX_PERIOD_YEARS_OFFSET("keyAnalyticsPeriodYearsOffset", 22, Integer.class), + ANALYTICS_MAX_PERIOD_YEARS_OFFSET("keyAnalyticsPeriodYearsOffset", -1, Integer.class), /** * @deprecated use {@link #TRACKED_ENTITY_MAX_LIMIT} instead diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/DefaultEnrollmentService.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/DefaultEnrollmentService.java index 5afa2c546a99..6c61016c279f 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/DefaultEnrollmentService.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/DefaultEnrollmentService.java @@ -27,13 +27,8 @@ */ package org.hisp.dhis.tracker.export.enrollment; -import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; -import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; import static org.hisp.dhis.common.OrganisationUnitSelectionMode.ACCESSIBLE; -import static org.hisp.dhis.common.OrganisationUnitSelectionMode.ALL; import static org.hisp.dhis.common.OrganisationUnitSelectionMode.CHILDREN; -import static org.hisp.dhis.common.Pager.DEFAULT_PAGE_SIZE; -import static org.hisp.dhis.common.SlimPager.FIRST_PAGE; import java.util.ArrayList; import java.util.HashSet; @@ -45,8 +40,6 @@ import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.common.IllegalQueryException; import org.hisp.dhis.common.OrganisationUnitSelectionMode; -import org.hisp.dhis.common.Pager; -import org.hisp.dhis.common.SlimPager; import org.hisp.dhis.feedback.BadRequestException; import org.hisp.dhis.feedback.ForbiddenException; import org.hisp.dhis.feedback.NotFoundException; @@ -61,12 +54,16 @@ import org.hisp.dhis.trackedentity.TrackerAccessManager; import org.hisp.dhis.trackedentity.TrackerOwnershipManager; import org.hisp.dhis.trackedentityattributevalue.TrackedEntityAttributeValue; +import org.hisp.dhis.tracker.export.Page; +import org.hisp.dhis.tracker.export.PageParams; import org.hisp.dhis.user.CurrentUserService; import org.hisp.dhis.user.User; import org.hisp.dhis.util.DateUtils; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; @Slf4j +@Transactional(readOnly = true) @RequiredArgsConstructor @Service("org.hisp.dhis.tracker.export.enrollment.EnrollmentService") class DefaultEnrollmentService @@ -194,8 +191,8 @@ private Set getTrackedEntityAttributeValues( } @Override - public Enrollments getEnrollments(EnrollmentOperationParams params) - throws ForbiddenException, BadRequestException { + public List getEnrollments(EnrollmentOperationParams params) + throws ForbiddenException, BadRequestException, NotFoundException { EnrollmentQueryParams queryParams = paramsMapper.map(params); decideAccess(queryParams); @@ -217,27 +214,42 @@ public Enrollments getEnrollments(EnrollmentOperationParams params) queryParams.setOrganisationUnits(organisationUnits); } - List enrollmentList = - getEnrollments( - new ArrayList<>(enrollmentStore.getEnrollments(queryParams)), - params.getEnrollmentParams(), - params.isIncludeDeleted()); + return getEnrollments( + new ArrayList<>(enrollmentStore.getEnrollments(queryParams)), + params.getEnrollmentParams(), + params.isIncludeDeleted()); + } - if (params.isSkipPaging()) { - return Enrollments.withoutPagination(enrollmentList); - } + @Override + public Page getEnrollments(EnrollmentOperationParams params, PageParams pageParams) + throws ForbiddenException, BadRequestException, NotFoundException { + EnrollmentQueryParams queryParams = paramsMapper.map(params); + + decideAccess(queryParams); + validate(queryParams); + + User user = currentUserService.getCurrentUser(); + + if (user != null + && queryParams.isOrganisationUnitMode(OrganisationUnitSelectionMode.ACCESSIBLE)) { + queryParams.setOrganisationUnits(user.getTeiSearchOrganisationUnitsWithFallback()); + queryParams.setOrganisationUnitMode(OrganisationUnitSelectionMode.DESCENDANTS); + } else if (queryParams.isOrganisationUnitMode(CHILDREN)) { + Set organisationUnits = new HashSet<>(queryParams.getOrganisationUnits()); - Pager pager; + for (OrganisationUnit organisationUnit : queryParams.getOrganisationUnits()) { + organisationUnits.addAll(organisationUnit.getChildren()); + } - if (params.isTotalPages()) { - queryParams.setSkipPaging(true); - int count = enrollmentStore.countEnrollments(queryParams); - pager = new Pager(params.getPageWithDefault(), count, params.getPageSizeWithDefault()); - } else { - pager = handleLastPageFlag(queryParams, enrollmentList); + queryParams.setOrganisationUnits(organisationUnits); } - return Enrollments.of(enrollmentList, pager); + Page enrollmentsPage = enrollmentStore.getEnrollments(queryParams, pageParams); + List enrollments = + getEnrollments( + enrollmentsPage.getItems(), params.getEnrollmentParams(), params.isIncludeDeleted()); + + return Page.of(enrollments, enrollmentsPage.getPager()); } public void decideAccess(EnrollmentQueryParams params) { @@ -274,11 +286,6 @@ public void validate(EnrollmentQueryParams params) throws IllegalQueryException User user = params.getUser(); - if (!params.hasOrganisationUnits() - && !(params.isOrganisationUnitMode(ALL) || params.isOrganisationUnitMode(ACCESSIBLE))) { - violation = "At least one organisation unit must be specified"; - } - if (params.isOrganisationUnitMode(ACCESSIBLE) && (user == null || !user.hasDataViewOrganisationUnitWithFallback())) { violation = @@ -321,37 +328,6 @@ public void validate(EnrollmentQueryParams params) throws IllegalQueryException } } - /** - * This method will apply the logic related to the parameter 'totalPages=false'. This works in - * conjunction with the method: {@link - * HibernateEnrollmentStore#getEnrollments(EnrollmentQueryParams)} - * - *

This is needed because we need to query (pageSize + 1) at DB level. The resulting query will - * allow us to evaluate if we are in the last page or not. And this is what his method does, - * returning the respective Pager object. - * - * @param params the request params - * @param enrollments the reference to the list of Enrollment - * @return the populated SlimPager instance - */ - private Pager handleLastPageFlag(EnrollmentQueryParams params, List enrollments) { - Integer originalPage = defaultIfNull(params.getPage(), FIRST_PAGE); - Integer originalPageSize = defaultIfNull(params.getPageSize(), DEFAULT_PAGE_SIZE); - boolean isLastPage = false; - - if (isNotEmpty(enrollments)) { - isLastPage = enrollments.size() <= originalPageSize; - if (!isLastPage) { - // Get the same number of elements of the pageSize, forcing - // the removal of the last additional element added at querying - // time. - enrollments.retainAll(enrollments.subList(0, originalPageSize)); - } - } - - return new SlimPager(originalPage, originalPageSize, isLastPage); - } - private List getEnrollments( Iterable enrollments, EnrollmentParams params, boolean includeDeleted) throws ForbiddenException { diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentOperationParams.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentOperationParams.java index e1885ad4ad61..620fcf52dd0e 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentOperationParams.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentOperationParams.java @@ -45,14 +45,13 @@ @Builder(toBuilder = true) @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public class EnrollmentOperationParams { - public static final int DEFAULT_PAGE = 1; - - public static final int DEFAULT_PAGE_SIZE = 50; - static final EnrollmentOperationParams EMPTY = EnrollmentOperationParams.builder().build(); @Builder.Default private final EnrollmentParams enrollmentParams = EnrollmentParams.FALSE; + /** Set of te uids to explicitly select. */ + @Builder.Default private final Set enrollmentUids = new HashSet<>(); + /** Last updated for enrollment. */ private final Date lastUpdated; @@ -89,38 +88,11 @@ public class EnrollmentOperationParams { /** Tracked entity. */ private final String trackedEntityUid; - /** Page number. */ - private final Integer page; - - /** Page size. */ - private final Integer pageSize; - - /** Indicates whether to include the total number of pages in the paging response. */ - private final boolean totalPages; - - /** Indicates whether paging should be skipped. */ - private final boolean skipPaging; - /** Indicates whether to include soft-deleted enrollments */ private final boolean includeDeleted; private final List order; - /** Indicates whether paging is enabled. */ - public boolean isPaging() { - return page != null || pageSize != null; - } - - /** Returns the page number, falls back to default value of 1 if not specified. */ - public int getPageWithDefault() { - return page != null && page > 0 ? page : DEFAULT_PAGE; - } - - /** Returns the page size, falls back to default value of 50 if not specified. */ - public int getPageSizeWithDefault() { - return pageSize != null && pageSize >= 0 ? pageSize : DEFAULT_PAGE_SIZE; - } - public static class EnrollmentOperationParamsBuilder { private List order = new ArrayList<>(); diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentOperationParamsMapper.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentOperationParamsMapper.java index 5bb84ef4b766..afeedfbd83a1 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentOperationParamsMapper.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentOperationParamsMapper.java @@ -85,13 +85,10 @@ public EnrollmentQueryParams map(EnrollmentOperationParams operationParams) params.setTrackedEntity(trackedEntity); params.addOrganisationUnits(orgUnits); params.setOrganisationUnitMode(operationParams.getOrgUnitMode()); - params.setPage(operationParams.getPage()); - params.setPageSize(operationParams.getPageSize()); - params.setSkipPaging(operationParams.isSkipPaging()); - params.setTotalPages(operationParams.isTotalPages()); params.setIncludeDeleted(operationParams.isIncludeDeleted()); params.setUser(user); params.setOrder(operationParams.getOrder()); + params.setEnrollmentUids(operationParams.getEnrollmentUids()); return params; } diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentQueryParams.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentQueryParams.java index 2900bb243222..d80e33fd2c5a 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentQueryParams.java @@ -27,6 +27,8 @@ */ package org.hisp.dhis.tracker.export.enrollment; +import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; + import java.util.Date; import java.util.HashSet; import java.util.List; @@ -49,9 +51,9 @@ @Data @Accessors(chain = true) class EnrollmentQueryParams { - public static final int DEFAULT_PAGE = 1; - public static final int DEFAULT_PAGE_SIZE = 50; + /** Set of enrollment uids to explicitly select. */ + private Set enrollmentUids = new HashSet<>(); /** Last updated for enrollment. */ private Date lastUpdated; @@ -91,18 +93,6 @@ class EnrollmentQueryParams { /** Tracked entity instance. */ private TrackedEntity trackedEntity; - /** Page number. */ - private Integer page; - - /** Page size. */ - private Integer pageSize; - - /** Indicates whether to include the total number of pages in the paging response. */ - private boolean totalPages; - - /** Indicates whether paging should be skipped. */ - private boolean skipPaging; - /** Indicates whether to include soft-deleted enrollments */ private boolean includeDeleted; @@ -187,38 +177,15 @@ public boolean hasTrackedEntity() { return this.trackedEntity != null; } + public boolean hasEnrollmentUids() { + return isNotEmpty(this.enrollmentUids); + } + /** Indicates whether this params is of the given organisation unit mode. */ public boolean isOrganisationUnitMode(OrganisationUnitSelectionMode mode) { return organisationUnitMode != null && organisationUnitMode.equals(mode); } - /** Indicates whether paging is enabled. */ - public boolean isPaging() { - return page != null || pageSize != null; - } - - /** Returns the page number, falls back to default value of 1 if not specified. */ - public int getPageWithDefault() { - return page != null && page > 0 ? page : DEFAULT_PAGE; - } - - /** Returns the page size, falls back to default value of 50 if not specified. */ - public int getPageSizeWithDefault() { - return pageSize != null && pageSize >= 0 ? pageSize : DEFAULT_PAGE_SIZE; - } - - /** Returns the offset based on the page number and page size. */ - public int getOffset() { - return (getPageWithDefault() - 1) * getPageSizeWithDefault(); - } - - /** Sets paging properties to default values. */ - public void setDefaultPaging() { - this.page = DEFAULT_PAGE; - this.pageSize = DEFAULT_PAGE_SIZE; - this.skipPaging = false; - } - /** * Order by an enrollment field of the given {@code field} name in given sort {@code direction}. */ diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentService.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentService.java index 3680c5b3a6b6..75738283b3a7 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentService.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentService.java @@ -27,11 +27,14 @@ */ package org.hisp.dhis.tracker.export.enrollment; +import java.util.List; import java.util.Set; import org.hisp.dhis.feedback.BadRequestException; import org.hisp.dhis.feedback.ForbiddenException; import org.hisp.dhis.feedback.NotFoundException; import org.hisp.dhis.program.Enrollment; +import org.hisp.dhis.tracker.export.Page; +import org.hisp.dhis.tracker.export.PageParams; public interface EnrollmentService { Enrollment getEnrollment(String uid, EnrollmentParams params, boolean includeDeleted) @@ -40,8 +43,13 @@ Enrollment getEnrollment(String uid, EnrollmentParams params, boolean includeDel Enrollment getEnrollment(Enrollment enrollment, EnrollmentParams params, boolean includeDeleted) throws ForbiddenException; - Enrollments getEnrollments(EnrollmentOperationParams params) - throws ForbiddenException, BadRequestException; + /** Get all enrollments matching given params. */ + List getEnrollments(EnrollmentOperationParams params) + throws BadRequestException, ForbiddenException, NotFoundException; + + /** Get a page of enrollments matching given params. */ + Page getEnrollments(EnrollmentOperationParams params, PageParams pageParams) + throws BadRequestException, ForbiddenException, NotFoundException; /** * Fields the {@link #getEnrollments(EnrollmentOperationParams)} can order enrollments by. diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentStore.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentStore.java index 7df07362b425..674371d3bafa 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentStore.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentStore.java @@ -31,6 +31,8 @@ import java.util.Set; import org.hisp.dhis.common.IdentifiableObjectStore; import org.hisp.dhis.program.Enrollment; +import org.hisp.dhis.tracker.export.Page; +import org.hisp.dhis.tracker.export.PageParams; interface EnrollmentStore extends IdentifiableObjectStore { String ID = EnrollmentStore.class.getName(); @@ -43,14 +45,12 @@ interface EnrollmentStore extends IdentifiableObjectStore { */ int countEnrollments(EnrollmentQueryParams params); - /** - * Get all enrollments by enrollment query params. - * - * @param params EnrollmentQueryParams to use - * @return Enrollments matching params - */ + /** Get all enrollments matching given params. */ List getEnrollments(EnrollmentQueryParams params); + /** Get a page of enrollments matching given params. */ + Page getEnrollments(EnrollmentQueryParams params, PageParams pageParams); + /** * Fields the {@link #getEnrollments(EnrollmentQueryParams)} can order enrollments by. Ordering by * fields other than these is considered a programmer error. Validation of user provided field diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/HibernateEnrollmentStore.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/HibernateEnrollmentStore.java index aca2e24fdaa9..79a0d3cdf3a0 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/HibernateEnrollmentStore.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/HibernateEnrollmentStore.java @@ -38,6 +38,7 @@ import java.util.Set; import java.util.StringJoiner; import java.util.function.Function; +import java.util.function.IntSupplier; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.persistence.criteria.CriteriaBuilder; @@ -49,12 +50,15 @@ import org.hibernate.SessionFactory; import org.hibernate.query.Query; import org.hisp.dhis.common.OrganisationUnitSelectionMode; +import org.hisp.dhis.common.Pager; import org.hisp.dhis.common.hibernate.SoftDeleteHibernateObjectStore; import org.hisp.dhis.commons.util.SqlHelper; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.program.Enrollment; import org.hisp.dhis.security.acl.AclService; import org.hisp.dhis.tracker.export.Order; +import org.hisp.dhis.tracker.export.Page; +import org.hisp.dhis.tracker.export.PageParams; import org.hisp.dhis.user.CurrentUserService; import org.springframework.context.ApplicationEventPublisher; import org.springframework.jdbc.core.JdbcTemplate; @@ -118,26 +122,46 @@ public List getEnrollments(EnrollmentQueryParams params) { Query query = getQuery(hql); - if (!params.isSkipPaging()) { - query.setFirstResult(params.getOffset()); - query.setMaxResults(params.getPageSizeWithDefault()); - } + return query.list(); + } + + @Override + public Page getEnrollments(EnrollmentQueryParams params, PageParams pageParams) { + String hql = buildEnrollmentHql(params).getFullQuery(); + + Query query = getQuery(hql); + query.setFirstResult((pageParams.getPage() - 1) * pageParams.getPageSize()); + query.setMaxResults(pageParams.getPageSize()); - // When the clients choose to not show the total of pages. - if (!params.isTotalPages() && !params.isSkipPaging()) { - // Get pageSize + 1, so we are able to know if there is another - // page available. It adds one additional element into the list, - // as consequence. The caller needs to remove the last element. - query.setMaxResults(params.getPageSizeWithDefault() + 1); + IntSupplier enrollmentCount = () -> countEnrollments(params); + return getPage(pageParams, query.list(), enrollmentCount); + } + + private Page getPage( + PageParams pageParams, List enrollments, IntSupplier enrollmentCount) { + if (pageParams.isPageTotal()) { + Pager pager = + new Pager(pageParams.getPage(), enrollmentCount.getAsInt(), pageParams.getPageSize()); + return Page.of(enrollments, pager); } - return query.list(); + Pager pager = new Pager(pageParams.getPage(), 0, pageParams.getPageSize()); + pager.force(pageParams.getPage(), pageParams.getPageSize()); + return Page.of(enrollments, pager); } private QueryWithOrderBy buildEnrollmentHql(EnrollmentQueryParams params) { String hql = "from Enrollment en"; SqlHelper hlp = new SqlHelper(true); + if (params.hasEnrollmentUids()) { + hql += + hlp.whereAnd() + + "en.uid in (" + + getQuotedCommaDelimitedString(params.getEnrollmentUids()) + + ")"; + } + if (params.hasLastUpdatedDuration()) { hql += hlp.whereAnd() diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/EventQuery.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/EventQuery.java index 80953e877180..3e8d6e5e7ddb 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/EventQuery.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/EventQuery.java @@ -65,7 +65,7 @@ public enum COLUMNS { new Subselect( "( " + "SELECT string_agg(opt.uid::text, ',') " - + "FROM dataelementcategoryoption opt " + + "FROM categoryoption opt " + "join categoryoptioncombos_categoryoptions ccc " + "on opt.categoryoptionid = ccc.categoryoptionid " + "WHERE coc.categoryoptioncomboid = ccc.categoryoptioncomboid )", diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/JdbcEventStore.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/JdbcEventStore.java index 55a847928716..d67b27d3c8df 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/JdbcEventStore.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/JdbcEventStore.java @@ -197,7 +197,7 @@ class JdbcEventStore implements EventStore { entry("organisationUnit.uid", COLUMN_ORG_UNIT_UID), entry("enrollment.trackedEntity.uid", COLUMN_TRACKEDENTITY_UID), entry("executionDate", COLUMN_EVENT_EXECUTION_DATE), - entry("enrollment.followup", COLUMN_ENROLLMENT_FOLLOWUP), + entry("enrollment.followUp", COLUMN_ENROLLMENT_FOLLOWUP), entry("status", COLUMN_EVENT_STATUS), entry("dueDate", COLUMN_EVENT_DUE_DATE), entry("storedBy", COLUMN_EVENT_STORED_BY), @@ -1393,7 +1393,7 @@ private String getCategoryOptionComboQuery(User user) { + " string_agg(co.uid, ',') as co_uids, count(co.categoryoptionid) as co_count" + " from categoryoptioncombo coc " + " inner join categoryoptioncombos_categoryoptions cocco on coc.categoryoptioncomboid = cocco.categoryoptioncomboid" - + " inner join dataelementcategoryoption co on cocco.categoryoptionid = co.categoryoptionid" + + " inner join categoryoption co on cocco.categoryoptionid = co.categoryoptionid" + " group by coc.categoryoptioncomboid "; if (!isSuper(user)) { diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/relationship/DefaultRelationshipService.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/relationship/DefaultRelationshipService.java index a6505e35378a..01937ef0929d 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/relationship/DefaultRelationshipService.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/relationship/DefaultRelationshipService.java @@ -188,6 +188,7 @@ private Relationship map(Relationship relationship) { result.setRelationshipType(relationship.getRelationshipType()); result.setFrom(withNestedEntity(relationship.getFrom())); result.setTo(withNestedEntity(relationship.getTo())); + result.setCreatedAtClient(relationship.getCreatedAtClient()); return result; } diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/relationship/HibernateRelationshipStore.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/relationship/HibernateRelationshipStore.java index 3fa5a36e1f73..ee08284d38e7 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/relationship/HibernateRelationshipStore.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/relationship/HibernateRelationshipStore.java @@ -65,7 +65,7 @@ class HibernateRelationshipStore extends SoftDeleteHibernateObjectStore ORDERABLE_FIELDS = Set.of("created"); + private static final Set ORDERABLE_FIELDS = Set.of("created", "createdAtClient"); private static final String TRACKED_ENTITY = "trackedEntity"; diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/TrackedEntityQueryParams.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/TrackedEntityQueryParams.java index 7172eab56e83..6ff1fc60719f 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/TrackedEntityQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/TrackedEntityQueryParams.java @@ -117,8 +117,8 @@ public class TrackedEntityQueryParams { /** Tracked entity types to fetch. */ private List trackedEntityTypes = Lists.newArrayList(); - /** Selection mode for the specified organisation units, default is DESCENDANTS. */ - private OrganisationUnitSelectionMode orgUnitMode = OrganisationUnitSelectionMode.DESCENDANTS; + /** Selection mode for the specified organisation units */ + private OrganisationUnitSelectionMode orgUnitMode; private AssignedUserQueryParam assignedUserQueryParam = AssignedUserQueryParam.ALL; diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/AbstractStore.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/AbstractStore.java index 9a89c8dd9093..187589c677f1 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/AbstractStore.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/AbstractStore.java @@ -51,7 +51,7 @@ abstract class AbstractStore { + "where ri.%s in (:ids)"; private static final String GET_RELATIONSHIP_BY_RELATIONSHIP_ID = "select " - + "r.uid as rel_uid, r.created, r.lastupdated, rst.name as reltype_name, rst.uid as reltype_uid, rst.bidirectional as reltype_bi, " + + "r.uid as rel_uid, r.created, r.createdatclient, r.lastupdated, rst.name as reltype_name, rst.uid as reltype_uid, rst.bidirectional as reltype_bi, " + "coalesce((select 'te|' || te.uid from trackedentity te " + "join relationshipitem ri on te.trackedentityid = ri.trackedentityid " + "where ri.relationshipitemid = r.to_relationshipitemid) , (select 'en|' || en.uid " diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/DefaultEventStore.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/DefaultEventStore.java index f7c982734f8d..9c3b05d2ddfe 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/DefaultEventStore.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/DefaultEventStore.java @@ -108,7 +108,7 @@ private String getAttributeOptionComboClause(Context ctx) { // Get inaccessible category options "where cocco.categoryoptionid not in ( " + "select co.categoryoptionid " - + "from dataelementcategoryoption co " + + "from categoryoption co " + " where " + JpaQueryUtils.generateSQlQueryForSharingCheck( "co.sharing", ctx.getUserUid(), ctx.getUserGroups(), AclService.LIKE_READ_DATA) diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/mapper/EnrollmentRowCallbackHandler.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/mapper/EnrollmentRowCallbackHandler.java index 43e7642613a5..568b8c57e093 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/mapper/EnrollmentRowCallbackHandler.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/mapper/EnrollmentRowCallbackHandler.java @@ -91,8 +91,8 @@ private Enrollment getEnrollment(ResultSet rs) throws SQLException { program.setUid(rs.getString(EnrollmentQuery.getColumnName(COLUMNS.PROGRAM_UID))); enrollment.setProgram(program); - final boolean followup = rs.getBoolean(EnrollmentQuery.getColumnName(COLUMNS.FOLLOWUP)); - enrollment.setFollowup(rs.wasNull() ? null : followup); + final boolean followUp = rs.getBoolean(EnrollmentQuery.getColumnName(COLUMNS.FOLLOWUP)); + enrollment.setFollowup(rs.wasNull() ? null : followUp); enrollment.setStatus( ProgramStatus.valueOf(rs.getString(EnrollmentQuery.getColumnName(COLUMNS.STATUS)))); enrollment.setEnrollmentDate( diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/mapper/EventRowCallbackHandler.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/mapper/EventRowCallbackHandler.java index 2347362c37f9..456cef3ce660 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/mapper/EventRowCallbackHandler.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/mapper/EventRowCallbackHandler.java @@ -95,8 +95,8 @@ private Event getEvent(ResultSet rs) throws SQLException { Program program = new Program(); program.setUid(rs.getString(EventQuery.getColumnName(COLUMNS.PROGRAM_UID))); enrollment.setProgram(program); - final boolean followup = rs.getBoolean(EventQuery.getColumnName(COLUMNS.ENROLLMENT_FOLLOWUP)); - enrollment.setFollowup(rs.wasNull() ? null : followup); + final boolean followUp = rs.getBoolean(EventQuery.getColumnName(COLUMNS.ENROLLMENT_FOLLOWUP)); + enrollment.setFollowup(rs.wasNull() ? null : followUp); enrollment.setStatus( ProgramStatus.valueOf(rs.getString(EventQuery.getColumnName(COLUMNS.ENROLLMENT_STATUS)))); TrackedEntity trackedEntity = new TrackedEntity(); diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/mapper/RelationshipRowCallbackHandler.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/mapper/RelationshipRowCallbackHandler.java index 5724d6087c7d..a5b22ba6af21 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/mapper/RelationshipRowCallbackHandler.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/mapper/RelationshipRowCallbackHandler.java @@ -82,6 +82,7 @@ private Relationship getRelationship(ResultSet rs) throws SQLException { relationship.setRelationshipType(type); relationship.setCreated(rs.getTimestamp("created")); relationship.setLastUpdated(rs.getTimestamp("lastupdated")); + relationship.setCreatedAtClient(rs.getTimestamp("createdatclient")); RelationshipItem from = createItem(rs.getString("from_uid")); from.setRelationship(relationship); diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/query/EventQuery.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/query/EventQuery.java index ce0fb462f9ee..d1b2daf90705 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/query/EventQuery.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/aggregates/query/EventQuery.java @@ -71,7 +71,7 @@ public enum COLUMNS { new Subselect( "( " + "SELECT string_agg(opt.uid::text, ';') " - + "FROM dataelementcategoryoption opt " + + "FROM categoryoption opt " + "join categoryoptioncombos_categoryoptions ccc " + "on opt.categoryoptionid = ccc.categoryoptionid " + "WHERE coc.categoryoptioncomboid = ccc.categoryoptioncomboid )", diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/converter/EventTrackerConverterService.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/converter/EventTrackerConverterService.java index d229fdbe288a..c1f4f26556c6 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/converter/EventTrackerConverterService.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/converter/EventTrackerConverterService.java @@ -37,7 +37,6 @@ import java.util.Set; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; -import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.category.CategoryOption; import org.hisp.dhis.category.CategoryOptionCombo; @@ -89,7 +88,6 @@ public List to(List events) { new org.hisp.dhis.tracker.imports.domain.Event(); e.setEvent(event.getUid()); - e.setFollowup(BooleanUtils.toBoolean(event.getEnrollment().getFollowup())); e.setStatus(event.getStatus()); e.setOccurredAt(DateUtils.instantFromDate(event.getExecutionDate())); e.setScheduledAt(DateUtils.instantFromDate(event.getDueDate())); diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/converter/RelationshipTrackerConverterService.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/converter/RelationshipTrackerConverterService.java index 84e1d1f30c66..e68fd2ac6c00 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/converter/RelationshipTrackerConverterService.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/converter/RelationshipTrackerConverterService.java @@ -120,16 +120,17 @@ private org.hisp.dhis.relationship.Relationship from( org.hisp.dhis.relationship.RelationshipItem toItem = new org.hisp.dhis.relationship.RelationshipItem(); + Date now = new Date(); if (toRelationship == null) { - Date now = new Date(); toRelationship = new org.hisp.dhis.relationship.Relationship(); toRelationship.setUid(fromRelationship.getRelationship()); toRelationship.setCreated(now); - toRelationship.setLastUpdated(now); } + toRelationship.setLastUpdated(now); toRelationship.setRelationshipType(relationshipType); + toRelationship.setCreatedAtClient(DateUtils.fromInstant(fromRelationship.getCreatedAtClient())); if (fromRelationship.getRelationship() != null) { toRelationship.setUid(fromRelationship.getRelationship()); diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/domain/Event.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/domain/Event.java index 17bdb105a399..4b30f68fb4c1 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/domain/Event.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/domain/Event.java @@ -70,8 +70,6 @@ public class Event implements TrackerDto { @JsonProperty private String storedBy; - @JsonProperty private boolean followup; - @JsonProperty private boolean deleted; @JsonProperty private Instant createdAt; diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/domain/Relationship.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/domain/Relationship.java index 792c24b72d58..5219571ab623 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/domain/Relationship.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/domain/Relationship.java @@ -51,6 +51,8 @@ public class Relationship implements TrackerDto { @JsonProperty private Instant createdAt; + @JsonProperty private Instant createdAtClient; + @JsonProperty private Instant updatedAt; @JsonProperty private boolean bidirectional; diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/preheat/mappers/RelationshipMapper.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/preheat/mappers/RelationshipMapper.java index e7ebe801660c..9f9b845680f7 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/preheat/mappers/RelationshipMapper.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/preheat/mappers/RelationshipMapper.java @@ -52,6 +52,7 @@ public interface RelationshipMapper extends PreheatMapper { @Mapping(target = "createdBy") @Mapping(target = "lastUpdated") @Mapping(target = "lastUpdatedBy") + @Mapping(target = "createdAtClient") @Mapping(target = "deleted") Relationship map(Relationship relationship); diff --git a/dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.41/V2_41_29__Multi_program_eventvisualization.sql b/dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.41/V2_41_29__Multi_program_eventvisualization.sql new file mode 100644 index 000000000000..2360124541ed --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.41/V2_41_29__Multi_program_eventvisualization.sql @@ -0,0 +1,11 @@ +-- Migration related to https://dhis2.atlassian.net/browse/DHIS2-15725 + +-- This can be null. Now, in the case of multi-program, the program is set in the base object itself. +alter table if exists eventvisualization alter column programid drop not null; + +-- Add a new column for the tracked entity type associates with the EventVisualization. +alter table eventvisualization add column if not exists trackedentitytypeid int8; + +-- Adds a new FK to the trackedentitytype table. We drop and create it, so we can ensure idempotency. +alter table eventvisualization drop constraint if exists fk_evisualization_trackedentitytypeid; +alter table eventvisualization add constraint fk_evisualization_trackedentitytypeid foreign key (trackedentitytypeid) references trackedentitytype(trackedentitytypeid); diff --git a/dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.41/V2_41_30__Rename_DataElementCategoryOption_to_CategoryOption.sql b/dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.41/V2_41_30__Rename_DataElementCategoryOption_to_CategoryOption.sql new file mode 100644 index 000000000000..b3115a0da861 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.41/V2_41_30__Rename_DataElementCategoryOption_to_CategoryOption.sql @@ -0,0 +1,5 @@ +-- rename dataelementcategoryoption to categoryoption +alter table if exists dataelementcategoryoption rename to categoryoption; + +-- rename dataelementcategory to category +alter table if exists dataelementcategory rename to category; \ No newline at end of file diff --git a/dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.41/V2_41_31__Add_createdatclient_to_relationship.sql b/dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.41/V2_41_31__Add_createdatclient_to_relationship.sql new file mode 100644 index 000000000000..da50863ca511 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.41/V2_41_31__Add_createdatclient_to_relationship.sql @@ -0,0 +1 @@ +alter table relationship add column if not exists "createdatclient" timestamp; diff --git a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/ExpressionItem.java b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/ExpressionItem.java index eafa4acc0468..77468ebe755c 100644 --- a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/ExpressionItem.java +++ b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/ExpressionItem.java @@ -122,9 +122,9 @@ default Object evaluate(ExprContext ctx, AntlrExpressionVisitor visitor) { } /** - * Finds the value of an expression function, evaluating all the arguments of logical functions - * that might not always evaluate all arguments based on the truth value of some arguments (e.g. - * if, and, or, firstNonNull). + * Finds the value of an expression item, evaluating all the arguments of logical functions that + * might not always evaluate all arguments based on the truth value of some arguments (e.g. if, + * and, or, firstNonNull). * *

For those few logical functions that may not normally evaluate all arguments, this method * must be overridden. @@ -140,7 +140,7 @@ default Object evaluateAllPaths(ExprContext ctx, CommonExpressionVisitor visitor } /** - * Generates the SQL for a program indicator expression item. + * Generates the SQL for an expression item. * *

This method must be overridden for all items used in program indicator expressions, * otherwise an exception will be thrown. @@ -149,7 +149,7 @@ default Object evaluateAllPaths(ExprContext ctx, CommonExpressionVisitor visitor * * @param ctx the expression context * @param visitor the tree visitor - * @return the generated SQL (as a String) for the function + * @return the generated SQL (as a String) for the item */ default Object getSql(ExprContext ctx, CommonExpressionVisitor visitor) { throw new ParserExceptionWithoutContext("Not valid in this context: " + ctx.getText()); diff --git a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/ExpressionItemWithSql.java b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/ExpressionItemWithSql.java new file mode 100644 index 000000000000..9a788e23b425 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/ExpressionItemWithSql.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.parser.expression; + +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.ExprContext; + +/** + * A parsed item from an ANTLR expression, using the evaluate method to generate SQL (or to pass + * through the call to generate SQL.) + * + * @author Jim Grace + */ +public interface ExpressionItemWithSql extends ExpressionItem { + + /** + * Generates the SQL for an expression item using the evaluate method. + * + * @param ctx the expression context + * @param visitor the tree visitor + * @return the generated SQL (as a String) for the item + */ + @Override + default Object getSql(ExprContext ctx, CommonExpressionVisitor visitor) { + return evaluate(ctx, visitor); + } +} diff --git a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/ParserUtils.java b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/ParserUtils.java index b51cd3448f14..1d3f393a004f 100644 --- a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/ParserUtils.java +++ b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/ParserUtils.java @@ -63,6 +63,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Date; +import org.hisp.dhis.analytics.DataType; import org.hisp.dhis.antlr.ParserExceptionWithoutContext; import org.hisp.dhis.parser.expression.dataitem.ItemConstant; import org.hisp.dhis.parser.expression.function.FunctionFirstNonNull; @@ -99,7 +100,9 @@ * @author Jim Grace */ public class ParserUtils { - private ParserUtils() {} + private ParserUtils() { + throw new UnsupportedOperationException("util"); + } public static final double DOUBLE_VALUE_IF_NULL = 0.0; @@ -238,4 +241,37 @@ public static void assumeProgramExpressionProgramAttribute(ExprContext ctx) { "Program attribute must have one UID: " + ctx.getText()); } } + + /** + * Generate SQL to replace a null value with a dataType-appropriate default. + * + * @param column SQL column to default if null + * @param dataType data type for default value + * @return a coalesce statement to default the value + */ + public static String replaceSqlNull(String column, DataType dataType) { + return "coalesce(" + + column + + switch (dataType) { + case NUMERIC -> ",0)"; + case BOOLEAN -> ",false)"; + case TEXT -> ",'')"; + }; + } + + /** + * Generate SQL to cast an analytics value to a data type. + * + * @param column SQL column to cast + * @param dataType data type to cast to + * @return a coalesce statement casting the column + */ + public static String castSql(String column, DataType dataType) { + return column + + switch (dataType) { + case NUMERIC -> "::numeric"; + case BOOLEAN -> "::numeric!=0"; + case TEXT -> "::text"; + }; + } } diff --git a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/function/FunctionAggregationType.java b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/function/FunctionAggregationType.java index 1e043bde5692..9a7172a3d070 100644 --- a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/function/FunctionAggregationType.java +++ b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/function/FunctionAggregationType.java @@ -33,7 +33,7 @@ import org.hisp.dhis.antlr.ParserExceptionWithoutContext; import org.hisp.dhis.common.QueryModifiers; import org.hisp.dhis.parser.expression.CommonExpressionVisitor; -import org.hisp.dhis.parser.expression.ExpressionItem; +import org.hisp.dhis.parser.expression.ExpressionItemWithSql; /** * Function aggregationType (for indicator expressions) @@ -42,23 +42,9 @@ * * @author Jim Grace */ -public class FunctionAggregationType implements ExpressionItem { +public class FunctionAggregationType implements ExpressionItemWithSql { @Override public Object evaluate(ExprContext ctx, CommonExpressionVisitor visitor) { - return visitWithAggregationType(ctx, visitor); - } - - @Override - public Object getSql(ExprContext ctx, CommonExpressionVisitor visitor) { - return visitWithAggregationType(ctx, visitor); - } - - // ------------------------------------------------------------------------- - // Supportive methods - // ------------------------------------------------------------------------- - - /** Visits the expression fragment with aggregation type applied */ - private Object visitWithAggregationType(ExprContext ctx, CommonExpressionVisitor visitor) { AggregationType aggregationType = parseAggregationType(ctx.aggregationType.getText()); QueryModifiers queryMods = @@ -67,6 +53,10 @@ private Object visitWithAggregationType(ExprContext ctx, CommonExpressionVisitor return visitor.visitWithQueryMods(ctx.expr(0), queryMods); } + // ------------------------------------------------------------------------- + // Supportive methods + // ------------------------------------------------------------------------- + /** Parses the aggregation type */ private AggregationType parseAggregationType(String text) { try { diff --git a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/function/PeriodOffset.java b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/function/PeriodOffset.java index c46f059aef5a..f4acc0459fbe 100644 --- a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/function/PeriodOffset.java +++ b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/function/PeriodOffset.java @@ -33,7 +33,7 @@ import org.hisp.dhis.common.QueryModifiers; import org.hisp.dhis.parser.expression.CommonExpressionVisitor; -import org.hisp.dhis.parser.expression.ExpressionItem; +import org.hisp.dhis.parser.expression.ExpressionItemWithSql; import org.hisp.dhis.parser.expression.ExpressionState; /** @@ -41,7 +41,7 @@ * * @author Enrico Colasante */ -public class PeriodOffset implements ExpressionItem { +public class PeriodOffset implements ExpressionItemWithSql { @Override public Object evaluate(ExprContext ctx, CommonExpressionVisitor visitor) { ExpressionState state = visitor.getState(); diff --git a/dhis-2/dhis-support/dhis-support-external/src/main/java/org/hisp/dhis/external/conf/ConfigurationKey.java b/dhis-2/dhis-support/dhis-support-external/src/main/java/org/hisp/dhis/external/conf/ConfigurationKey.java index 0f8541a7f297..63e8e3360825 100644 --- a/dhis-2/dhis-support/dhis-support-external/src/main/java/org/hisp/dhis/external/conf/ConfigurationKey.java +++ b/dhis-2/dhis-support/dhis-support-external/src/main/java/org/hisp/dhis/external/conf/ConfigurationKey.java @@ -31,11 +31,13 @@ import java.util.Arrays; import java.util.Optional; +import lombok.Getter; import org.hisp.dhis.security.utils.CspConstants; /** * @author Lars Helge Overland */ +@Getter public enum ConfigurationKey { /** System mode for database read operations only, can be 'off', 'on'. (default: off). */ SYSTEM_READ_ONLY_MODE("system.read_only_mode", Constants.OFF, false), @@ -82,15 +84,28 @@ public enum ConfigurationKey { /** JDBC driver class. */ CONNECTION_DRIVER_CLASS("connection.driver_class", "org.postgresql.Driver", false), + /** Analytics JDBC driver class. */ + ANALYTICS_CONNECTION_DRIVER_CLASS( + "analytics.connection.driver_class", "org.postgresql.Driver", false), + /** Database connection URL. */ CONNECTION_URL("connection.url", "", false), + /** Analytics Database connection URL. */ + ANALYTICS_CONNECTION_URL("analytics.connection.url", "", false), + /** Database username. */ CONNECTION_USERNAME("connection.username", "", false), + /** Analytics Database username. */ + ANALYTICS_CONNECTION_USERNAME("analytics.connection.username", "", false), + /** Database password (sensitive). */ CONNECTION_PASSWORD("connection.password", "", true), + /** Analytics Database password (sensitive). */ + ANALYTICS_CONNECTION_PASSWORD("analytics.connection.password", "", true), + /** Sets 'hibernate.cache.use_second_level_cache'. (default: true) */ USE_SECOND_LEVEL_CACHE("hibernate.cache.use_second_level_cache", "true", false), @@ -107,57 +122,118 @@ public enum ConfigurationKey { /** Max size of connection pool (default: 80). */ CONNECTION_POOL_MAX_SIZE("connection.pool.max_size", "80", false), + /** Analytics Max size of connection pool (default: 80). */ + ANALYTICS_CONNECTION_POOL_MAX_SIZE("analytics.connection.pool.max_size", "80", false), + /** Minimum number of Connections a pool will maintain at any given time (default: 5). */ CONNECTION_POOL_MIN_SIZE("connection.pool.min_size", "5", false), + /** + * Analytics Minimum number of Connections a pool will maintain at any given time (default: 5). + */ + ANALYTICS_CONNECTION_POOL_MIN_SIZE("analytics.connection.pool.min_size", "5", false), + /** * Number of Connections a pool will try to acquire upon startup. Should be between minPoolSize * and maxPoolSize. (default: 5). */ CONNECTION_POOL_INITIAL_SIZE("connection.pool.initial_size", "5", false), + /** + * Number of Connections a pool will try to acquire upon startup. Should be between minPoolSize + * and maxPoolSize. (default: 5). + */ + ANALYTICS_CONNECTION_POOL_INITIAL_SIZE("analytics.connection.pool.initial_size", "5", false), + /** * Determines how many connections at a time will try to acquire when the pool is exhausted. * (default: 5). */ CONNECTION_POOL_ACQUIRE_INCR("connection.pool.acquire_incr", "5", false), + /** + * Determines how many connections at a time will try to acquire when the pool is exhausted. + * (default: 5). + */ + ANALYTICS_CONNECTION_POOL_ACQUIRE_INCR("analytics.connection.pool.acquire_incr", "5", false), + /** * Seconds a Connection can remain pooled but unused before being discarded. Zero means idle * connections never expire (default: 7200). */ CONNECTION_POOL_MAX_IDLE_TIME("connection.pool.max_idle_time", "7200", false), + /** + * Seconds a Connection can remain pooled but unused before being discarded. Zero means idle + * connections never expire (default: 7200). + */ + ANALYTICS_CONNECTION_POOL_MAX_IDLE_TIME("analytics.connection.pool.max_idle_time", "7200", false), + /** * Number of seconds that Connections in excess of minPoolSize should be permitted to remain idle * in the pool before being culled (default: 0). */ CONNECTION_POOL_MAX_IDLE_TIME_EXCESS_CON("connection.pool.max_idle_time_excess_con", "0", false), + /** + * Number of seconds that Connections in excess of minPoolSize should be permitted to remain idle + * in the pool before being culled (default: 0). + */ + ANALYTICS_CONNECTION_POOL_MAX_IDLE_TIME_EXCESS_CON( + "analytics.connection.pool.max_idle_time_excess_con", "0", false), + /** * If this is a number greater than 0, dhis2 will test all idle, pooled but unchecked-out * connections, every this number of seconds (default: 0). */ CONNECTION_POOL_IDLE_CON_TEST_PERIOD("connection.pool.idle.con.test.period", "0", false), + /** + * If this is a number greater than 0, dhis2 will test all idle, pooled but unchecked-out + * connections, every this number of seconds (default: 0). + */ + ANALYTICS_CONNECTION_POOL_IDLE_CON_TEST_PERIOD( + "analytics.connection.pool.idle.con.test.period", "0", false), + /** * If true, an operation will be performed at every connection checkout to verify that the * connection is valid (default: false). */ CONNECTION_POOL_TEST_ON_CHECKOUT("connection.pool.test.on.checkout", Constants.OFF, false), + /** + * If true, an operation will be performed at every connection checkout to verify that the + * connection is valid (default: false). + */ + ANALYTICS_CONNECTION_POOL_TEST_ON_CHECKOUT( + "analytics.connection.pool.test.on.checkout", Constants.OFF, false), + /** * If true, an operation will be performed asynchronously at every connection checkin to verify * that the connection is valid (default: true). */ CONNECTION_POOL_TEST_ON_CHECKIN("connection.pool.test.on.checkin", Constants.ON, false), + /** + * If true, an operation will be performed asynchronously at every connection checkin to verify + * that the connection is valid (default: true). + */ + ANALYTICS_CONNECTION_POOL_TEST_ON_CHECKIN( + "analytics.connection.pool.test.on.checkin", Constants.ON, false), + /** * Hikari DB pool feature. Connection pool timeout: Set the maximum number of milliseconds that a * client will wait for a connection from the pool. (default: 30s) */ CONNECTION_POOL_TIMEOUT("connection.pool.timeout", String.valueOf(SECONDS.toMillis(30)), false), + /** + * Analytics Hikari DB pool feature. Connection pool timeout: Set the maximum number of + * milliseconds that a client will wait for a connection from the pool. (default: 30s) + */ + ANALYTICS_CONNECTION_POOL_TIMEOUT( + "analytics.connection.pool.timeout", String.valueOf(SECONDS.toMillis(30)), false), + /** * Sets the maximum number of milliseconds that the Hikari pool will wait for a connection to be * validated as alive. (default: 5ms) @@ -165,9 +241,22 @@ public enum ConfigurationKey { CONNECTION_POOL_VALIDATION_TIMEOUT( "connection.pool.validation_timeout", String.valueOf(SECONDS.toMillis(5)), false), + /** + * Sets the maximum number of milliseconds that the Analytics Hikari pool will wait for a + * connection to be validated as alive. (default: 5ms) + */ + ANALYTICS_CONNECTION_POOL_VALIDATION_TIMEOUT( + "analytics.connection.pool.validation_timeout", String.valueOf(SECONDS.toMillis(5)), false), + /** Configure the number of helper threads used by C3P0 pool for jdbc operations (default: 3). */ CONNECTION_POOL_NUM_THREADS("connection.pool.num.helper.threads", "3", false), + /** + * Configure the number of helper threads used by Analytics C3P0 pool for jdbc operations + * (default: 3). + */ + ANALYTICS_CONNECTION_POOL_NUM_THREADS("analytics.connection.pool.num.helper.threads", "3", false), + /** * Defines the query that will be executed for all connection tests. Ideally this config is not * needed as postgresql driver already provides an efficient test query. The config is exposed @@ -175,6 +264,9 @@ public enum ConfigurationKey { */ CONNECTION_POOL_TEST_QUERY("connection.pool.preferred.test.query"), + /** Defines the query that will be executed for all Analytics connection tests. */ + ANALYTICS_CONNECTION_POOL_TEST_QUERY("analytics.connection.pool.preferred.test.query"), + /** LDAP server URL. (default: ldaps://0:1) */ LDAP_URL("ldap.url", "ldaps://0:1", false), @@ -552,22 +644,6 @@ public enum ConfigurationKey { this.aliases = aliases; } - public String getKey() { - return key; - } - - public String getDefaultValue() { - return defaultValue; - } - - public boolean isConfidential() { - return confidential; - } - - public String[] getAliases() { - return aliases; - } - public static Optional getByKey(String key) { return Arrays.stream(ConfigurationKey.values()).filter(k -> k.key.equals(key)).findFirst(); } diff --git a/dhis-2/dhis-support/dhis-support-external/src/main/java/org/hisp/dhis/external/conf/GoogleAccessToken.java b/dhis-2/dhis-support/dhis-support-external/src/main/java/org/hisp/dhis/external/conf/GoogleAccessToken.java index a7bddc6e24a3..02ebbd680257 100644 --- a/dhis-2/dhis-support/dhis-support-external/src/main/java/org/hisp/dhis/external/conf/GoogleAccessToken.java +++ b/dhis-2/dhis-support/dhis-support-external/src/main/java/org/hisp/dhis/external/conf/GoogleAccessToken.java @@ -30,54 +30,25 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import java.time.LocalDateTime; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; /** * @author Lars Helge Overland */ +@Getter +@Setter +@NoArgsConstructor public class GoogleAccessToken { - private String accessToken; - - private String clientId; - - private long expiresInSeconds; - - private LocalDateTime expiresOn; - - public GoogleAccessToken() {} - @JsonProperty(value = "access_token") - public String getAccessToken() { - return accessToken; - } - - public void setAccessToken(String accessToken) { - this.accessToken = accessToken; - } + private String accessToken; @JsonProperty(value = "client_id") - public String getClientId() { - return clientId; - } - - public void setClientId(String clientId) { - this.clientId = clientId; - } + private String clientId; @JsonProperty(value = "expires_in") - public long getExpiresInSeconds() { - return expiresInSeconds; - } - - public void setExpiresInSeconds(long expiresInSeconds) { - this.expiresInSeconds = expiresInSeconds; - } - - @JsonIgnore - public LocalDateTime getExpiresOn() { - return expiresOn; - } + private long expiresInSeconds; - public void setExpiresOn(LocalDateTime expiresOn) { - this.expiresOn = expiresOn; - } + @JsonIgnore private LocalDateTime expiresOn; } diff --git a/dhis-2/dhis-support/dhis-support-hibernate/pom.xml b/dhis-2/dhis-support/dhis-support-hibernate/pom.xml index 63d0e5de3b22..f8367f1ea616 100644 --- a/dhis-2/dhis-support/dhis-support-hibernate/pom.xml +++ b/dhis-2/dhis-support/dhis-support-hibernate/pom.xml @@ -76,6 +76,7 @@ + org.hamcrest hamcrest @@ -86,13 +87,23 @@ junit-jupiter test - + + org.mockito + mockito-core + test + + + org.mockito + mockito-junit-jupiter + test + net.ttddyy datasource-proxy + org.springframework spring-core diff --git a/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/config/AnalyticsDataSourceConfig.java b/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/config/AnalyticsDataSourceConfig.java new file mode 100644 index 000000000000..d89586f1ef2d --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/config/AnalyticsDataSourceConfig.java @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.config; + +import static org.hisp.dhis.config.DataSourceConfig.createLoggingDataSource; +import static org.hisp.dhis.datasource.DatabasePoolUtils.ConfigKeyMapper.ANALYTICS; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_CONNECTION_URL; + +import com.google.common.base.MoreObjects; +import java.beans.PropertyVetoException; +import java.sql.SQLException; +import javax.sql.DataSource; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.hisp.dhis.commons.util.DebugUtils; +import org.hisp.dhis.datasource.DatabasePoolUtils; +import org.hisp.dhis.datasource.ReadOnlyDataSourceManager; +import org.hisp.dhis.external.conf.ConfigurationKey; +import org.hisp.dhis.external.conf.DhisConfigurationProvider; +import org.hisp.dhis.hibernate.HibernateConfigurationProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; + +@Configuration +@Slf4j +@RequiredArgsConstructor +public class AnalyticsDataSourceConfig { + + private final DhisConfigurationProvider dhisConfig; + + @Bean("analyticsDataSource") + @DependsOn("analyticsActualDataSource") + public DataSource jdbcDataSource( + @Qualifier("analyticsActualDataSource") DataSource actualDataSource) { + return createLoggingDataSource(dhisConfig, actualDataSource); + } + + @Bean("analyticsActualDataSource") + public DataSource jdbcActualDataSource( + @Qualifier("actualDataSource") DataSource actualDataSource, + HibernateConfigurationProvider hibernateConfigurationProvider) { + + String jdbcUrl = dhisConfig.getProperty(ANALYTICS_CONNECTION_URL); + + if (StringUtils.isNotBlank(jdbcUrl)) { + return createActualDataSourceFromAnalyticsConfiguration(); + } + // if no analytics connection url is specified, use the same datasource as the main database + log.info( + "No analytics connection url is specified (" + + ANALYTICS_CONNECTION_URL.getKey() + + "). Analytics won't have a dedicated datasource"); + return actualDataSource; + } + + private DataSource createActualDataSourceFromAnalyticsConfiguration() { + String jdbcUrl = dhisConfig.getProperty(ANALYTICS_CONNECTION_URL); + + String dbPoolType = dhisConfig.getProperty(ConfigurationKey.DB_POOL_TYPE); + + DatabasePoolUtils.PoolConfig poolConfig = + DatabasePoolUtils.PoolConfig.builder() + .dhisConfig(dhisConfig) + .mapper(ANALYTICS) + .dbPoolType(dbPoolType) + .build(); + + try { + return DatabasePoolUtils.createDbPool(poolConfig); + } catch (SQLException | PropertyVetoException e) { + String message = + String.format( + "Connection test failed for analytics database pool, " + "jdbcUrl: '%s'", jdbcUrl); + + log.error(message); + log.error(DebugUtils.getStackTrace(e)); + + throw new IllegalStateException(message, e); + } + } + + @Bean("analyticsNamedParameterJdbcTemplate") + @DependsOn("analyticsDataSource") + public NamedParameterJdbcTemplate namedParameterJdbcTemplate( + @Qualifier("analyticsDataSource") DataSource dataSource) { + return new NamedParameterJdbcTemplate(dataSource); + } + + @Bean("executionPlanJdbcTemplate") + @DependsOn("analyticsDataSource") + public JdbcTemplate executionPlanJdbcTemplate( + @Qualifier("analyticsDataSource") DataSource dataSource) { + JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource); + jdbcTemplate.setFetchSize(1000); + jdbcTemplate.setQueryTimeout(10); + return jdbcTemplate; + } + + @Bean("analyticsReadOnlyJdbcTemplate") + @DependsOn("analyticsDataSource") + public JdbcTemplate readOnlyJdbcTemplate( + @Qualifier("analyticsDataSource") DataSource dataSource) { + ReadOnlyDataSourceManager manager = new ReadOnlyDataSourceManager(dhisConfig); + + JdbcTemplate jdbcTemplate = + new JdbcTemplate(MoreObjects.firstNonNull(manager.getReadOnlyDataSource(), dataSource)); + jdbcTemplate.setFetchSize(1000); + + return jdbcTemplate; + } + + @Bean("analyticsJdbcTemplate") + @DependsOn("analyticsDataSource") + public JdbcTemplate jdbcTemplate(@Qualifier("analyticsDataSource") DataSource dataSource) { + JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource); + jdbcTemplate.setFetchSize(1000); + return jdbcTemplate; + } +} diff --git a/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/config/DataSourceConfig.java b/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/config/DataSourceConfig.java index 137ee6203cad..0e63ba933fa7 100644 --- a/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/config/DataSourceConfig.java +++ b/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/config/DataSourceConfig.java @@ -33,6 +33,7 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; import javax.sql.DataSource; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import net.ttddyy.dsproxy.listener.MethodExecutionContext; import net.ttddyy.dsproxy.listener.logging.DefaultQueryLogEntryCreator; @@ -44,11 +45,10 @@ import org.hisp.dhis.common.CodeGenerator; import org.hisp.dhis.commons.util.DebugUtils; import org.hisp.dhis.datasource.DatabasePoolUtils; -import org.hisp.dhis.datasource.DefaultReadOnlyDataSourceManager; +import org.hisp.dhis.datasource.ReadOnlyDataSourceManager; import org.hisp.dhis.external.conf.ConfigurationKey; import org.hisp.dhis.external.conf.DhisConfigurationProvider; import org.hisp.dhis.hibernate.HibernateConfigurationProvider; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -62,10 +62,13 @@ */ @Slf4j @Configuration +@RequiredArgsConstructor public class DataSourceConfig { - @Autowired private DhisConfigurationProvider dhisConfig; - @Bean + private final DhisConfigurationProvider dhisConfig; + + @Bean("namedParameterJdbcTemplate") + @Primary @DependsOn("dataSource") public NamedParameterJdbcTemplate namedParameterJdbcTemplate( @Qualifier("dataSource") DataSource dataSource) { @@ -81,19 +84,10 @@ public JdbcTemplate jdbcTemplate(@Qualifier("dataSource") DataSource dataSource) return jdbcTemplate; } - @Bean("executionPlanJdbcTemplate") - @DependsOn("dataSource") - public JdbcTemplate executionPlanJdbcTemplate(@Qualifier("dataSource") DataSource dataSource) { - JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource); - jdbcTemplate.setFetchSize(1000); - jdbcTemplate.setQueryTimeout(10); - return jdbcTemplate; - } - @Bean("readOnlyJdbcTemplate") @DependsOn("dataSource") public JdbcTemplate readOnlyJdbcTemplate(@Qualifier("dataSource") DataSource dataSource) { - DefaultReadOnlyDataSourceManager manager = new DefaultReadOnlyDataSourceManager(dhisConfig); + ReadOnlyDataSourceManager manager = new ReadOnlyDataSourceManager(dhisConfig); JdbcTemplate jdbcTemplate = new JdbcTemplate(MoreObjects.firstNonNull(manager.getReadOnlyDataSource(), dataSource)); @@ -102,8 +96,8 @@ public JdbcTemplate readOnlyJdbcTemplate(@Qualifier("dataSource") DataSource dat return jdbcTemplate; } - @Bean("actualDataSource") - public DataSource actualDataSource( + static DataSource createActualDataSource( + DhisConfigurationProvider dhisConfig, HibernateConfigurationProvider hibernateConfigurationProvider) { String jdbcUrl = dhisConfig.getProperty(ConfigurationKey.CONNECTION_URL); String username = dhisConfig.getProperty(ConfigurationKey.CONNECTION_USERNAME); @@ -131,10 +125,8 @@ public DataSource actualDataSource( } } - @Bean("dataSource") - @DependsOn("actualDataSource") - @Primary - public DataSource dataSource(@Qualifier("actualDataSource") DataSource actualDataSource) { + static DataSource createLoggingDataSource( + DhisConfigurationProvider dhisConfig, DataSource actualDataSource) { boolean enableQueryLogging = dhisConfig.isEnabled(ConfigurationKey.ENABLE_QUERY_LOGGING); if (!enableQueryLogging) { @@ -182,6 +174,19 @@ public DataSource dataSource(@Qualifier("actualDataSource") DataSource actualDat return builder.build(); } + @Bean("dataSource") + @DependsOn("actualDataSource") + @Primary + public DataSource dataSource(@Qualifier("actualDataSource") DataSource actualDataSource) { + return createLoggingDataSource(dhisConfig, actualDataSource); + } + + @Bean("actualDataSource") + public DataSource actualDataSource( + HibernateConfigurationProvider hibernateConfigurationProvider) { + return createActualDataSource(dhisConfig, hibernateConfigurationProvider); + } + private static void executeAfterMethod(MethodExecutionContext executionContext) { Thread thread = Thread.currentThread(); StackTraceElement[] stackTrace = thread.getStackTrace(); diff --git a/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/datasource/DatabasePoolUtils.java b/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/datasource/DatabasePoolUtils.java index 46f31e29e606..ffbcc6ef9d81 100644 --- a/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/datasource/DatabasePoolUtils.java +++ b/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/datasource/DatabasePoolUtils.java @@ -27,7 +27,45 @@ */ package org.hisp.dhis.datasource; -import com.google.common.base.MoreObjects; +import static com.google.common.base.MoreObjects.firstNonNull; +import static java.lang.Integer.parseInt; +import static java.lang.Long.parseLong; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_CONNECTION_DRIVER_CLASS; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_CONNECTION_PASSWORD; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_CONNECTION_POOL_ACQUIRE_INCR; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_CONNECTION_POOL_IDLE_CON_TEST_PERIOD; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_CONNECTION_POOL_INITIAL_SIZE; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_CONNECTION_POOL_MAX_IDLE_TIME; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_CONNECTION_POOL_MAX_IDLE_TIME_EXCESS_CON; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_CONNECTION_POOL_MAX_SIZE; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_CONNECTION_POOL_MIN_SIZE; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_CONNECTION_POOL_NUM_THREADS; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_CONNECTION_POOL_TEST_ON_CHECKIN; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_CONNECTION_POOL_TEST_ON_CHECKOUT; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_CONNECTION_POOL_TEST_QUERY; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_CONNECTION_POOL_TIMEOUT; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_CONNECTION_POOL_VALIDATION_TIMEOUT; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_CONNECTION_URL; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_CONNECTION_USERNAME; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_DRIVER_CLASS; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_PASSWORD; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_POOL_ACQUIRE_INCR; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_POOL_IDLE_CON_TEST_PERIOD; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_POOL_INITIAL_SIZE; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_POOL_MAX_IDLE_TIME; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_POOL_MAX_IDLE_TIME_EXCESS_CON; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_POOL_MAX_SIZE; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_POOL_MIN_SIZE; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_POOL_NUM_THREADS; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_POOL_TEST_ON_CHECKIN; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_POOL_TEST_ON_CHECKOUT; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_POOL_TEST_QUERY; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_POOL_TIMEOUT; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_POOL_VALIDATION_TIMEOUT; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_URL; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_USERNAME; + +import com.google.common.collect.ImmutableMap; import com.mchange.v2.c3p0.ComboPooledDataSource; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; @@ -35,10 +73,14 @@ import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; +import java.util.Collections; +import java.util.Map; import java.util.Objects; +import java.util.Optional; import javax.sql.DataSource; import lombok.Builder; import lombok.Data; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.common.CodeGenerator; import org.hisp.dhis.external.conf.ConfigurationKey; @@ -51,7 +93,49 @@ @Slf4j public class DatabasePoolUtils { - public enum dbPoolTypes { + /** + * This enums maps each database config key into a corresponding analytics config key. This is + * used to allow the analytics database to be configured separately from the main database. + */ + @RequiredArgsConstructor + public enum ConfigKeyMapper { + ANALYTICS( + ImmutableMap.builder() + /* common keys for all connection pools */ + .put(CONNECTION_URL, ANALYTICS_CONNECTION_URL) + .put(CONNECTION_USERNAME, ANALYTICS_CONNECTION_USERNAME) + .put(CONNECTION_PASSWORD, ANALYTICS_CONNECTION_PASSWORD) + .put(CONNECTION_DRIVER_CLASS, ANALYTICS_CONNECTION_DRIVER_CLASS) + .put(CONNECTION_POOL_MAX_SIZE, ANALYTICS_CONNECTION_POOL_MAX_SIZE) + .put(CONNECTION_POOL_TEST_QUERY, ANALYTICS_CONNECTION_POOL_TEST_QUERY) + /* hikari-specific */ + .put(CONNECTION_POOL_TIMEOUT, ANALYTICS_CONNECTION_POOL_TIMEOUT) + .put(CONNECTION_POOL_VALIDATION_TIMEOUT, ANALYTICS_CONNECTION_POOL_VALIDATION_TIMEOUT) + /* C3P0-specific */ + .put(CONNECTION_POOL_ACQUIRE_INCR, ANALYTICS_CONNECTION_POOL_ACQUIRE_INCR) + .put(CONNECTION_POOL_MAX_IDLE_TIME, ANALYTICS_CONNECTION_POOL_MAX_IDLE_TIME) + .put(CONNECTION_POOL_MIN_SIZE, ANALYTICS_CONNECTION_POOL_MIN_SIZE) + .put(CONNECTION_POOL_INITIAL_SIZE, ANALYTICS_CONNECTION_POOL_INITIAL_SIZE) + .put(CONNECTION_POOL_TEST_ON_CHECKIN, ANALYTICS_CONNECTION_POOL_TEST_ON_CHECKIN) + .put(CONNECTION_POOL_TEST_ON_CHECKOUT, ANALYTICS_CONNECTION_POOL_TEST_ON_CHECKOUT) + .put( + CONNECTION_POOL_MAX_IDLE_TIME_EXCESS_CON, + ANALYTICS_CONNECTION_POOL_MAX_IDLE_TIME_EXCESS_CON) + .put( + CONNECTION_POOL_IDLE_CON_TEST_PERIOD, + ANALYTICS_CONNECTION_POOL_IDLE_CON_TEST_PERIOD) + .put(CONNECTION_POOL_NUM_THREADS, ANALYTICS_CONNECTION_POOL_NUM_THREADS) + .build()), + POSTGRESQL(Collections.emptyMap()); + + private final Map keyMap; + + public ConfigurationKey getConfigKey(ConfigurationKey key) { + return keyMap.getOrDefault(key, key); + } + } + + public enum DbPoolType { C3P0, HIKARI } @@ -76,17 +160,23 @@ public static class PoolConfig { private String acquireIncrement; private String maxIdleTime; + + private ConfigKeyMapper mapper; + + public ConfigKeyMapper getMapper() { + return Optional.ofNullable(mapper).orElse(ConfigKeyMapper.POSTGRESQL); + } } public static DataSource createDbPool(PoolConfig config) throws PropertyVetoException, SQLException { Objects.requireNonNull(config); - dbPoolTypes dbType = dbPoolTypes.valueOf(config.dbPoolType.toUpperCase()); + DbPoolType dbType = DbPoolType.valueOf(config.dbPoolType.toUpperCase()); - if (dbType == dbPoolTypes.C3P0) { + if (dbType == DbPoolType.C3P0) { return createC3p0DbPool(config); - } else if (dbType == dbPoolTypes.HIKARI) { + } else if (dbType == DbPoolType.HIKARI) { return createHikariDbPool(config); } @@ -99,30 +189,39 @@ public static DataSource createDbPool(PoolConfig config) throw new IllegalArgumentException(msg); } - public static DataSource createHikariDbPool(PoolConfig config) throws SQLException { + private static DataSource createHikariDbPool(PoolConfig config) throws SQLException { + ConfigKeyMapper mapper = config.getMapper(); + DhisConfigurationProvider dhisConfig = config.getDhisConfig(); - final String driverClassName = dhisConfig.getProperty(ConfigurationKey.CONNECTION_DRIVER_CLASS); + final String driverClassName = + dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_DRIVER_CLASS)); + final String jdbcUrl = - MoreObjects.firstNonNull( - config.getJdbcUrl(), dhisConfig.getProperty(ConfigurationKey.CONNECTION_URL)); + firstNonNull( + config.getJdbcUrl(), dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_URL))); + final String username = - MoreObjects.firstNonNull( - config.getUsername(), dhisConfig.getProperty(ConfigurationKey.CONNECTION_USERNAME)); + firstNonNull( + config.getUsername(), dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_USERNAME))); + final String password = - MoreObjects.firstNonNull( - config.getPassword(), dhisConfig.getProperty(ConfigurationKey.CONNECTION_PASSWORD)); + firstNonNull( + config.getPassword(), dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_PASSWORD))); + final long connectionTimeout = - Long.parseLong(dhisConfig.getProperty(ConfigurationKey.CONNECTION_POOL_TIMEOUT)); + parseLong(dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_POOL_TIMEOUT))); final long validationTimeout = - Long.parseLong(dhisConfig.getProperty(ConfigurationKey.CONNECTION_POOL_VALIDATION_TIMEOUT)); + parseLong(dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_POOL_VALIDATION_TIMEOUT))); + final int maxPoolSize = Integer.parseInt( - MoreObjects.firstNonNull( + firstNonNull( config.getMaxPoolSize(), - dhisConfig.getProperty(ConfigurationKey.CONNECTION_POOL_MAX_SIZE))); + dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_POOL_MAX_SIZE)))); + final String connectionTestQuery = - dhisConfig.getProperty(ConfigurationKey.CONNECTION_POOL_TEST_QUERY); + dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_POOL_TEST_QUERY)); HikariConfig hc = new HikariConfig(); hc.setPoolName("HikariDataSource_" + CodeGenerator.generateCode(10)); @@ -145,53 +244,56 @@ public static DataSource createHikariDbPool(PoolConfig config) throws SQLExcepti return ds; } - public static DataSource createC3p0DbPool(PoolConfig config) + private static DataSource createC3p0DbPool(PoolConfig config) throws PropertyVetoException, SQLException { + ConfigKeyMapper mapper = config.getMapper(); + DhisConfigurationProvider dhisConfig = config.getDhisConfig(); - final String driverClassName = dhisConfig.getProperty(ConfigurationKey.CONNECTION_DRIVER_CLASS); + final String driverClassName = + dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_DRIVER_CLASS)); final String jdbcUrl = - MoreObjects.firstNonNull( - config.getJdbcUrl(), dhisConfig.getProperty(ConfigurationKey.CONNECTION_URL)); + firstNonNull( + config.getJdbcUrl(), dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_URL))); final String username = - MoreObjects.firstNonNull( - config.getUsername(), dhisConfig.getProperty(ConfigurationKey.CONNECTION_USERNAME)); + firstNonNull( + config.getUsername(), dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_USERNAME))); final String password = - MoreObjects.firstNonNull( - config.getPassword(), dhisConfig.getProperty(ConfigurationKey.CONNECTION_PASSWORD)); + firstNonNull( + config.getPassword(), dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_PASSWORD))); final int maxPoolSize = - Integer.parseInt( - MoreObjects.firstNonNull( + parseInt( + firstNonNull( config.getMaxPoolSize(), - dhisConfig.getProperty(ConfigurationKey.CONNECTION_POOL_MAX_SIZE))); + dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_POOL_MAX_SIZE)))); final int acquireIncrement = - Integer.parseInt( - MoreObjects.firstNonNull( + parseInt( + firstNonNull( config.getAcquireIncrement(), - dhisConfig.getProperty(ConfigurationKey.CONNECTION_POOL_ACQUIRE_INCR))); + dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_POOL_ACQUIRE_INCR)))); final int maxIdleTime = - Integer.parseInt( - MoreObjects.firstNonNull( + parseInt( + firstNonNull( config.maxIdleTime, - dhisConfig.getProperty(ConfigurationKey.CONNECTION_POOL_MAX_IDLE_TIME))); + dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_POOL_MAX_IDLE_TIME)))); final int minPoolSize = - Integer.parseInt(dhisConfig.getProperty(ConfigurationKey.CONNECTION_POOL_MIN_SIZE)); + parseInt(dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_POOL_MIN_SIZE))); final int initialSize = - Integer.parseInt(dhisConfig.getProperty(ConfigurationKey.CONNECTION_POOL_INITIAL_SIZE)); - boolean testOnCheckIn = dhisConfig.isEnabled(ConfigurationKey.CONNECTION_POOL_TEST_ON_CHECKIN); + parseInt(dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_POOL_INITIAL_SIZE))); + boolean testOnCheckIn = + dhisConfig.isEnabled(mapper.getConfigKey(CONNECTION_POOL_TEST_ON_CHECKIN)); boolean testOnCheckOut = - dhisConfig.isEnabled(ConfigurationKey.CONNECTION_POOL_TEST_ON_CHECKOUT); + dhisConfig.isEnabled(mapper.getConfigKey(CONNECTION_POOL_TEST_ON_CHECKOUT)); final int maxIdleTimeExcessConnections = - Integer.parseInt( - dhisConfig.getProperty(ConfigurationKey.CONNECTION_POOL_MAX_IDLE_TIME_EXCESS_CON)); + parseInt( + dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_POOL_MAX_IDLE_TIME_EXCESS_CON))); final int idleConnectionTestPeriod = - Integer.parseInt( - dhisConfig.getProperty(ConfigurationKey.CONNECTION_POOL_IDLE_CON_TEST_PERIOD)); + parseInt(dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_POOL_IDLE_CON_TEST_PERIOD))); final String preferredTestQuery = - dhisConfig.getProperty(ConfigurationKey.CONNECTION_POOL_TEST_QUERY); + dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_POOL_TEST_QUERY)); final int numHelperThreads = - Integer.parseInt(dhisConfig.getProperty(ConfigurationKey.CONNECTION_POOL_NUM_THREADS)); + parseInt(dhisConfig.getProperty(mapper.getConfigKey(CONNECTION_POOL_NUM_THREADS))); ComboPooledDataSource dataSource = new ComboPooledDataSource(); dataSource.setDriverClass(driverClassName); diff --git a/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/datasource/DefaultReadOnlyDataSourceManager.java b/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/datasource/DefaultReadOnlyDataSourceManager.java deleted file mode 100644 index 6d61b53283bc..000000000000 --- a/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/datasource/DefaultReadOnlyDataSourceManager.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright (c) 2004-2022, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.datasource; - -import static com.google.common.base.Preconditions.checkNotNull; -import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_PASSWORD; -import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_URL; -import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_USERNAME; - -import java.beans.PropertyVetoException; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; -import javax.sql.DataSource; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.hisp.dhis.commons.util.DebugUtils; -import org.hisp.dhis.external.conf.ConfigurationKey; -import org.hisp.dhis.external.conf.DhisConfigurationProvider; -import org.hisp.dhis.util.ObjectUtils; -import org.springframework.beans.factory.InitializingBean; - -/** - * @author Lars Helge Overland - */ -@Slf4j -public class DefaultReadOnlyDataSourceManager - implements ReadOnlyDataSourceManager, InitializingBean { - private static final String FORMAT_READ_PREFIX = "read%d."; - - private static final String FORMAT_CONNECTION_URL = FORMAT_READ_PREFIX + CONNECTION_URL.getKey(); - - private static final String FORMAT_CONNECTION_USERNAME = - FORMAT_READ_PREFIX + CONNECTION_USERNAME.getKey(); - - private static final String FORMAT_CONNECTION_PASSWORD = - FORMAT_READ_PREFIX + CONNECTION_PASSWORD.getKey(); - - private static final int VAL_ACQUIRE_INCREMENT = 6; - - private static final int VAL_MAX_IDLE_TIME = 21600; - - private static final int MAX_READ_REPLICAS = 5; - - private final DhisConfigurationProvider config; - - public DefaultReadOnlyDataSourceManager(DhisConfigurationProvider config) { - checkNotNull(config); - this.config = config; - } - - /** State holder for the resolved read only data source. */ - private DataSource internalReadOnlyDataSource; - - /** State holder for explicitly defined read only data sources. */ - private List internalReadOnlyInstanceList; - - @Override - public void afterPropertiesSet() { - List ds = getReadOnlyDataSources(); - - this.internalReadOnlyInstanceList = ds; - this.internalReadOnlyDataSource = !ds.isEmpty() ? new CircularRoutingDataSource(ds) : null; - } - - // ------------------------------------------------------------------------- - // DataSourceManager implementation - // ------------------------------------------------------------------------- - - @Override - public DataSource getReadOnlyDataSource() { - return internalReadOnlyDataSource; - } - - @Override - public int getReadReplicaCount() { - return internalReadOnlyInstanceList != null ? internalReadOnlyInstanceList.size() : 0; - } - - // ------------------------------------------------------------------------- - // Supportive methods - // ------------------------------------------------------------------------- - - private List getReadOnlyDataSources() { - String mainUser = config.getProperty(ConfigurationKey.CONNECTION_USERNAME); - String mainPassword = config.getProperty(ConfigurationKey.CONNECTION_PASSWORD); - String driverClass = config.getProperty(ConfigurationKey.CONNECTION_DRIVER_CLASS); - String maxPoolSize = config.getProperty(ConfigurationKey.CONNECTION_POOL_MAX_SIZE); - String dbPoolType = config.getProperty(ConfigurationKey.DB_POOL_TYPE); - - Properties props = config.getProperties(); - - List dataSources = new ArrayList<>(); - - for (int i = 1; i <= MAX_READ_REPLICAS; i++) { - String jdbcUrl = props.getProperty(String.format(FORMAT_CONNECTION_URL, i)); - String username = props.getProperty(String.format(FORMAT_CONNECTION_USERNAME, i)); - String password = props.getProperty(String.format(FORMAT_CONNECTION_PASSWORD, i)); - - username = StringUtils.defaultIfEmpty(username, mainUser); - password = StringUtils.defaultIfEmpty(password, mainPassword); - - DatabasePoolUtils.PoolConfig.PoolConfigBuilder builder = - DatabasePoolUtils.PoolConfig.builder(); - builder.dhisConfig(config); - builder.password(password); - builder.username(username); - builder.jdbcUrl(jdbcUrl); - builder.dbPoolType(dbPoolType); - builder.maxPoolSize(maxPoolSize); - builder.acquireIncrement(String.valueOf(VAL_ACQUIRE_INCREMENT)); - builder.maxIdleTime(String.valueOf(VAL_MAX_IDLE_TIME)); - - if (ObjectUtils.allNonNull(jdbcUrl, username, password)) { - try { - dataSources.add(DatabasePoolUtils.createDbPool(builder.build())); - } catch (SQLException | PropertyVetoException e) { - String message = - String.format( - "Connection test failed for read replica database pool, " - + "driver class: '%s', URL: '%s', user: '%s'", - driverClass, jdbcUrl, username); - - log.error(message); - log.error(DebugUtils.getStackTrace(e)); - - throw new IllegalStateException(message, e); - } - } - } - - log.info("Read only configuration initialized, read replicas found: " + dataSources.size()); - - config - .getProperties() - .setProperty( - ConfigurationKey.ACTIVE_READ_REPLICAS.getKey(), String.valueOf(dataSources.size())); - - return dataSources; - } -} diff --git a/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/datasource/ReadOnlyDataSourceManager.java b/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/datasource/ReadOnlyDataSourceManager.java index 2c85aef2a279..c4691bd76b41 100644 --- a/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/datasource/ReadOnlyDataSourceManager.java +++ b/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/datasource/ReadOnlyDataSourceManager.java @@ -27,26 +27,177 @@ */ package org.hisp.dhis.datasource; +import static com.google.common.base.Preconditions.checkNotNull; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_PASSWORD; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_URL; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_USERNAME; + +import java.beans.PropertyVetoException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; import javax.sql.DataSource; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.hisp.dhis.commons.util.DebugUtils; +import org.hisp.dhis.external.conf.ConfigurationKey; +import org.hisp.dhis.external.conf.DhisConfigurationProvider; +import org.hisp.dhis.hibernate.ReadOnlyDataSourceConfig; +import org.hisp.dhis.util.ObjectUtils; /** + * Class responsible for detecting read-only databases configured in the DHIS 2 configuration file. + * * @author Lars Helge Overland */ -public interface ReadOnlyDataSourceManager { - /** - * Returns a data source which should be used for read only queries only. If read only replicas - * have been explicitly defined in the configuration, the data source implementation will be - * routing to potentially multiple underlying data sources. If not, the data source will point to - * the main data source. - * - * @return a DataSource instance. - */ - DataSource getReadOnlyDataSource(); +@Slf4j +@NoArgsConstructor(access = AccessLevel.PUBLIC) +public class ReadOnlyDataSourceManager { + private static final String FORMAT_READ_PREFIX = "read%d."; + + private static final String FORMAT_CONNECTION_URL = FORMAT_READ_PREFIX + CONNECTION_URL.getKey(); + + private static final String FORMAT_CONNECTION_USERNAME = + FORMAT_READ_PREFIX + CONNECTION_USERNAME.getKey(); + + private static final String FORMAT_CONNECTION_PASSWORD = + FORMAT_READ_PREFIX + CONNECTION_PASSWORD.getKey(); + + private static final int VAL_ACQUIRE_INCREMENT = 6; + + private static final int VAL_MAX_IDLE_TIME = 21600; + + private static final int MAX_READ_REPLICAS = 5; + + public ReadOnlyDataSourceManager(DhisConfigurationProvider config) { + checkNotNull(config); + init(config); + } + + /** State holder for the resolved read only data source. */ + private DataSource internalReadOnlyDataSource; + + /** State holder for explicitly defined read only data sources. */ + private List internalReadOnlyInstanceList; + + // ------------------------------------------------------------------------- + // Public methods + // ------------------------------------------------------------------------- + + public void init(DhisConfigurationProvider config) { + List ds = getReadOnlyDataSources(config); + + this.internalReadOnlyInstanceList = ds; + this.internalReadOnlyDataSource = !ds.isEmpty() ? new CircularRoutingDataSource(ds) : null; + } + + public DataSource getReadOnlyDataSource() { + return internalReadOnlyDataSource; + } + + public int getReadReplicaCount() { + return internalReadOnlyInstanceList != null ? internalReadOnlyInstanceList.size() : 0; + } + + // ------------------------------------------------------------------------- + // Supportive methods + // ------------------------------------------------------------------------- + + private List getReadOnlyDataSources(DhisConfigurationProvider config) { + String mainUser = config.getProperty(ConfigurationKey.CONNECTION_USERNAME); + String mainPassword = config.getProperty(ConfigurationKey.CONNECTION_PASSWORD); + String driverClass = config.getProperty(ConfigurationKey.CONNECTION_DRIVER_CLASS); + String maxPoolSize = config.getProperty(ConfigurationKey.CONNECTION_POOL_MAX_SIZE); + String dbPoolType = config.getProperty(ConfigurationKey.DB_POOL_TYPE); + + List dataSources = new ArrayList<>(); + + List dataSourceConfigs = getReadOnlyDataSourceConfigs(config); + + for (ReadOnlyDataSourceConfig dataSourceConfig : dataSourceConfigs) { + String url = dataSourceConfig.getUrl(); + String username = StringUtils.defaultIfEmpty(dataSourceConfig.getUsername(), mainUser); + String password = StringUtils.defaultIfEmpty(dataSourceConfig.getPassword(), mainPassword); + + DatabasePoolUtils.PoolConfig.PoolConfigBuilder builder = + DatabasePoolUtils.PoolConfig.builder(); + builder.dhisConfig(config); + builder.password(password); + builder.username(username); + builder.jdbcUrl(url); + builder.dbPoolType(dbPoolType); + builder.maxPoolSize(maxPoolSize); + builder.acquireIncrement(String.valueOf(VAL_ACQUIRE_INCREMENT)); + builder.maxIdleTime(String.valueOf(VAL_MAX_IDLE_TIME)); + + try { + dataSources.add(DatabasePoolUtils.createDbPool(builder.build())); + log.info("Created read-only data source with connection URL: '{}'", url); + } catch (SQLException | PropertyVetoException e) { + String message = + String.format( + "Connection test failed for read replica database pool with " + + "driver class: '%s', URL: '%s', username: '%s'", + driverClass, url, username); + + log.error(message); + log.error(DebugUtils.getStackTrace(e)); + + throw new IllegalStateException(message, e); + } + } + + config + .getProperties() + .setProperty( + ConfigurationKey.ACTIVE_READ_REPLICAS.getKey(), String.valueOf(dataSources.size())); + + log.info("Read only configuration initialized, read replicas found: " + dataSources.size()); + + return dataSources; + } /** - * Returns the number of explicitly defined read only database instances. + * Returns a list of read-only data source configurations. The configurations are detected from + * the DHIS 2 configuration file. * - * @return the number of explicitly defined read only database instances. + * @param config the {@link DhisConfigurationProvider}. + * @return a list of {@link ReadOnlyDataSourceConfig}. */ - int getReadReplicaCount(); + List getReadOnlyDataSourceConfigs(DhisConfigurationProvider config) { + List dataSources = new ArrayList<>(); + + Properties props = config.getProperties(); + + String mainUser = config.getProperty(ConfigurationKey.CONNECTION_USERNAME); + String mainPassword = config.getProperty(ConfigurationKey.CONNECTION_PASSWORD); + + for (int i = 1; i <= MAX_READ_REPLICAS; i++) { + String connectionUrlKey = String.format(FORMAT_CONNECTION_URL, i); + String connectionUsernameKey = String.format(FORMAT_CONNECTION_USERNAME, i); + String connectionPasswordKey = String.format(FORMAT_CONNECTION_PASSWORD, i); + + log.debug("Searching read-only data source with connection URL key: '{}'", connectionUrlKey); + + String url = props.getProperty(connectionUrlKey); + String username = props.getProperty(connectionUsernameKey); + String password = props.getProperty(connectionPasswordKey); + + username = StringUtils.defaultIfEmpty(username, mainUser); + password = StringUtils.defaultIfEmpty(password, mainPassword); + + if (ObjectUtils.allNonNull(url, username, password)) { + dataSources.add(new ReadOnlyDataSourceConfig(url, username, password)); + log.info( + "Read-only data source found with connection URL key: '{}' and value: '{}'", + connectionUrlKey, + url); + } + } + + return dataSources; + } } diff --git a/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/dbms/HibernateDbmsManager.java b/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/dbms/HibernateDbmsManager.java index 1efd1d4ff6ae..e21535dc380d 100644 --- a/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/dbms/HibernateDbmsManager.java +++ b/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/dbms/HibernateDbmsManager.java @@ -303,8 +303,8 @@ public void emptyDatabase() { emptyTable("expressiondimensionitem"); emptyTable("categoryoptioncombo"); emptyTable("categorycombo"); - emptyTable("dataelementcategory"); - emptyTable("dataelementcategoryoption"); + emptyTable("category"); + emptyTable("categoryoption"); emptyTable("optionvalue"); emptyTable("optionset"); diff --git a/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/hibernate/ReadOnlyDataSourceConfig.java b/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/hibernate/ReadOnlyDataSourceConfig.java new file mode 100644 index 000000000000..028d1d24b3f8 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/hibernate/ReadOnlyDataSourceConfig.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.hibernate; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; + +/** + * Encapsulation of a read only data source configuration. + * + * @author Lars Helge Overland + */ +@Getter +@Setter +@RequiredArgsConstructor +public class ReadOnlyDataSourceConfig { + private final String url; + + private final String username; + + private final String password; +} diff --git a/dhis-2/dhis-support/dhis-support-hibernate/src/test/java/org/hisp/dhis/datasource/ReadOnlyDataSourceManagerTest.java b/dhis-2/dhis-support/dhis-support-hibernate/src/test/java/org/hisp/dhis/datasource/ReadOnlyDataSourceManagerTest.java new file mode 100644 index 000000000000..0b5ebcffda93 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-hibernate/src/test/java/org/hisp/dhis/datasource/ReadOnlyDataSourceManagerTest.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.datasource; + +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_PASSWORD; +import static org.hisp.dhis.external.conf.ConfigurationKey.CONNECTION_USERNAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Properties; +import org.hisp.dhis.external.conf.DhisConfigurationProvider; +import org.hisp.dhis.hibernate.ReadOnlyDataSourceConfig; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ReadOnlyDataSourceManagerTest { + @Mock private DhisConfigurationProvider config; + + @Test + void testGetReadOnlyDataSourceConfigs() { + Properties props = new Properties(); + + props.put("read1.connection.url", "jdbc:postgresql:dev_read_1"); + props.put("read1.connection.username", "dhis1"); + props.put("read1.connection.password", "pw1"); + + when(config.getProperties()).thenReturn(props); + + when(config.getProperty(CONNECTION_USERNAME)).thenReturn(CONNECTION_USERNAME.getDefaultValue()); + when(config.getProperty(CONNECTION_PASSWORD)).thenReturn(CONNECTION_PASSWORD.getDefaultValue()); + + ReadOnlyDataSourceManager manager = new ReadOnlyDataSourceManager(); + List dataSourceConfigs = manager.getReadOnlyDataSourceConfigs(config); + assertEquals(1, dataSourceConfigs.size()); + ReadOnlyDataSourceConfig dataSourceConfig = dataSourceConfigs.get(0); + assertEquals("jdbc:postgresql:dev_read_1", dataSourceConfig.getUrl()); + assertEquals("dhis1", dataSourceConfig.getUsername()); + assertEquals("pw1", dataSourceConfig.getPassword()); + } +} diff --git a/dhis-2/dhis-test-e2e/README.md b/dhis-2/dhis-test-e2e/README.md index 533a6d16764f..24295f9df793 100644 --- a/dhis-2/dhis-test-e2e/README.md +++ b/dhis-2/dhis-test-e2e/README.md @@ -162,8 +162,8 @@ We have the capability to auto-generate analytics e2e tests. The class located at `src/test/java/org/hisp/dhis/analytics/generator/Main.java` can be executed in order to generate e2e tests based on the URL(s) present in `src/test/java/org/hisp/dhis/analytics/generator/test-urls.txt` -There are a few different generators that can be used. The correct generator to use depends on the URL(s) to be tested. -The respective generator implementation should be set at `src/test/java/org/hisp/dhis/analytics/generator/TestGenerator.java`. +There are a few different generators available. The usage of the correct one depends on the URL/API to be tested. +Based on the URL/API, the respective generator implementation should be set at `src/test/java/org/hisp/dhis/analytics/generator/TestGenerator.java`. Currently, the supported generators are (along with their respective accepted URL format): ``` @@ -174,8 +174,7 @@ EventAggregatedTestGenerator.java -> /analytics/events/aggregate/{program}.json? EventQueryTestGenerator.java -> /analytics/events/query/{program}.json? TeiQueryTestGenerator.java -> /analytics/trackedEntities/query/{trackedEntityType}.json? ``` -_**NOTE**_: `.json` behind the program uid is mandatory - +_**NOTE**_: The `.json` extension in some URLs above. It's mandatory for all cases where we expect and `uid` of the respective entity/object. ### How to generate the test(s) 1. Add the URL(s) into `test-urls.txt` (check inside the file for examples) @@ -188,5 +187,5 @@ that is up and running. The tests are based on the request/response of each URL. **Important**: This generator only supports "happy" paths at the moment. In order to test validation errors or invalid requests, one should implement them programmatically. Also, if multiple URL(s) are defined -in the `test-urls.txt` file, they all have to have the same format - remember that the implementation of the generator -must match the URL(s) format expected. +in the `test-urls.txt` file, they must have the same format - remember that the implementation of the generator +must match the URL(s) format expected, and we can pick only one generator at time. diff --git a/dhis-2/dhis-test-e2e/pom.xml b/dhis-2/dhis-test-e2e/pom.xml index 82a4ebfcf65a..d9b2fdc7139e 100644 --- a/dhis-2/dhis-test-e2e/pom.xml +++ b/dhis-2/dhis-test-e2e/pom.xml @@ -11,11 +11,11 @@ UTF-8 2.40.0 3.11.0 - 3.1.2 + 3.2.1 1.2.1 5.10.0 2.10.1 - 2.20.0 + 2.21.1 5.3.2 2.15.3 32.1.3-jre @@ -35,7 +35,7 @@ 2.0.9 4.5.14 5.2.1 - 2.14.0 + 2.15.0 diff --git a/dhis-2/dhis-test-e2e/src/main/java/org/hisp/dhis/actions/RestApiActions.java b/dhis-2/dhis-test-e2e/src/main/java/org/hisp/dhis/actions/RestApiActions.java index 9adc34a4e1a9..df9eb334883f 100644 --- a/dhis-2/dhis-test-e2e/src/main/java/org/hisp/dhis/actions/RestApiActions.java +++ b/dhis-2/dhis-test-e2e/src/main/java/org/hisp/dhis/actions/RestApiActions.java @@ -35,6 +35,7 @@ import io.restassured.RestAssured; import io.restassured.config.ObjectMapperConfig; import io.restassured.http.ContentType; +import io.restassured.http.Headers; import io.restassured.mapper.ObjectMapperType; import io.restassured.response.Response; import io.restassured.specification.RequestSpecification; @@ -164,6 +165,25 @@ public ApiResponse get(String resourceId, QueryParamsBuilder queryParamsBuilder) return new ApiResponse(response); } + /** + * Sends get request with provided path, headers & queryParams appended to URL. + * + * @param resourceId Id of resource + * @param queryParamsBuilder Query params to append to url + * @param headers headers to send as part of the request + */ + public ApiResponse getWithHeaders( + String resourceId, QueryParamsBuilder queryParamsBuilder, Headers headers) { + String path = queryParamsBuilder == null ? "" : queryParamsBuilder.build(); + + addCoverage("GET", resourceId + path); + + Response response = + this.given().contentType(ContentType.TEXT).headers(headers).when().get(resourceId + path); + + return new ApiResponse(response); + } + public ApiResponse get(QueryParamsBuilder queryParamsBuilder) { return this.get("", queryParamsBuilder); } diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/AnalyticsApiTest.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/AnalyticsApiTest.java index dff5eb46c73c..95a58b518f31 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/AnalyticsApiTest.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/AnalyticsApiTest.java @@ -71,7 +71,7 @@ public abstract class AnalyticsApiTest { protected static final int DEFAULT_LIMIT_EXECUTION_TIME = 30; - protected static final String JSON = ContentType.JSON.toString(); + public static final String JSON = ContentType.JSON.toString(); @BeforeAll public void beforeAll() { diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/NoAnalyticsTablesErrorsScenariosTest.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/NoAnalyticsTablesErrorsScenariosTest.java new file mode 100644 index 000000000000..de8551762fc7 --- /dev/null +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/NoAnalyticsTablesErrorsScenariosTest.java @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics; + +import static org.hamcrest.Matchers.equalTo; +import static org.hisp.dhis.AnalyticsApiTest.JSON; + +import org.hisp.dhis.actions.LoginActions; +import org.hisp.dhis.actions.RestApiActions; +import org.hisp.dhis.actions.analytics.AnalyticsEnrollmentsActions; +import org.hisp.dhis.actions.analytics.AnalyticsEventActions; +import org.hisp.dhis.actions.analytics.AnalyticsTeiActions; +import org.hisp.dhis.dto.ApiResponse; +import org.hisp.dhis.helpers.QueryParamsBuilder; +import org.hisp.dhis.helpers.extensions.ConfigurationExtension; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * This test class has to run before all tests, because the scenarios are testing errors related to + * missing analytics tables. For this reason they have to run first (before analytics tables are + * created), hence @Order(1). + */ +@Order(1) +@ExtendWith(ConfigurationExtension.class) +@Tag("analytics") +public class NoAnalyticsTablesErrorsScenariosTest { + + private final AnalyticsEnrollmentsActions analyticsEnrollmentsActions = + new AnalyticsEnrollmentsActions(); + private final AnalyticsEventActions analyticsEventActions = new AnalyticsEventActions(); + private final AnalyticsTeiActions analyticsTeiActions = new AnalyticsTeiActions(); + private final RestApiActions analyticsAggregateActions = new RestApiActions("analytics"); + + @BeforeAll + public static void beforeAll() { + new LoginActions().loginAsAdmin(); + } + + @Test + void testAggregateAnalyticsWhenAnalyticsTablesAreMissing() { + // Given + QueryParamsBuilder params = + new QueryParamsBuilder() + .add("dimension=dx:Uvn6LCg7dVU;sB79w2hiLp8,ou:USER_ORGUNIT;USER_ORGUNIT_CHILDREN") + .add("filter=pe:THIS_YEAR"); + + // When + ApiResponse response = analyticsAggregateActions.get(params); + + // Then + assertNoAnalyticsTableResponse(response); + } + + @Test + void testEventsQueryAnalyticsWhenAnalyticsTablesAreMissing() { + // Given + QueryParamsBuilder params = + new QueryParamsBuilder() + .add("dimension=ou:ImspTQPwCqd,eMyVanycQSC") + .add("eventDate=THIS_QUARTER"); + + // When + ApiResponse response = analyticsEventActions.query().get("eBAyeGv0exc", JSON, JSON, params); + + // Then + assertNoAnalyticsTableResponse(response); + } + + @Test + void testEnrollmentsQueryAnalyticsWhenAnalyticsTablesAreMissing() { + // Given + QueryParamsBuilder params = new QueryParamsBuilder().add("dimension=ou:ImspTQPwCqd"); + + // When + ApiResponse response = + analyticsEnrollmentsActions.query().get("IpHINAT79UW", JSON, JSON, params); + + // Then + assertNoAnalyticsTableResponse(response); + } + + @Test + void testEventsAggregateAnalyticsWhenAnalyticsTablesAreMissing() { + // Given + QueryParamsBuilder params = + new QueryParamsBuilder() + .add("dimension=ou:ImspTQPwCqd,pe:LAST_12_MONTHS") + .add("stage=A03MvHHogjR"); + + // When + ApiResponse response = analyticsEventActions.aggregate().get("IpHINAT79UW", JSON, JSON, params); + + // Then + assertNoAnalyticsTableResponse(response); + } + + @Test + public void queryWithProgramAndEnrollmentDateAndInvalidEnrollmentOffset() { + // Given + QueryParamsBuilder params = + new QueryParamsBuilder() + .add("program=IpHINAT79UW") + .add("enrollmentDate=IpHINAT79UW.LAST_YEAR"); + + // When + ApiResponse response = analyticsTeiActions.query().get("nEenWmSyUEp", JSON, JSON, params); + + // Then + assertNoAnalyticsTableResponse(response); + } + + private void assertNoAnalyticsTableResponse(ApiResponse response) { + response + .validate() + .statusCode(409) + .body("status", equalTo("ERROR")) + .body( + "message", + equalTo( + "Query failed because a referenced table does not exist. Please ensure analytics job was run (SqlState: 42P01)")) + .body("errorCode", equalTo("E7144")) + .body("devMessage", equalTo("SqlState: 42P01")); + } +} diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/aggregate/AnalyticsQueryTest.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/aggregate/AnalyticsQueryTest.java index 866931bef69e..4b40b531eaea 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/aggregate/AnalyticsQueryTest.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/aggregate/AnalyticsQueryTest.java @@ -112,7 +112,7 @@ public void query1And3CoverageYearly() { // Assert metaData. assertEquals( response.extract("metaData").toString().replaceAll(" ", ""), - "{items={sB79w2hiLp8={name=ANC 3 Coverage}, jUb8gELQApl={name=Kailahun}, TEQlaapDQoK={name=Port Loko}, eIQbndfxQMb={name=Tonkolili}, Vth0fbpFcsO={name=Kono}, PMa2VCrupOd={name=Kambia}, ou={name=Organisation unit}, THIS_YEAR={name=This year}, O6uvpzGd5pu={name=Bo}, bL4ooGhyHRQ={name=Pujehun}, 2022={name=2022}, kJq2mPyFEHo={name=Kenema}, fdc6uOvgoji={name=Bombali}, ImspTQPwCqd={name=Sierra Leone}, at6UHUQatSo={name=Western Area}, dx={name=Data}, pe={name=Period}, Uvn6LCg7dVU={name=ANC 1 Coverage}, lc3eMKXaEfw={name=Bonthe}, qhqAxPSTUXp={name=Koinadugu}, jmIPBj66vD6={name=Moyamba}}, dimensions={dx=[Uvn6LCg7dVU,sB79w2hiLp8], pe=[2022], ou=[ImspTQPwCqd,O6uvpzGd5pu,fdc6uOvgoji,lc3eMKXaEfw,jUb8gELQApl,PMa2VCrupOd,kJq2mPyFEHo,qhqAxPSTUXp,Vth0fbpFcsO,jmIPBj66vD6,TEQlaapDQoK,bL4ooGhyHRQ,eIQbndfxQMb,at6UHUQatSo], co=[]}}" + "{items={sB79w2hiLp8={name=ANC 3 Coverage}, jUb8gELQApl={name=Kailahun}, TEQlaapDQoK={name=Port Loko}, eIQbndfxQMb={name=Tonkolili}, Vth0fbpFcsO={name=Kono}, PMa2VCrupOd={name=Kambia}, ou={name=Organisation unit}, USER_ORGUNIT={organisationUnits=[ImspTQPwCqd]}, THIS_YEAR={name=This year}, O6uvpzGd5pu={name=Bo}, bL4ooGhyHRQ={name=Pujehun}, 2022={name=2022}, kJq2mPyFEHo={name=Kenema}, USER_ORGUNIT_CHILDREN={organisationUnits=[at6UHUQatSo,TEQlaapDQoK,PMa2VCrupOd,qhqAxPSTUXp,kJq2mPyFEHo,jmIPBj66vD6,Vth0fbpFcsO,jUb8gELQApl,fdc6uOvgoji,eIQbndfxQMb,O6uvpzGd5pu,lc3eMKXaEfw,bL4ooGhyHRQ]}, fdc6uOvgoji={name=Bombali}, ImspTQPwCqd={name=Sierra Leone}, at6UHUQatSo={name=Western Area}, dx={name=Data}, pe={name=Period}, Uvn6LCg7dVU={name=ANC 1 Coverage}, lc3eMKXaEfw={name=Bonthe}, qhqAxPSTUXp={name=Koinadugu}, jmIPBj66vD6={name=Moyamba}}, dimensions={dx=[Uvn6LCg7dVU,sB79w2hiLp8], pe=[2022], ou=[ImspTQPwCqd,O6uvpzGd5pu,fdc6uOvgoji,lc3eMKXaEfw,jUb8gELQApl,PMa2VCrupOd,kJq2mPyFEHo,qhqAxPSTUXp,Vth0fbpFcsO,jmIPBj66vD6,TEQlaapDQoK,bL4ooGhyHRQ,eIQbndfxQMb,at6UHUQatSo], co=[]}}" .replaceAll(" ", "")); // Assert headers. validateHeader(response, 0, "dx", "Data", "TEXT", "java.lang.String", false, true); @@ -199,4 +199,21 @@ public void testAnalyticsGetWithLongTextDataElementAggregationTypeSum() { "202210", "Cholera is an infection of the small intestine caused by the bacterium Vibrio cholerae.\n\nThe main symptoms are watery diarrhea and vomiting. This may result in dehydration and in severe cases grayish-bluish skin.[1] Transmission occurs primarily by drinking water or eating food that has been contaminated by the feces (waste product) of an infected person, including one with no apparent symptoms.\n\nThe severity of the diarrhea and vomiting can lead to rapid dehydration and electrolyte imbalance, and death in some cases. The primary treatment is oral rehydration therapy, typically with oral rehydration solution, to replace water and electrolytes. If this is not tolerated or does not provide improvement fast enough, intravenous fluids can also be used. Antibacterial drugs are beneficial in those with severe disease to shorten its duration and severity.")); } + + @Test + public void testQueryFailsGracefullyIfMultipleQueries() { + // Given + QueryParamsBuilder params = + new QueryParamsBuilder() + .add( + "dimension=cX5k9anHEHd:apsOixVZlf1;jRbMi0aBjYn,dx:luLGbE2WKGP;nq5ohBSWj6E,pe:LAST_12_MONTHS") + .add("filter=ou:USER_ORGUNIT") + .add("displayProperty=SHORTNAME"); + + // When + ApiResponse response = analyticsActions.get(params); + + // Then + response.validate().statusCode(200); + } } diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/DataSetMetadataTest.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/DataSetMetadataTest.java new file mode 100644 index 000000000000..4e2a141d42a8 --- /dev/null +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/DataSetMetadataTest.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.metadata; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import io.restassured.http.Header; +import io.restassured.http.Headers; +import io.restassured.response.Response; +import org.apache.http.HttpHeaders; +import org.hisp.dhis.ApiTest; +import org.hisp.dhis.actions.LoginActions; +import org.hisp.dhis.actions.RestApiActions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * @author david mackessy + */ +class DataSetMetadataTest extends ApiTest { + + private RestApiActions restApiActions; + private RestApiActions dataSetActions; + + @BeforeAll + public void beforeAll() { + restApiActions = new RestApiActions("dataEntry/metadata"); + dataSetActions = new RestApiActions("dataSets"); + LoginActions loginActions = new LoginActions(); + loginActions.loginAsSuperUser(); + } + + @Test + void dataSetMetadataEtagFunctionalityTest() { + // call endpoint to get current state + Response response1 = restApiActions.get().validate().extract().response(); + + int statusCode1 = response1.getStatusCode(); + assertEquals(200, statusCode1); + String responseBody1 = response1.body().asString(); + assertNotNull(responseBody1); + // get etag value from current state + String eTagValue1 = response1.getHeader(HttpHeaders.ETAG); + + // make the same call again, this time passing the 'If-None-Match' header and the ETag value + // from response 1 + Headers headers = new Headers(new Header(HttpHeaders.IF_NONE_MATCH, eTagValue1)); + Response response2 = + restApiActions.getWithHeaders("", null, headers).validate().extract().response(); + + int statusCode2 = response2.getStatusCode(); + + // response status code should be 304 to indicate that the data has not changed + assertEquals(304, statusCode2); + + // body should be empty as no data returned when no change in data + assertEquals("", response2.body().asString()); + + // ETags should match from response 1 & 2 + String eTagValue2 = response2.getHeader(HttpHeaders.ETAG); + assertNotNull(eTagValue1); + assertEquals(34, eTagValue2.length()); + assertEquals(eTagValue1, eTagValue2); + + // create new data set to trigger a change of data seen by the API + dataSetActions.post("", newDataSet()).validateStatus(201); + + // call again with 'If-None-Match' header and the previous ETag header value + Response response3 = + restApiActions.getWithHeaders("", null, headers).validate().extract().response(); + + // new ETag should be received + String eTagValue3 = response3.getHeader(HttpHeaders.ETAG); + assertNotEquals(eTagValue1, eTagValue3); + + int statusCode3 = response3.getStatusCode(); + + // response 1 & 3 bodies should not match + assertNotEquals(responseBody1, response3.body().asString()); + assertEquals(200, statusCode3); + + assertNotNull(eTagValue3); + assertEquals(34, eTagValue3.length()); + // ETags should not match + assertNotEquals(eTagValue1, eTagValue3); + } + + private String newDataSet() { + return """ + { + "name": "e2e test dataset 1", + "shortName": "e2e test dataset 1", + "periodType": "Daily", + "organisationUnits": [] + } + """ + .strip(); + } +} diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/metadata_import/GeoJsonImportTest.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/metadata_import/GeoJsonImportTest.java new file mode 100644 index 000000000000..dfba18faabf1 --- /dev/null +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/metadata_import/GeoJsonImportTest.java @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.metadata.metadata_import; + +import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.hisp.dhis.ApiTest; +import org.hisp.dhis.actions.LoginActions; +import org.hisp.dhis.actions.RestApiActions; +import org.hisp.dhis.actions.SystemActions; +import org.hisp.dhis.dto.ApiResponse; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * @author david mackessy + */ +class GeoJsonImportTest extends ApiTest { + private LoginActions loginActions; + private SystemActions systemActions; + private RestApiActions restApiActions; + + private String geoJson; + + @BeforeEach + public void before() { + loginActions = new LoginActions(); + systemActions = new SystemActions(); + restApiActions = new RestApiActions("organisationUnits"); + + loginActions.loginAsSuperUser(); + + // make sure org unit has no geo json data + ApiResponse deleteOrgUnitResponse = restApiActions.delete("/ImspTQPwCqd/geometry"); + assertEquals(200, deleteOrgUnitResponse.statusCode()); + + // create geo json + geoJson = geoJson(); + } + + @Test + void geoJsonImportAsync() { + // get org unit geometry to show currently empty + ApiResponse getOrgUnitResponse = restApiActions.get("/ImspTQPwCqd"); + assertNull(getOrgUnitResponse.getBody().get("geometry")); + + // post geo json async + ApiResponse postGeoJsonAsyncResponse = restApiActions.post("/geometry?async=true", geoJson); + assertEquals(200, postGeoJsonAsyncResponse.statusCode()); + + assertTrue( + postGeoJsonAsyncResponse + .getBody() + .get("message") + .getAsString() + .contains("Initiated GEOJSON_IMPORT")); + + String taskId = + postGeoJsonAsyncResponse.getBody().getAsJsonObject("response").get("id").getAsString(); + assertEquals(11, taskId.length()); + + // wait for job to be completed (24 seconds used as the job schedule loop is 20 seconds) + ApiResponse taskStatus = systemActions.waitUntilTaskCompleted("GEOJSON_IMPORT", taskId, 24); + assertTrue(taskStatus.getAsString().contains("\"completed\":true")); + + // get org unit again which should now contain geometry property + ApiResponse getUpdatedOrgUnit = restApiActions.get("/ImspTQPwCqd"); + + // validate async-completed geo json import + getUpdatedOrgUnit + .validate() + .statusCode(200) + .body("geometry.type", equalTo("Point")) + .body("geometry.coordinates.size()", equalTo(2)); + } + + @Test + void geoJsonImportSync() { + // get org unit geometry to show currently empty + ApiResponse getOrgUnitResponse = restApiActions.get("/ImspTQPwCqd"); + assertNull(getOrgUnitResponse.getBody().get("geometry")); + + // post geo json sync + ApiResponse postGeoJsonSyncResponse = restApiActions.post("/geometry", geoJson); + assertEquals(200, postGeoJsonSyncResponse.statusCode()); + + assertTrue( + postGeoJsonSyncResponse + .getBody() + .get("message") + .getAsString() + .contains("Import successful.")); + + // get org unit again which should now contain geometry property + ApiResponse getUpdatedOrgUnit = restApiActions.get("/ImspTQPwCqd"); + + // validate sync-completed geo json import + getUpdatedOrgUnit + .validate() + .statusCode(200) + .body("geometry.type", equalTo("Point")) + .body("geometry.coordinates.size()", equalTo(2)); + } + + private String geoJson() { + return """ + { + "type": "FeatureCollection", + "features":[ + { + "type": "Feature", + "id": "ImspTQPwCqd", + "geometry": { + "type": "Point", + "coordinates": [-12.56262192106476,7.376283621891673] + }, + "properties": { + "Shape_Leng": 20.8503761122, + "Shape_Area": 5.96427565724, + "shapeName": "Republic of Sierra Leone", + "Level": "ADM0", + "shapeISO": "SLE", + "shapeID": "SLE-ADM0-89611731B79725766", + "shapeGroup": "SLE", + "shapeType": "ADM0", + "code": "OU_525" + } + } + ] + } + """; + } +} diff --git a/dhis-2/dhis-test-e2e/src/test/resources/junit-platform.properties b/dhis-2/dhis-test-e2e/src/test/resources/junit-platform.properties new file mode 100644 index 000000000000..46edb4f8e427 --- /dev/null +++ b/dhis-2/dhis-test-e2e/src/test/resources/junit-platform.properties @@ -0,0 +1,2 @@ +# Enables the execution of ordering of classes through the annotation @Order +junit.jupiter.testclass.order.default=org.junit.jupiter.api.ClassOrderer$OrderAnnotation diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceTest.java index 7d058e9f9f66..87e3304a1d0f 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceTest.java @@ -155,6 +155,8 @@ class AnalyticsServiceTest extends SingleSetupIntegrationTestBase { private String deBUid; + private String deDUid; + private DataElement deH; private OrganisationUnit ouA; @@ -311,6 +313,7 @@ private void setUpMetadata() { deAUid = deA.getUid(); deBUid = deB.getUid(); + deDUid = deD.getUid(); dataElementService.addDataElement(deA); dataElementService.addDataElement(deB); @@ -518,7 +521,7 @@ private void parseDataValues(List lines) { dataValueService.addDataValue(dataValue); } assertEquals( - 30, + 32, dataValueService.getAllDataValues().size(), "Import of data values failed, number of imports are wrong"); } @@ -1083,12 +1086,18 @@ void testIndicatorSummingBooleans() { withIndicator(inA, "#{" + deF.getUid() + "}"); assertDataValues( - Map.of("indicatorAA-ouabcdefghA-201701", 1.0), + Map.of( + "indicatorAA-ouabcdefghA-201701", + 1.0, + "indicatorAA-ouabcdefghA-201702", + 0.0, + "indicatorAA-ouabcdefghA-201703", + 1.0), DataQueryParams.newBuilder() .withOrganisationUnit(ouA) .withIndicators(List.of(inA)) .withAggregationType(AnalyticsAggregationType.SUM) - .withPeriods(List.of(peJan, peFeb)) + .withPeriods(List.of(peJan, peFeb, peMar, peApr)) .withOutputFormat(OutputFormat.ANALYTICS) .build()); } @@ -1113,12 +1122,18 @@ void testIndicatorSubexpressionBoolean() { withIndicator(inA, "subExpression( if( #{" + deF.getUid() + "}, 3, 4 ) )"); assertDataValues( - Map.of("indicatorAA-ouabcdefghA-201701", 3.0), + Map.of( + "indicatorAA-ouabcdefghA-201701", + 3.0, + "indicatorAA-ouabcdefghA-201702", + 4.0, + "indicatorAA-ouabcdefghA-201703", + 3.0), DataQueryParams.newBuilder() .withOrganisationUnit(ouA) .withIndicators(List.of(inA)) .withAggregationType(AnalyticsAggregationType.SUM) - .withPeriods(List.of(peJan, peFeb)) + .withPeriods(List.of(peJan, peFeb, peMar, peApr)) .withOutputFormat(OutputFormat.ANALYTICS) .build()); } @@ -1129,12 +1144,78 @@ void testIndicatorSubexpressionBooleanSum() { inA, "subExpression( if( #{" + deF.getUid() + "}.aggregationType(SUM) > 0, 5, 6 ) )"); assertDataValues( - Map.of("indicatorAA-ouabcdefghA-201701", 5.0), + Map.of( + "indicatorAA-ouabcdefghA-201701", + 5.0, + "indicatorAA-ouabcdefghA-201702", + 6.0, + "indicatorAA-ouabcdefghA-201703", + 5.0), DataQueryParams.newBuilder() .withOrganisationUnit(ouA) .withIndicators(List.of(inA)) .withAggregationType(AnalyticsAggregationType.SUM) - .withPeriods(List.of(peJan, peFeb)) + .withPeriods(List.of(peJan, peFeb, peMar, peApr)) + .withOutputFormat(OutputFormat.ANALYTICS) + .build()); + } + + @Test + void testIndicatorSubexpressionBooleanMinQuarter() { + withIndicator(inA, "subExpression(#{" + deF.getUid() + "}.aggregationType(MIN))"); + + assertDataValues( + Map.of("indicatorAA-ouabcdefghA-2017Q1", 0.0), + DataQueryParams.newBuilder() + .withOrganisationUnit(ouA) + .withIndicators(List.of(inA)) + .withAggregationType(AnalyticsAggregationType.SUM) + .withPeriods(List.of(quarter)) + .withOutputFormat(OutputFormat.ANALYTICS) + .build()); + } + + @Test + void testIndicatorSubexpressionBooleanMaxQuarter() { + withIndicator(inA, "subExpression(#{" + deF.getUid() + "}.aggregationType(MAX))"); + + assertDataValues( + Map.of("indicatorAA-ouabcdefghA-2017Q1", 1.0), + DataQueryParams.newBuilder() + .withOrganisationUnit(ouA) + .withIndicators(List.of(inA)) + .withAggregationType(AnalyticsAggregationType.SUM) + .withPeriods(List.of(quarter)) + .withOutputFormat(OutputFormat.ANALYTICS) + .build()); + } + + @Test + void testIndicatorSubexpressionBooleanSumQuarter() { + withIndicator(inA, "subExpression(#{" + deF.getUid() + "}.aggregationType(SUM))"); + + assertDataValues( + Map.of("indicatorAA-ouabcdefghA-2017Q1", 2.0), + DataQueryParams.newBuilder() + .withOrganisationUnit(ouA) + .withIndicators(List.of(inA)) + .withAggregationType(AnalyticsAggregationType.SUM) + .withPeriods(List.of(quarter)) + .withOutputFormat(OutputFormat.ANALYTICS) + .build()); + } + + @Test + void testIndicatorSubexpressionBooleanCountQuarter() { + withIndicator(inA, "subExpression(#{" + deF.getUid() + "}.aggregationType(COUNT))"); + + assertDataValues( + Map.of("indicatorAA-ouabcdefghA-2017Q1", 3.0), + DataQueryParams.newBuilder() + .withOrganisationUnit(ouA) + .withIndicators(List.of(inA)) + .withAggregationType(AnalyticsAggregationType.SUM) + .withPeriods(List.of(quarter)) .withOutputFormat(OutputFormat.ANALYTICS) .build()); } @@ -1244,6 +1325,41 @@ void testIndicatorSubexpressionAverageWithAndWithoutMax() { .build()); } + @Test + void testIndicatorSubexpressionPeriodOffset() { + withIndicator( + inA, + "subExpression(if(#{" + + deDUid + + "}+#{" + + deDUid + + "}.periodOffset(-1)+#{" + + deDUid + + "}.periodOffset(-2)>0,1,0))"); + + assertDataValues( + Map.of( + "indicatorAA-ouabcdefghA-201702", + 3.0, + "indicatorAA-ouabcdefghA-201703", + 3.0, + "indicatorAA-ouabcdefghA-201704", + 4.0, + "indicatorAA-ouabcdefghB-201702", + 1.0, + "indicatorAA-ouabcdefghB-201703", + 1.0, + "indicatorAA-ouabcdefghB-201704", + 2.0), + DataQueryParams.newBuilder() + .withOrganisationUnits(List.of(ouA, ouB)) + .withIndicators(List.of(inA)) + .withAggregationType(AnalyticsAggregationType.SUM) + .withPeriods(List.of(peFeb, peMar, peApr)) + .withOutputFormat(OutputFormat.ANALYTICS) + .build()); + } + @Test void testIndicatorWithTwoSubexpressions() { // Note: Expressions and values are the sum of the two previous tests. diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/expression/ExpressionServiceTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/expression/ExpressionServiceTest.java index 076f4c6c4b3d..dfc9c81b2355 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/expression/ExpressionServiceTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/expression/ExpressionServiceTest.java @@ -701,12 +701,15 @@ private String itemNameOrSubexpression(DimensionalItemObject item) { + subex.getQueryMods().getValueType().name() : item.getName(); + String periodOffset = + (item.getPeriodOffset() != 0) ? ".periodOffset(" + item.getPeriodOffset() + ")" : ""; + String typeOverride = (item.getQueryMods() != null && item.getQueryMods().getAggregationType() != null) ? ".aggregationType(" + item.getQueryMods().getAggregationType().name() + ")" : ""; - return name + typeOverride; + return name + periodOffset + typeOverride; } /** @@ -1156,52 +1159,58 @@ void testSubExpressions() { Map valueMap = emptyMap(); assertEquals( - "0 DeX [ case when \"dataElemenX\" > 99 then 1 else 2 end]::NUMBER", + "0 DeX [ case when coalesce(\"dataElemenX\",0) > 99 then 1 else 2 end]::NUMBER", evalIndicator("subExpression(if(#{dataElemenX}>99,1,2))", valueMap)); assertEquals( - "0 DeX [ case when \"dataElemenX\" > 0 and \"dataElemenX\" < 3 then \"dataElemenX\" else 3 end]::NUMBER", + "0 DeX [ case when coalesce(\"dataElemenX\",0) > 0 and coalesce(\"dataElemenX\",0) < 3 then coalesce(\"dataElemenX\",0) else 3 end]::NUMBER", evalIndicator( "subExpression(if(#{dataElemenX}>0 && #{dataElemenX}<3,#{dataElemenX},3))", valueMap)); assertEquals( - "5 DeX [ case when \"dataElemenX\" > 99 then 'a' else 'b' end]::TEXT", + "5 DeX [ case when coalesce(\"dataElemenX\",0) > 99 then 'a' else 'b' end]::TEXT", evalIndicator("if( subExpression(if(#{dataElemenX}>99,'a','b')) == 'a', 4, 5)", valueMap)); assertEquals( - "7 DeZ [ case when \"dataElemenZ\" != 'a' then 1 else 2 end]::NUMBER", + "7 DeZ [ case when coalesce(\"dataElemenZ\",'') != 'a' then 1 else 2 end]::NUMBER", evalIndicator("if( subExpression(if(#{dataElemenZ} != 'a', 1, 2)) == 2, 6, 7)", valueMap)); assertEquals( - "9 DeZ [ case when \"dataElemenZ\" != 'a' and \"dataElemenZ\" != 'b' then 'c' else 'd' end]::TEXT", + "9 DeZ [ case when coalesce(\"dataElemenZ\",'') != 'a' and coalesce(\"dataElemenZ\",'') != 'b' then 'c' else 'd' end]::TEXT", evalIndicator( "if(subExpression(if(#{dataElemenZ}!='a'&&#{dataElemenZ}!='b','c','d')) == 'd',8,9)", valueMap)); assertEquals( - "0 DeX CocA [ case when \"dataElemenX_catOptCombA\" > 99 then 10 else 11 end]::NUMBER", + "0 DeX CocA [ case when coalesce(\"dataElemenX_catOptCombA\",0) > 99 then 10 else 11 end]::NUMBER", evalIndicator("subExpression( if( #{dataElemenX.catOptCombA} > 99, 10, 11 ) )", valueMap)); assertEquals( - "0 DeX DeY [\"dataElemenX\" / \"dataElemenY\"]::NUMBER", + "0 DeX DeY [coalesce(\"dataElemenX\",0) / coalesce(\"dataElemenY\",0)]::NUMBER", evalIndicator("subExpression( #{dataElemenX} / #{dataElemenY} )", valueMap)); assertEquals( - "0 DeX DeY [\"dataElemenX\" / \"dataElemenY\"]::NUMBER.aggregationType(MAX)", + "0 DeX DeY [coalesce(\"dataElemenX\",0) / coalesce(\"dataElemenY\",0)]::NUMBER.aggregationType(MAX)", evalIndicator( "subExpression( #{dataElemenX} / #{dataElemenY} ).aggregationType(MAX)", valueMap)); assertEquals( - "0 DeX.aggregationType(AVERAGE) DeY [\"dataElemenXAVERAGE\" / \"dataElemenY\"]::NUMBER.aggregationType(MAX)", + "0 DeX.aggregationType(AVERAGE) DeY [coalesce(\"dataElemenX_agg_AVERAGE\",0) / coalesce(\"dataElemenY\",0)]::NUMBER.aggregationType(MAX)", evalIndicator( "subExpression( #{dataElemenX}.aggregationType(AVERAGE) / #{dataElemenY} ).aggregationType(MAX)", valueMap)); assertEquals( - "0 DeX DeX.aggregationType(AVERAGE) DeY [\"dataElemenX\" + \"dataElemenXAVERAGE\" / \"dataElemenY\"]::NUMBER.aggregationType(MAX)", + "0 DeX DeX.aggregationType(AVERAGE) DeY [coalesce(\"dataElemenX\",0) + coalesce(\"dataElemenX_agg_AVERAGE\",0) / coalesce(\"dataElemenY\",0)]::NUMBER.aggregationType(MAX)", evalIndicator( "subExpression( #{dataElemenX} + #{dataElemenX}.aggregationType(AVERAGE) / #{dataElemenY} ).aggregationType(MAX)", valueMap)); + + assertEquals( + "0 DeX.periodOffset(-2).aggregationType(AVERAGE) DeX.periodOffset(1) [coalesce(\"dataElemenX_plus_1\",0) + coalesce(\"dataElemenX_minus_2_agg_AVERAGE\",0)]::NUMBER.aggregationType(MAX)", + evalIndicator( + "subExpression( #{dataElemenX}.periodOffset(1) + #{dataElemenX}.periodOffset(-2).aggregationType(AVERAGE)).aggregationType(MAX)", + valueMap)); } @Test diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/OrderAndPaginationExporterTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/OrderAndPaginationExporterTest.java index 7f0aec4df651..724235fb1e8e 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/OrderAndPaginationExporterTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/OrderAndPaginationExporterTest.java @@ -67,9 +67,7 @@ import org.hisp.dhis.tracker.TrackerTest; import org.hisp.dhis.tracker.TrackerType; import org.hisp.dhis.tracker.export.enrollment.EnrollmentOperationParams; -import org.hisp.dhis.tracker.export.enrollment.EnrollmentOperationParams.EnrollmentOperationParamsBuilder; import org.hisp.dhis.tracker.export.enrollment.EnrollmentService; -import org.hisp.dhis.tracker.export.enrollment.Enrollments; import org.hisp.dhis.tracker.export.event.EventOperationParams; import org.hisp.dhis.tracker.export.event.EventOperationParams.EventOperationParamsBuilder; import org.hisp.dhis.tracker.export.event.EventService; @@ -540,69 +538,69 @@ void shouldOrderTrackedEntitiesByMultipleAttributesDesc() @Test void shouldReturnPaginatedEnrollmentsGivenNonDefaultPageSize() - throws ForbiddenException, BadRequestException { - EnrollmentOperationParamsBuilder builder = + throws ForbiddenException, BadRequestException, NotFoundException { + EnrollmentOperationParams operationParams = EnrollmentOperationParams.builder() .orgUnitUids(Set.of(orgUnit.getUid())) - .orderBy("enrollmentDate", SortDirection.ASC); - - EnrollmentOperationParams params = builder.page(1).pageSize(1).build(); + .orderBy("enrollmentDate", SortDirection.ASC) + .build(); - Enrollments firstPage = enrollmentService.getEnrollments(params); + Page firstPage = + enrollmentService.getEnrollments(operationParams, new PageParams(1, 1, false)); assertAll( "first page", - () -> assertSlimPager(1, 1, false, firstPage.getPager()), - () -> assertEquals(List.of("nxP7UnKhomJ"), uids(firstPage.getEnrollments()))); - - params = builder.page(2).pageSize(1).build(); + () -> assertPager(1, 1, firstPage), + () -> assertEquals(List.of("nxP7UnKhomJ"), uids(firstPage.getItems()))); - Enrollments secondPage = enrollmentService.getEnrollments(params); + Page secondPage = + enrollmentService.getEnrollments(operationParams, new PageParams(2, 1, false)); assertAll( - "second (last) page", - () -> assertSlimPager(2, 1, true, secondPage.getPager()), - () -> assertEquals(List.of("TvctPPhpD8z"), uids(secondPage.getEnrollments()))); + "second page is last page", + () -> assertPager(2, 1, secondPage), + () -> assertEquals(List.of("TvctPPhpD8z"), uids(secondPage.getItems()))); - params = builder.page(3).pageSize(1).build(); + Page thirdPage = + enrollmentService.getEnrollments(operationParams, new PageParams(3, 1, false)); - assertIsEmpty(getEnrollments(params)); + assertIsEmpty(thirdPage.getItems()); } @Test void shouldReturnPaginatedEnrollmentsGivenNonDefaultPageSizeAndTotalPages() - throws ForbiddenException, BadRequestException { - EnrollmentOperationParamsBuilder builder = + throws ForbiddenException, BadRequestException, NotFoundException { + EnrollmentOperationParams operationParams = EnrollmentOperationParams.builder() .orgUnitUids(Set.of(orgUnit.getUid())) - .orderBy("enrollmentDate", SortDirection.ASC); - - EnrollmentOperationParams params = builder.page(1).pageSize(1).totalPages(true).build(); + .orderBy("enrollmentDate", SortDirection.ASC) + .build(); - Enrollments firstPage = enrollmentService.getEnrollments(params); + Page firstPage = + enrollmentService.getEnrollments(operationParams, new PageParams(1, 1, true)); assertAll( "first page", () -> assertPager(1, 1, 2, firstPage.getPager()), - () -> assertEquals(List.of("nxP7UnKhomJ"), uids(firstPage.getEnrollments()))); + () -> assertEquals(List.of("nxP7UnKhomJ"), uids(firstPage.getItems()))); - params = builder.page(2).pageSize(1).totalPages(true).build(); - - Enrollments secondPage = enrollmentService.getEnrollments(params); + Page secondPage = + enrollmentService.getEnrollments(operationParams, new PageParams(2, 1, true)); assertAll( "second (last) page", () -> assertPager(2, 1, 2, secondPage.getPager()), - () -> assertEquals(List.of("TvctPPhpD8z"), uids(secondPage.getEnrollments()))); + () -> assertEquals(List.of("TvctPPhpD8z"), uids(secondPage.getItems()))); - params = builder.page(3).pageSize(1).totalPages(true).build(); + Page thirdPage = + enrollmentService.getEnrollments(operationParams, new PageParams(3, 1, true)); - assertIsEmpty(getEnrollments(params)); + assertIsEmpty(thirdPage.getItems()); } @Test void shouldOrderEnrollmentsByPrimaryKeyDescByDefault() - throws ForbiddenException, BadRequestException { + throws ForbiddenException, BadRequestException, NotFoundException { Enrollment nxP7UnKhomJ = get(Enrollment.class, "nxP7UnKhomJ"); Enrollment TvctPPhpD8z = get(Enrollment.class, "TvctPPhpD8z"); List expected = @@ -620,7 +618,8 @@ void shouldOrderEnrollmentsByPrimaryKeyDescByDefault() } @Test - void shouldOrderEnrollmentsByEnrolledAtAsc() throws ForbiddenException, BadRequestException { + void shouldOrderEnrollmentsByEnrolledAtAsc() + throws ForbiddenException, BadRequestException, NotFoundException { EnrollmentOperationParams params = EnrollmentOperationParams.builder() .orgUnitUids(Set.of(orgUnit.getUid())) @@ -633,7 +632,8 @@ void shouldOrderEnrollmentsByEnrolledAtAsc() throws ForbiddenException, BadReque } @Test - void shouldOrderEnrollmentsByEnrolledAtDesc() throws ForbiddenException, BadRequestException { + void shouldOrderEnrollmentsByEnrolledAtDesc() + throws ForbiddenException, BadRequestException, NotFoundException { EnrollmentOperationParams params = EnrollmentOperationParams.builder() .orgUnitUids(Set.of(orgUnit.getUid())) @@ -1321,8 +1321,8 @@ private List getTrackedEntities(TrackedEntityOperationParams params) } private List getEnrollments(EnrollmentOperationParams params) - throws ForbiddenException, BadRequestException { - return uids(enrollmentService.getEnrollments(params).getEnrollments()); + throws ForbiddenException, BadRequestException, NotFoundException { + return uids(enrollmentService.getEnrollments(params)); } private List getEvents(EventOperationParams params) diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentServiceTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentServiceTest.java index 2d0c89211b88..1ba53d8ceb73 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentServiceTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/enrollment/EnrollmentServiceTest.java @@ -27,6 +27,7 @@ */ package org.hisp.dhis.tracker.export.enrollment; +import static org.hisp.dhis.tracker.TrackerTestUtils.uids; import static org.hisp.dhis.utils.Assertions.assertContains; import static org.hisp.dhis.utils.Assertions.assertContainsOnly; import static org.hisp.dhis.utils.Assertions.assertIsEmpty; @@ -365,7 +366,7 @@ void shouldFailGettingEnrollmentWhenUserHasNoAccessToProgramButAccessToOrgUnit() @Test void shouldGetEnrollmentsWhenUserHasReadAccessToProgramAndSearchScopeAccessToOrgUnit() - throws ForbiddenException, BadRequestException { + throws ForbiddenException, BadRequestException, NotFoundException { programA.getSharing().setPublicAccess(AccessStringHelper.FULL); manager.updateNoAcl(programA); @@ -376,15 +377,50 @@ void shouldGetEnrollmentsWhenUserHasReadAccessToProgramAndSearchScopeAccessToOrg .orgUnitMode(OrganisationUnitSelectionMode.ACCESSIBLE) .build(); - Enrollments enrollments = enrollmentService.getEnrollments(params); + List enrollments = enrollmentService.getEnrollments(params); assertNotNull(enrollments); - assertContainsOnly(List.of(enrollmentA.getUid(), enrollmentB.getUid()), toUid(enrollments)); + assertContainsOnly(List.of(enrollmentA.getUid(), enrollmentB.getUid()), uids(enrollments)); + } + + @Test + void shouldGetEnrollmentsWhenUserHasReadAccessToProgramAndNoOrgUnitNorOrgUnitModeSpecified() + throws ForbiddenException, BadRequestException, NotFoundException { + programA.getSharing().setPublicAccess(AccessStringHelper.FULL); + + manager.updateNoAcl(programA); + + EnrollmentOperationParams params = + EnrollmentOperationParams.builder().programUid(programA.getUid()).build(); + + List enrollments = enrollmentService.getEnrollments(params); + + assertNotNull(enrollments); + assertContainsOnly(List.of(enrollmentA.getUid(), enrollmentB.getUid()), uids(enrollments)); + } + + @Test + void shouldGetEnrollmentWhenEnrollmentsAndOtherParamsAreSpecified() + throws ForbiddenException, BadRequestException, NotFoundException { + programA.getSharing().setPublicAccess(AccessStringHelper.FULL); + + manager.updateNoAcl(programA); + + EnrollmentOperationParams params = + EnrollmentOperationParams.builder() + .programUid(programA.getUid()) + .enrollmentUids(Set.of(enrollmentA.getUid())) + .build(); + + List enrollments = enrollmentService.getEnrollments(params); + + assertNotNull(enrollments); + assertContainsOnly(List.of(enrollmentA.getUid()), uids(enrollments)); } @Test void shouldGetEnrollmentsByTrackedEntityWhenUserHasAccessToTrackedEntityType() - throws ForbiddenException, BadRequestException { + throws ForbiddenException, BadRequestException, NotFoundException { programA.getSharing().setPublicAccess(AccessStringHelper.DATA_READ); manager.updateNoAcl(programA); @@ -394,10 +430,10 @@ void shouldGetEnrollmentsByTrackedEntityWhenUserHasAccessToTrackedEntityType() .trackedEntityUid(trackedEntityA.getUid()) .build(); - Enrollments enrollments = enrollmentService.getEnrollments(params); + List enrollments = enrollmentService.getEnrollments(params); assertNotNull(enrollments); - assertContainsOnly(List.of(enrollmentA.getUid()), toUid(enrollments)); + assertContainsOnly(List.of(enrollmentA.getUid()), uids(enrollments)); } @Test @@ -420,12 +456,6 @@ void shouldFailGettingEnrollmentsByTrackedEntityWhenUserHasNoAccessToTrackedEnti assertContains("access to tracked entity type", exception.getMessage()); } - private static List toUid(Enrollments enrollments) { - return enrollments.getEnrollments().stream() - .map(Enrollment::getUid) - .collect(Collectors.toList()); - } - private static List attributeUids(Enrollment enrollment) { return enrollment.getTrackedEntity().getTrackedEntityAttributeValues().stream() .map(v -> v.getAttribute().getUid()) diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/relationship/RelationshipServiceTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/relationship/RelationshipServiceTest.java index 23e929f7093b..c7878ea472e4 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/relationship/RelationshipServiceTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/relationship/RelationshipServiceTest.java @@ -31,12 +31,14 @@ import static org.hisp.dhis.tracker.TrackerType.EVENT; import static org.hisp.dhis.tracker.TrackerType.TRACKED_ENTITY; import static org.hisp.dhis.utils.Assertions.assertContainsOnly; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import org.hisp.dhis.common.BaseIdentifiableObject; import org.hisp.dhis.common.CodeGenerator; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.commons.util.RelationshipUtils; @@ -59,6 +61,8 @@ import org.hisp.dhis.trackedentity.TrackedEntityType; import org.hisp.dhis.user.User; import org.hisp.dhis.user.UserService; +import org.hisp.dhis.webapi.controller.event.mapper.SortDirection; +import org.joda.time.DateTime; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -277,11 +281,60 @@ void shouldNotReturnRelationshipByEventIfUserHasNoAccessToProgramStage() .collect(Collectors.toList())); } + @Test + void shouldOrderRelationshipsByUpdatedAtClientInDescOrder() + throws ForbiddenException, NotFoundException { + Relationship relationshipA = relationship(teA, teB, DateTime.now().toDate()); + Relationship relationshipB = relationship(teA, eventA, DateTime.now().minusDays(1).toDate()); + + RelationshipOperationParams operationParams = + RelationshipOperationParams.builder() + .type(TRACKED_ENTITY) + .identifier(teA.getUid()) + .orderBy("createdAtClient", SortDirection.DESC) + .build(); + List relationshipIds = + relationshipService.getRelationships(operationParams).getRelationships().stream() + .map(BaseIdentifiableObject::getUid) + .collect(Collectors.toList()); + + assertEquals(List.of(relationshipA.getUid(), relationshipB.getUid()), relationshipIds); + } + + @Test + void shouldOrderRelationshipsByUpdatedAtClientInAscOrder() + throws ForbiddenException, NotFoundException { + Relationship relationshipA = relationship(teA, teB, DateTime.now().toDate()); + Relationship relationshipB = relationship(teA, eventA, DateTime.now().minusDays(1).toDate()); + + RelationshipOperationParams operationParams = + RelationshipOperationParams.builder() + .type(TRACKED_ENTITY) + .identifier(teA.getUid()) + .orderBy("createdAtClient", SortDirection.ASC) + .build(); + List relationshipIds = + relationshipService.getRelationships(operationParams).getRelationships().stream() + .map(BaseIdentifiableObject::getUid) + .collect(Collectors.toList()); + + assertEquals(List.of(relationshipB.getUid(), relationshipA.getUid()), relationshipIds); + } + private Relationship relationship(TrackedEntity from, TrackedEntity to) { - return relationship(from, to, teToTeType); + return relationship(from, to, teToTeType, new Date()); + } + + private Relationship relationship(TrackedEntity from, TrackedEntity to, Date createdAtClient) { + return relationship(from, to, teToTeType, createdAtClient); } private Relationship relationship(TrackedEntity from, TrackedEntity to, RelationshipType type) { + return relationship(from, to, type, new Date()); + } + + private Relationship relationship( + TrackedEntity from, TrackedEntity to, RelationshipType type, Date createdAtClient) { Relationship relationship = new Relationship(); relationship.setUid(CodeGenerator.generateUid()); relationship.setRelationshipType(type); @@ -289,7 +342,7 @@ private Relationship relationship(TrackedEntity from, TrackedEntity to, Relation relationship.setTo(item(to)); relationship.setKey(RelationshipUtils.generateRelationshipKey(relationship)); relationship.setInvertedKey(RelationshipUtils.generateRelationshipInvertedKey(relationship)); - + relationship.setCreatedAtClient(createdAtClient); manager.save(relationship); return relationship; @@ -313,11 +366,16 @@ private Relationship relationship(TrackedEntity from, Enrollment to, Relationshi return relationship; } + private Relationship relationship(TrackedEntity from, Event to, Date createdAtClient) { + return relationship(from, to, teToEvType, createdAtClient); + } + private Relationship relationship(TrackedEntity from, Event to) { - return relationship(from, to, teToEvType); + return relationship(from, to, teToEvType, new Date()); } - private Relationship relationship(TrackedEntity from, Event to, RelationshipType type) { + private Relationship relationship( + TrackedEntity from, Event to, RelationshipType type, Date createdAtClient) { Relationship relationship = new Relationship(); relationship.setUid(CodeGenerator.generateUid()); relationship.setRelationshipType(type); @@ -325,6 +383,7 @@ private Relationship relationship(TrackedEntity from, Event to, RelationshipType relationship.setTo(item(to)); relationship.setKey(RelationshipUtils.generateRelationshipKey(relationship)); relationship.setInvertedKey(RelationshipUtils.generateRelationshipInvertedKey(relationship)); + relationship.setCreatedAtClient(createdAtClient); manager.save(relationship); diff --git a/dhis-2/dhis-test-integration/src/test/resources/analytics/csv/dataValues.csv b/dhis-2/dhis-test-integration/src/test/resources/analytics/csv/dataValues.csv index 917e058ba872..b72c96f54e0b 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/analytics/csv/dataValues.csv +++ b/dhis-2/dhis-test-integration/src/test/resources/analytics/csv/dataValues.csv @@ -27,5 +27,7 @@ "deabcdefghE", "2017-06", "ouabcdefghA", "2" "deabcdefghE", "2017-07", "ouabcdefghA", "4" "deabcdefghF", "2017-01", "ouabcdefghA", "true" +"deabcdefghF", "2017-02", "ouabcdefghA", "false" +"deabcdefghF", "2017-03", "ouabcdefghA", "true" "deabcdefghG", "2017-01", "ouabcdefghA", "abc" "deabcdefghH", "2017-01", "ouabcdefghA", "2017-01-01" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/enrollment_data_with_program_rule_side_effects.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/enrollment_data_with_program_rule_side_effects.json index 665bb4e3bf04..d7a5e8458eae 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/enrollment_data_with_program_rule_side_effects.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/enrollment_data_with_program_rule_side_effects.json @@ -90,7 +90,6 @@ }, "enrolledAt": "2020-02-20T12:09:21.844", "occurredAt": "2020-02-20T12:09:21.844", - "followup": false, "deleted": false } ] diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_and_enrollment.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_and_enrollment.json index c226178483b5..31fe9146c4a4 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_and_enrollment.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_and_enrollment.json @@ -395,7 +395,7 @@ "occurredAt": "2019-01-25T12:10:38.100", "scheduledAt": "2019-01-28T12:32:38.100", "storedBy": "admin", - "followup": false, + "followUp": true, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -474,7 +474,7 @@ "executionDate": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, + "followUp": true, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -557,7 +557,7 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, + "followUp": true, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -592,7 +592,7 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, + "followUp": true, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -887,7 +887,7 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, + "followUp": true, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_and_enrollment_with_data_values.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_and_enrollment_with_data_values.json index b45a879c410d..e7b1f8e69ae8 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_and_enrollment_with_data_values.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_and_enrollment_with_data_values.json @@ -135,7 +135,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -190,7 +189,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_basic_data_for_deletion.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_basic_data_for_deletion.json index dfebeb619f59..2a7c4213c304 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_basic_data_for_deletion.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_basic_data_for_deletion.json @@ -53,7 +53,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_events.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_events.json index 8f90efc2b3f5..15b5ac2646e9 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_events.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_events.json @@ -52,7 +52,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -102,7 +101,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -148,7 +146,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -194,7 +191,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -240,7 +236,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -286,7 +281,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -372,7 +366,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -459,7 +452,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_events_and_enrollment.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_events_and_enrollment.json index 3e99b14f429a..42b1ddeb8e99 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_events_and_enrollment.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_events_and_enrollment.json @@ -135,7 +135,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -170,7 +169,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -205,7 +203,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -240,7 +237,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -275,7 +271,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -310,7 +305,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -345,7 +339,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -380,7 +373,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_update_no_datavalue.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_update_no_datavalue.json index 44dae0fac424..3a1032ffa653 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_update_no_datavalue.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_update_no_datavalue.json @@ -53,7 +53,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_with_data_values.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_with_data_values.json index 2fd225c46ad5..0be2a9e2d688 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_with_data_values.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_with_data_values.json @@ -53,7 +53,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_with_data_values_for_delete_audit.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_with_data_values_for_delete_audit.json index b3b657510b36..1031568e6247 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_with_data_values_for_delete_audit.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_with_data_values_for_delete_audit.json @@ -76,7 +76,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_with_data_values_for_update_audit.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_with_data_values_for_update_audit.json index 0b1a7648bc6c..42abeb8cd6e2 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_with_data_values_for_update_audit.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_with_data_values_for_update_audit.json @@ -76,7 +76,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_with_updated_data_values.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_with_updated_data_values.json index a9bdc662453b..67457fc25dbd 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/event_with_updated_data_values.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/event_with_updated_data_values.json @@ -53,7 +53,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/one_update_and_one_new_and_one_invalid_event.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/one_update_and_one_new_and_one_invalid_event.json index 3ff68f33cad9..159ff551b9cf 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/one_update_and_one_new_and_one_invalid_event.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/one_update_and_one_new_and_one_invalid_event.json @@ -53,7 +53,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" @@ -83,7 +82,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" @@ -113,7 +111,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/one_update_event_and_one_new_event.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/one_update_event_and_one_new_event.json index 58f8c6cd0bf1..31edb15d60e4 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/one_update_event_and_one_new_event.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/one_update_event_and_one_new_event.json @@ -53,7 +53,7 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, + "deleted": false, "attributeOptionCombo": { "idScheme": "UID" @@ -83,7 +83,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/ownership_event.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/ownership_event.json index e96c892dff6e..a8d36ca7b00a 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/ownership_event.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/ownership_event.json @@ -53,7 +53,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "createdAtClient": "2019-01-28T11:10:38.108", "updatedAtClient": "2019-01-28T10:10:38.109", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/program_event.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/program_event.json index a4863ddf6256..36be9d83d054 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/program_event.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/program_event.json @@ -52,7 +52,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/completed_event.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/completed_event.json index 9159df1e6a9d..139280c2c250 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/completed_event.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/completed_event.json @@ -53,7 +53,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "createdAtClient": "2019-01-28T11:10:38.108", "updatedAtClient": "2019-01-28T10:10:38.109", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/event.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/event.json index 097c17d1df55..95c56abc26f9 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/event.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/event.json @@ -53,7 +53,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "createdAtClient": "2019-01-28T11:10:38.108", "updatedAtClient": "2019-01-28T10:10:38.109", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/event_update_datavalue_different_value.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/event_update_datavalue_different_value.json index 3d1f5e51ba98..68b000c2374a 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/event_update_datavalue_different_value.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/event_update_datavalue_different_value.json @@ -53,7 +53,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/event_update_datavalue_empty_value.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/event_update_datavalue_empty_value.json index 483f2fdc9ef0..99690c0e6f9a 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/event_update_datavalue_empty_value.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/event_update_datavalue_empty_value.json @@ -53,7 +53,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/event_update_datavalue_same_value.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/event_update_datavalue_same_value.json index 1945da3e2094..cf75b08ce14a 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/event_update_datavalue_same_value.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/event_update_datavalue_same_value.json @@ -53,7 +53,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/program_event.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/program_event.json index 2f15d914cec8..96b1ca05cde1 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/program_event.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/program_event.json @@ -52,7 +52,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_completed_enrollment_event.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_completed_enrollment_event.json index 6b761ff3e874..24d03f7eeb59 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_completed_enrollment_event.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_completed_enrollment_event.json @@ -95,7 +95,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_completed_event.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_completed_event.json index e1d8d05e6de0..8967e0faca39 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_completed_event.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_completed_event.json @@ -95,7 +95,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_completed_event_from_another_program_stage.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_completed_event_from_another_program_stage.json index ff2b1c4aba7a..a3b3883d7b09 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_completed_event_from_another_program_stage.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_completed_event_from_another_program_stage.json @@ -95,7 +95,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_event_from_another_program_stage.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_event_from_another_program_stage.json index b298a7f44fdc..f2bb59fbab60 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_event_from_another_program_stage.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_event_from_another_program_stage.json @@ -95,7 +95,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_event_programevent.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_event_programevent.json index 31674f54c626..f4b23716174a 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_event_programevent.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_event_programevent.json @@ -95,7 +95,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" @@ -135,7 +134,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_event_with_no_data_value.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_event_with_no_data_value.json index 7cde3cff5abf..c107792a6daa 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_event_with_no_data_value.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_event_with_no_data_value.json @@ -95,7 +95,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_event_with_null_data_value.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_event_with_null_data_value.json index a3235227fd4a..b6ee534c1479 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_event_with_null_data_value.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/tei_enrollment_event_with_null_data_value.json @@ -95,7 +95,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/single_event.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/single_event.json index 9159df1e6a9d..139280c2c250 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/single_event.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/single_event.json @@ -53,7 +53,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "createdAtClient": "2019-01-28T11:10:38.108", "updatedAtClient": "2019-01-28T10:10:38.109", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/tei_enrollment_event.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/tei_enrollment_event.json index e1d8d05e6de0..8967e0faca39 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/tei_enrollment_event.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/tei_enrollment_event.json @@ -95,7 +95,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/tracker_basic_data_before_deletion.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/tracker_basic_data_before_deletion.json index d11d644b0c46..6eaf2061e11a 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/tracker_basic_data_before_deletion.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/tracker_basic_data_before_deletion.json @@ -339,7 +339,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" @@ -374,7 +373,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/event-data-delete.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/event-data-delete.json index f355e8499e4c..639fd2e534d1 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/event-data-delete.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/event-data-delete.json @@ -43,7 +43,6 @@ "idScheme": "UID" }, "relationships": [], - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-aoc-and-co-dont-match.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-aoc-and-co-dont-match.json index b5f045130c1e..ea6637d1b7e7 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-aoc-and-co-dont-match.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-aoc-and-co-dont-match.json @@ -51,7 +51,6 @@ "relationships": [], "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-aoc-not-in-program-cc.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-aoc-not-in-program-cc.json index 324ab6a842ba..27ab71793e3b 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-aoc-not-in-program-cc.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-aoc-not-in-program-cc.json @@ -51,7 +51,6 @@ "relationships": [], "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-cat-write-access.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-cat-write-access.json index b9020510ad7e..254eda322691 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-cat-write-access.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-cat-write-access.json @@ -54,7 +54,6 @@ "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with-notes-data.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with-notes-data.json index 1113da76fb0d..f115ee1d73a3 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with-notes-data.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with-notes-data.json @@ -53,7 +53,6 @@ "relationships": [], "occurredAt": "2020-07-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with-notes-update-data.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with-notes-update-data.json index 6fd8ef9ba8dd..c49569e3d7de 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with-notes-update-data.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with-notes-update-data.json @@ -53,7 +53,6 @@ "relationships": [], "occurredAt": "2020-07-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with-registration.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with-registration.json index 571236b80dc7..16a625d41a73 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with-registration.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with-registration.json @@ -54,7 +54,6 @@ "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with_invalid_option_value.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with_invalid_option_value.json index 866a24b2f720..faed9cedb811 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with_invalid_option_value.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with_invalid_option_value.json @@ -54,7 +54,6 @@ "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with_no_program.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with_no_program.json index 7cbb1750e5d2..b3a0bb06f105 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with_no_program.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with_no_program.json @@ -52,7 +52,6 @@ "occurredAt": "2019-01-28T00:00:00.000", "scheduledAt": "2019-01-28T12:10:38.100", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with_valid_option_value.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with_valid_option_value.json index 8428874e3942..f8aa967962ee 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with_valid_option_value.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-with_valid_option_value.json @@ -54,7 +54,6 @@ "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-without-attribute-option-combo.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-without-attribute-option-combo.json index 901c9393f6d9..87a7e3dc109c 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-without-attribute-option-combo.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-without-attribute-option-combo.json @@ -50,7 +50,6 @@ "relationships": [], "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-aoc-but-co-exists.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-aoc-but-co-exists.json index 3628aaef235d..37d20175e875 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-aoc-but-co-exists.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-aoc-but-co-exists.json @@ -53,7 +53,6 @@ "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-aoc-with-subset-of-cos.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-aoc-with-subset-of-cos.json index 578f6134e562..e88b7ef870fa 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-aoc-with-subset-of-cos.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-aoc-with-subset-of-cos.json @@ -51,7 +51,6 @@ "relationships": [], "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-cat-opt-combo.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-cat-opt-combo.json index 5d8302f44787..41c918eca302 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-cat-opt-combo.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-cat-opt-combo.json @@ -52,7 +52,6 @@ "relationships": [], "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-cat-option-combo-for-given-cc-and-co.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-cat-option-combo-for-given-cc-and-co.json index ecc458aef072..676db577c5a0 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-cat-option-combo-for-given-cc-and-co.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-cat-option-combo-for-given-cc-and-co.json @@ -52,7 +52,6 @@ "relationships": [], "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-cat-option.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-cat-option.json index 432acef2e4b7..57547a3d3b90 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-cat-option.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_cant-find-cat-option.json @@ -54,7 +54,6 @@ "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID" diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_combo-date-wrong.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_combo-date-wrong.json index d084bf4d264f..662775658988 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_combo-date-wrong.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_combo-date-wrong.json @@ -50,7 +50,6 @@ }, "relationships": [], "occurredAt": "2018-12-31T06:13:07.000", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -77,7 +76,6 @@ "relationships": [], "occurredAt": "2020-08-01T00:00:00.000", "scheduledAt": "2020-08-19T13:59:13.688", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_error-no-programStage-access.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_error-no-programStage-access.json index 19f2e29cd487..161d8763071e 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_error-no-programStage-access.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_error-no-programStage-access.json @@ -54,7 +54,6 @@ "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_error-no-uncomplete.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_error-no-uncomplete.json index 19f2e29cd487..161d8763071e 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_error-no-uncomplete.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_error-no-uncomplete.json @@ -54,7 +54,6 @@ "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_non-default-combo.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_non-default-combo.json index e31f32f59817..5ab3eef05e58 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_non-default-combo.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_non-default-combo.json @@ -54,7 +54,6 @@ "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_non-repeatable-programstage_part1.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_non-repeatable-programstage_part1.json index 5154799ffa12..fe761c1bf12d 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_non-repeatable-programstage_part1.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_non-repeatable-programstage_part1.json @@ -54,7 +54,6 @@ "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_non-repeatable-programstage_part2.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_non-repeatable-programstage_part2.json index 3b2118f79390..cf16a4ed74fa 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_non-repeatable-programstage_part2.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events_non-repeatable-programstage_part2.json @@ -54,7 +54,6 @@ "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/invalid_enrollment_with_valid_event.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/invalid_enrollment_with_valid_event.json index 34e25a76e37e..389bd56fb134 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/invalid_enrollment_with_valid_event.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/invalid_enrollment_with_valid_event.json @@ -77,7 +77,6 @@ "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/program_and_tracker_events.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/program_and_tracker_events.json index 34428ad73c9b..96e6aaaf80b4 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/program_and_tracker_events.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/program_and_tracker_events.json @@ -54,7 +54,6 @@ "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", @@ -82,7 +81,6 @@ "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/program_events_non-repeatable-programstage_part1.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/program_events_non-repeatable-programstage_part1.json index 748042e147ab..07afd8e203c4 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/program_events_non-repeatable-programstage_part1.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/program_events_non-repeatable-programstage_part1.json @@ -54,7 +54,6 @@ "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": {}, "attributeCategoryOptions": [], diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/program_events_non-repeatable-programstage_part2.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/program_events_non-repeatable-programstage_part2.json index dbe1a88d992e..1ed67ef61838 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/program_events_non-repeatable-programstage_part2.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/program_events_non-repeatable-programstage_part2.json @@ -54,7 +54,6 @@ "occurredAt": "2019-08-01T00:00:00.000", "scheduledAt": "2019-08-19T13:59:13.688", "storedBy": "admin", - "followup": false, "deleted": false, "attributeOptionCombo": { "idScheme": "UID", diff --git a/dhis-2/dhis-test-web-api/pom.xml b/dhis-2/dhis-test-web-api/pom.xml index 78fbb03d8b9f..162cc2ef550f 100644 --- a/dhis-2/dhis-test-web-api/pom.xml +++ b/dhis-2/dhis-test-web-api/pom.xml @@ -37,7 +37,7 @@ org.hisp.dhis dhis-service-core - compile + test org.hisp.dhis @@ -336,6 +336,11 @@ log4j-core test + + org.springframework + spring-jdbc + test + io.swagger.parser.v3 swagger-parser diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/WebTestConfiguration.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/WebTestConfiguration.java index 6d92dc855917..9ee325b6e56d 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/WebTestConfiguration.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/WebTestConfiguration.java @@ -35,6 +35,7 @@ import org.hisp.dhis.association.jdbc.JdbcOrgUnitAssociationStoreConfiguration; import org.hisp.dhis.commons.jackson.config.JacksonObjectMapperConfig; import org.hisp.dhis.commons.util.DebugUtils; +import org.hisp.dhis.config.AnalyticsDataSourceConfig; import org.hisp.dhis.config.DataSourceConfig; import org.hisp.dhis.config.HibernateConfig; import org.hisp.dhis.config.HibernateEncryptionConfig; @@ -54,6 +55,7 @@ import org.hisp.dhis.security.SystemAuthoritiesProvider; import org.hisp.dhis.webapi.mvc.ContentNegotiationConfig; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.ComponentScan.Filter; @@ -63,6 +65,7 @@ import org.springframework.context.annotation.ImportResource; import org.springframework.context.annotation.Primary; import org.springframework.core.annotation.Order; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.security.authentication.AuthenticationManager; import org.springframework.security.authentication.DefaultAuthenticationEventPublisher; import org.springframework.security.authentication.event.AuthenticationFailureBadCredentialsEvent; @@ -93,6 +96,7 @@ @Import({ HibernateConfig.class, DataSourceConfig.class, + AnalyticsDataSourceConfig.class, JdbcConfig.class, FlywayConfig.class, HibernateEncryptionConfig.class, @@ -125,7 +129,14 @@ public static SessionRegistryImpl sessionRegistry() { @Autowired private DhisConfigurationProvider dhisConfigurationProvider; - @Bean("dataSource") + @Bean(name = {"namedParameterJdbcTemplate", "analyticsNamedParameterJdbcTemplate"}) + @Primary + public NamedParameterJdbcTemplate namedParameterJdbcTemplate( + @Qualifier("dataSource") DataSource dataSource) { + return new NamedParameterJdbcTemplate(dataSource); + } + + @Bean(name = {"dataSource", "analyticsDataSource"}) @Primary public DataSource actualDataSource( HibernateConfigurationProvider hibernateConfigurationProvider) { diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/EventVisualizationControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/EventVisualizationControllerTest.java index 491237d31a3d..2a005f6c65ee 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/EventVisualizationControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/EventVisualizationControllerTest.java @@ -32,6 +32,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hisp.dhis.dataelement.DataElementDomain.TRACKER; import static org.hisp.dhis.web.HttpStatus.BAD_REQUEST; import static org.hisp.dhis.web.HttpStatus.CONFLICT; import static org.hisp.dhis.web.HttpStatus.CREATED; @@ -41,11 +42,15 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.jsontree.JsonObject; +import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramIndicator; import org.hisp.dhis.program.ProgramStage; +import org.hisp.dhis.trackedentity.TrackedEntityType; import org.hisp.dhis.webapi.DhisControllerConvenienceTest; +import org.hisp.dhis.webapi.json.domain.JsonError; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -64,9 +69,16 @@ class EventVisualizationControllerTest extends DhisControllerConvenienceTest { private ProgramIndicator mockProgramIndicator; + private DataElement mockDataElement; + + private OrganisationUnit mockOrganisationUnit; + + private TrackedEntityType mockTrackerEntityType; + @BeforeEach public void beforeEach() throws JsonProcessingException { mockProgram = createProgram('A'); + mockProgram.setUid("deabcdefghP"); POST("/programs", jsonMapper.writeValueAsString(mockProgram)).content(CREATED); mockProgramIndicator = createProgramIndicator('A', mockProgram, "exp", "filter"); @@ -74,9 +86,29 @@ public void beforeEach() throws JsonProcessingException { POST("/programIndicators", jsonMapper.writeValueAsString(mockProgramIndicator)) .content(CREATED); + mockDataElement = createDataElement('A'); + mockDataElement.setUid("deabcdefghC"); + mockDataElement.setDomainType(TRACKER); + POST("/dataElements", jsonMapper.writeValueAsString(mockDataElement)).content(CREATED); + + mockDataElement = createDataElement('B'); + mockDataElement.setUid("deabcdefghE"); + mockDataElement.setDomainType(TRACKER); + POST("/dataElements", jsonMapper.writeValueAsString(mockDataElement)).content(CREATED); + mockProgramStage = createProgramStage('A', mockProgram); - mockProgramStage.setUid("deabcdefghA"); + mockProgramStage.setUid("deabcdefghS"); POST("/programStages", jsonMapper.writeValueAsString(mockProgramStage)).content(CREATED); + + mockOrganisationUnit = createOrganisationUnit('A'); + mockOrganisationUnit.setUid("ImspTQPwCqd"); + POST("/organisationUnits", jsonMapper.writeValueAsString(mockOrganisationUnit)) + .content(CREATED); + + mockTrackerEntityType = createTrackedEntityType('A'); + mockTrackerEntityType.setUid("nEenWmSyUEp"); + POST("/trackedEntityTypes", jsonMapper.writeValueAsString(mockTrackerEntityType)) + .content(CREATED); } @Test @@ -536,4 +568,212 @@ void testPost() { assertThat(response.get("program").node().get("id").value(), is(equalTo(mockProgram.getUid()))); assertThat(response.get("skipRounding").node().value(), is(equalTo(true))); } + + @Test + void testPostMultiPrograms() { + // Given + String body = + """ + {"name": "Test multi-programs post", "type": "LINE_LIST", + "program": {"id": "deabcdefghP"}, + "trackedEntityType": {"id": "nEenWmSyUEp"}, + "sorting": [ + { + "dimension": "deabcdefghP[-1].deabcdefghS[0].deabcdefghB", + "direction": "ASC" + }, + { + "dimension": "deabcdefghP.deabcdefghS.deabcdefghB", + "direction": "DESC" + } + ], + "columns": [ + { + "dimension": "deabcdefghB", + "programStage": { + "id": "deabcdefghS" + }, + "program": { + "id": "deabcdefghP" + } + }, + { + "dimension": "deabcdefghC", + "filter": "IN:Female" + }, + { + "dimension": "eventDate", + "program": { + "id": "deabcdefghP" + }, + "items": [ + { + "id": "2023-07-21_2023-08-01" + }, + { + "id": "2023-01-21_2023-02-01" + } + ] + } + ], + "filters": [ + { + "dimension": "ou", + "programStage": { + "id": "deabcdefghS" + }, + "repetition": { + "indexes": [ + 1, + 2, + 3, + -2, + -1, + 0 + ] + }, + "items": [ + { + "id": "ImspTQPwCqd" + } + ] + }, + { + "dimension": "deabcdefghE", + "repetition": { + "indexes": [ + 1, + 2, + 0 + ] + }, + "items": [] + } + ] + }"""; + + // When + String uid = assertStatus(CREATED, POST("/eventVisualizations/", body)); + + // Then + JsonObject response = + GET("/eventVisualizations/" + + uid + + "?fields=*,sorting,filters[:all,items[code, name, sharing, shortName, dimensionItemType, dimensionItem, displayShortName, displayName, displayFormName, id],repetitions],columns[:all,items[:all],repetitions]") + .content(); + + assertThat(response.get("name").node().value(), is(equalTo("Test multi-programs post"))); + assertThat(response.get("type").node().value(), is(equalTo("LINE_LIST"))); + assertThat(response.get("skipRounding").node().value(), is(equalTo(false))); + assertThat(response.get("legacy").node().value(), is(equalTo(false))); + assertThat( + response.get("trackedEntityType").node().value().toString(), + is(equalTo(""" +{"id":"nEenWmSyUEp"}"""))); + + assertThat( + response.get("simpleDimensions").node().value().toString(), + is( + equalTo( + """ +[{"parent":"COLUMN","dimension":"eventDate","program":"deabcdefghP","values":["2023-07-21_2023-08-01","2023-01-21_2023-02-01"]}]"""))); + + assertThat( + response.get("sorting").node().value().toString(), + is( + equalTo( + """ +[{"dimension":"deabcdefghP[-1].deabcdefghS[0].deabcdefghB","direction":"ASC"},{"dimension":"deabcdefghP.deabcdefghS.deabcdefghB","direction":"DESC"}]"""))); + + assertThat(response.get("rows").node().value().toString(), is(equalTo("[]"))); + assertThat(response.get("rowDimensions").node().value().toString(), is(equalTo("[]"))); + + assertThat( + response.get("columnDimensions").node().value().toString(), + is( + equalTo( + """ +["deabcdefghP.deabcdefghS.deabcdefghB","deabcdefghC","deabcdefghP.eventDate"]"""))); + + assertThat( + response.get("filterDimensions").node().value().toString(), + is(equalTo(""" +["deabcdefghP.deabcdefghS.ou","deabcdefghE"]"""))); + + assertThat( + response.get("dataElementDimensions").node().value().toString(), + is( + equalTo( + """ +[{"dataElement":{"id":"deabcdefghC"},"filter":"IN:Female"},{"dataElement":{"id":"deabcdefghE"}}]"""))); + + assertThat( + response.get("programIndicatorDimensions").node().value().toString(), + is(equalTo(""" +[{"programIndicator":{"id":"deabcdefghB"}}]"""))); + + assertThat( + response.get("organisationUnits").node().value().toString(), + is(equalTo(""" +[{"id":"ImspTQPwCqd"}]"""))); + + assertThat( + response.get("repetitions").node().value().toString(), + is( + equalTo( + """ +[{"parent":"FILTER","dimension":"ou","program":"deabcdefghP","programStage":"deabcdefghS","indexes":[1,2,3,-2,-1,0]},{"parent":"FILTER","dimension":"deabcdefghE","indexes":[1,2,0]}]"""))); + + assertThat( + response.get("columns").node().value().toString(), + is( + equalTo( + """ +[{"translations":[],"favorites":[],"sharing":{"external":false,"users":{},"userGroups":{}},"dimensionType":"PROGRAM_INDICATOR","dataDimension":true,"items":[],"allItems":false,"program":{"id":"deabcdefghP"},"dimension":"deabcdefghB","access":{"manage":true,"externalize":true,"write":true,"read":true,"update":true,"delete":true},"favorite":false,"id":"deabcdefghB","attributeValues":[]},{"translations":[],"favorites":[],"sharing":{"external":false,"users":{},"userGroups":{}},"dimensionType":"PROGRAM_DATA_ELEMENT","dataDimension":true,"items":[],"allItems":false,"filter":"IN:Female","dimension":"deabcdefghC","valueType":"INTEGER","access":{"manage":true,"externalize":true,"write":true,"read":true,"update":true,"delete":true},"favorite":false,"id":"deabcdefghC","attributeValues":[]},{"translations":[],"favorites":[],"sharing":{"external":false,"users":{},"userGroups":{}},"dimensionType":"PERIOD","dataDimension":true,"items":[{"code":"2023-07-21_2023-08-01","name":"2023-07-21_2023-08-01","translations":[],"favorites":[],"sharing":{"external":false,"users":{},"userGroups":{}},"legendSets":[],"dimensionItem":"2023-07-21_2023-08-01","access":{"manage":true,"externalize":true,"write":true,"read":true,"update":true,"delete":true},"displayName":"2023-07-21_2023-08-01","favorite":false,"displayFormName":"2023-07-21_2023-08-01","id":"2023-07-21_2023-08-01","attributeValues":[]},{"code":"2023-01-21_2023-02-01","name":"2023-01-21_2023-02-01","translations":[],"favorites":[],"sharing":{"external":false,"users":{},"userGroups":{}},"legendSets":[],"dimensionItem":"2023-01-21_2023-02-01","access":{"manage":true,"externalize":true,"write":true,"read":true,"update":true,"delete":true},"displayName":"2023-01-21_2023-02-01","favorite":false,"displayFormName":"2023-01-21_2023-02-01","id":"2023-01-21_2023-02-01","attributeValues":[]}],"allItems":false,"program":{"id":"deabcdefghP"},"dimension":"eventDate","access":{"manage":true,"externalize":true,"write":true,"read":true,"update":true,"delete":true},"favorite":false,"id":"eventDate","attributeValues":[]}]"""))); + + assertThat( + response.get("filters").node().value().toString(), + is( + equalTo( + """ +[{"translations":[],"favorites":[],"sharing":{"external":false,"users":{},"userGroups":{}},"dimensionType":"ORGANISATION_UNIT","dataDimension":true,"items":[{"code":"OrganisationUnitCodeA","name":"OrganisationUnitA","sharing":{"external":false,"users":{},"userGroups":{}},"shortName":"OrganisationUnitShortA","dimensionItemType":"ORGANISATION_UNIT","dimensionItem":"ImspTQPwCqd","displayShortName":"OrganisationUnitShortA","displayName":"OrganisationUnitA","displayFormName":"OrganisationUnitA","id":"ImspTQPwCqd"}],"allItems":false,"programStage":{"id":"deabcdefghS"},"program":{"id":"deabcdefghP"},"dimension":"ou","access":{"manage":true,"externalize":true,"write":true,"read":true,"update":true,"delete":true},"favorite":false,"id":"ou","attributeValues":[],"repetition":{"parent":"FILTER","dimension":"ou","program":"deabcdefghP","programStage":"deabcdefghS","indexes":[1,2,3,-2,-1,0]}},{"translations":[],"favorites":[],"sharing":{"external":false,"users":{},"userGroups":{}},"dimensionType":"PROGRAM_DATA_ELEMENT","dataDimension":true,"items":[],"allItems":false,"dimension":"deabcdefghE","valueType":"INTEGER","access":{"manage":true,"externalize":true,"write":true,"read":true,"update":true,"delete":true},"favorite":false,"id":"deabcdefghE","attributeValues":[],"repetition":{"parent":"FILTER","dimension":"deabcdefghE","indexes":[1,2,0]}}]"""))); + } + + @Test + void testGetDataForNonAllowedType() { + // Given + String body = + """ + {"name": "Test multi-programs post", "type": "LINE_LIST", + "program": {"id": "deabcdefghP"} + }"""; + + // When + String uid = assertStatus(CREATED, POST("/eventVisualizations/", body)); + + // Then + JsonError error = GET("/eventVisualizations/" + uid + "/data").error(CONFLICT); + assertEquals("ERROR", error.getStatus()); + assertEquals("Cannot generate chart for LINE_LIST", error.getMessage()); + } + + @Test + void testGetDataForMultiProgram() { + // Given + String body = + """ + {"name": "Test multi-programs post", "type": "STACKED_COLUMN", + "program": {"id": "deabcdefghP"}, + "trackedEntityType": {"id": "nEenWmSyUEp"} + }"""; + + // When + String uid = assertStatus(CREATED, POST("/eventVisualizations/", body)); + + // Then + JsonError error = GET("/eventVisualizations/" + uid + "/data").error(CONFLICT); + assertEquals("ERROR", error.getStatus()); + assertEquals( + "Cannot generate chart for multi-program visualization " + uid, error.getMessage()); + } } diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SystemSettingControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SystemSettingControllerTest.java index a7cdf1a6da7f..9865cd9e5670 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SystemSettingControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SystemSettingControllerTest.java @@ -28,6 +28,7 @@ package org.hisp.dhis.webapi.controller; import static java.util.Arrays.stream; +import static org.hisp.dhis.web.WebClient.Accept; import static org.hisp.dhis.web.WebClient.Body; import static org.hisp.dhis.web.WebClient.ContentType; import static org.hisp.dhis.web.WebClientUtils.assertStatus; @@ -163,4 +164,55 @@ void testGetSystemSettingsJson_AllKeys() { .filter(SettingKey::isConfidential) .forEach(key -> assertFalse(setting.get(key.getName()).exists(), key.getName())); } + + @Test + void testGetSystemSettingAsText_KeyExists() { + assertEquals( + "yyyy-MM-dd", + GET("/systemSettings/keyDateFormat", Accept("text/plain")).content("text/plain")); + } + + @Test + void testGetSystemSettingAsJson_KeyDoesNotExist() { + assertWebMessage( + "Not Found", + 404, + "ERROR", + "Setting does not exist or is marked as confidential", + GET("/systemSettings/keyDoesNotExist").content(HttpStatus.NOT_FOUND)); + } + + @Test + void testGetSystemSettingAsText_KeyDoesNotExist() { + HttpResponse response = GET("/systemSettings/keyDoesNotExist", Accept("text/plain")); + assertEquals(HttpStatus.NOT_FOUND, response.status()); + assertEquals( + "Setting does not exist or is marked as confidential", response.content("text/plain")); + } + + @Test + void testGetSystemSettingAsJsonQueryParam_KeyDoesNotExist() { + assertWebMessage( + "Not Found", + 404, + "ERROR", + "Setting does not exist or is marked as confidential", + GET("/systemSettings?key=keyDoesNotExist").content(HttpStatus.NOT_FOUND)); + } + + @Test + void testGetSystemSettingAsJsonQueryParam_MultipleKeysDoExist() { + JsonObject content = + GET("/systemSettings?key=keyDateFormat,jobsRescheduleAfterMinutes").content(HttpStatus.OK); + assertEquals( + "{\"keyDateFormat\":\"yyyy-MM-dd\",\"jobsRescheduleAfterMinutes\":10}", content.toString()); + } + + @Test + void testGetSystemSettingAsJsonQueryParam_OneKeyExistsFromTwo() { + JsonObject content = + GET("/systemSettings?key=keyDoesNotExist,jobsRescheduleAfterMinutes") + .content(HttpStatus.OK); + assertEquals("{\"jobsRescheduleAfterMinutes\":10}", content.toString()); + } } diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/JsonAssertions.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/JsonAssertions.java index f0ee7a9ab446..1d1c01c44d97 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/JsonAssertions.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/JsonAssertions.java @@ -45,6 +45,7 @@ import org.hisp.dhis.relationship.Relationship; import org.hisp.dhis.trackedentity.TrackedEntity; import org.hisp.dhis.tracker.TrackerType; +import org.hisp.dhis.util.DateUtils; public class JsonAssertions { @@ -67,6 +68,10 @@ public static JsonRelationship assertNthRelationship( public static void assertRelationship(Relationship expected, JsonRelationship actual) { assertFalse(actual.isEmpty(), "relationship should not be empty"); assertEquals(expected.getUid(), actual.getRelationship(), "relationship UID"); + assertEquals( + DateUtils.getIso8601NoTz(expected.getCreatedAtClient()), + actual.getCreatedAtClient(), + "createdAtClient date"); assertEquals( expected.getRelationshipType().getUid(), actual.getRelationshipType(), @@ -92,6 +97,8 @@ public static void assertEventWithinRelationshipItem( expected.getEnrollment().getUid(), jsonEvent.getEnrollment(), "event enrollment UID"); assertFalse( jsonEvent.has("relationships"), "relationships is not returned within relationship items"); + assertEquals(expected.getEnrollment().getFollowup(), jsonEvent.getFollowUp(), "followUp"); + assertEquals(expected.getEnrollment().getFollowup(), jsonEvent.getLegacyFollowUp(), "followup"); } public static void assertTrackedEntityWithinRelationshipItem( @@ -126,6 +133,7 @@ public static void assertEnrollmentWithinRelationship( jsonEnrollment.getTrackedEntity(), "trackedEntity UID"); assertEquals(expected.getProgram().getUid(), jsonEnrollment.getProgram(), "program UID"); + assertEquals(expected.getFollowup(), jsonEnrollment.getFollowUp(), "followUp"); assertEquals( expected.getOrganisationUnit().getUid(), jsonEnrollment.getOrgUnit(), "orgUnit UID"); assertTrue(jsonEnrollment.getArray("events").isEmpty(), "events should be empty"); diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/JsonRelationship.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/JsonRelationship.java index 62e78a86cb52..0e069617f0dc 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/JsonRelationship.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/JsonRelationship.java @@ -43,6 +43,10 @@ default String getRelationshipType() { return getString("relationshipType").string(); } + default String getCreatedAtClient() { + return getString("createdAtClient").string(); + } + default JsonRelationshipItem getFrom() { return get("from").as(JsonRelationshipItem.class); } diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/JsonRelationshipItem.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/JsonRelationshipItem.java index cab66d62b8d9..bb0ced94a2ca 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/JsonRelationshipItem.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/JsonRelationshipItem.java @@ -98,6 +98,10 @@ default JsonList getAttributes() { default JsonList getNotes() { return get("notes").asList(JsonNote.class); } + + default Boolean getFollowUp() { + return getBoolean("followUp").bool(); + } } interface JsonEvent extends JsonObject { @@ -128,5 +132,13 @@ default JsonList getDataValues() { default JsonList getNotes() { return get("notes").asList(JsonNote.class); } + + default Boolean getFollowUp() { + return getBoolean("followUp").bool(); + } + + default Boolean getLegacyFollowUp() { + return getBoolean("followup").bool(); + } } } diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/EnrollmentsExportControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/EnrollmentsExportControllerTest.java index 2fb3de5efa61..88a4659403a2 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/EnrollmentsExportControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/EnrollmentsExportControllerTest.java @@ -252,6 +252,7 @@ void getEnrollmentByIdWithEventsFields() { assertEquals(program.getUid(), event.getProgram()); assertHasMember(event, "status"); + assertHasMember(event, "followUp"); assertHasMember(event, "followup"); assertEquals(program.getUid(), event.getProgram()); assertEquals(orgUnit.getUid(), event.getOrgUnit()); diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java index e431217288f2..f728b3a0a74c 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java @@ -528,6 +528,7 @@ private void assertDefaultResponse(JsonObject json, Event event) { assertEquals(programStage.getUid(), json.getString("programStage").string()); assertEquals(event.getEnrollment().getUid(), json.getString("enrollment").string()); assertEquals(orgUnit.getUid(), json.getString("orgUnit").string()); + assertFalse(json.getBoolean("followUp").booleanValue()); assertFalse(json.getBoolean("followup").booleanValue()); assertFalse(json.getBoolean("deleted").booleanValue()); assertHasMember(json, "createdAt"); diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerUnitTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerUnitTest.java index 9b4dedb4821c..4812d526be1f 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerUnitTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerUnitTest.java @@ -34,10 +34,11 @@ import static org.mockito.Mockito.when; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; +import java.util.stream.Collectors; import org.hisp.dhis.fieldfiltering.FieldFilterService; import org.hisp.dhis.tracker.export.event.EventService; import org.hisp.dhis.webapi.controller.tracker.export.CsvService; @@ -68,9 +69,11 @@ void shouldFailInstantiatingControllerIfAnyOrderableFieldIsUnsupported() { Entry missing1 = iterator.next(); Entry missing2 = iterator.next(); Map orderableFields = new HashMap<>(EventMapper.ORDERABLE_FIELDS); - orderableFields.remove(missing1.getKey()); - orderableFields.remove(missing2.getKey()); - when(eventService.getOrderableFields()).thenReturn(new HashSet<>(orderableFields.values())); + Set orderableValues = + orderableFields.values().stream() + .filter(v -> !v.equals(missing1.getValue()) && !v.equals(missing2.getValue())) + .collect(Collectors.toSet()); + when(eventService.getOrderableFields()).thenReturn(orderableValues); Exception exception = assertThrows( diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipsExportControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipsExportControllerTest.java index 1d309360245f..b13245283228 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipsExportControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipsExportControllerTest.java @@ -171,7 +171,8 @@ void getRelationshipsById() { .content(HttpStatus.OK) .as(JsonRelationship.class); - assertHasOnlyMembers(relationship, "relationship", "relationshipType", "from", "to"); + assertHasOnlyMembers( + relationship, "relationship", "relationshipType", "createdAtClient", "from", "to"); assertRelationship(r, relationship); assertHasOnlyUid(from.getUid(), "event", relationship.getObject("from")); assertHasOnlyUid(to.getUid(), "trackedEntity", relationship.getObject("to")); @@ -245,11 +246,28 @@ void getRelationshipsByEvent() { .getList("instances", JsonRelationship.class); JsonObject relationship = assertFirstRelationship(r, relationships); - assertHasOnlyMembers(relationship, "relationship", "relationshipType", "from", "to"); + assertHasOnlyMembers( + relationship, "relationship", "relationshipType", "createdAtClient", "from", "to"); assertHasOnlyUid(from.getUid(), "event", relationship.getObject("from")); assertHasOnlyUid(to.getUid(), "trackedEntity", relationship.getObject("to")); } + @Test + void getRelationshipsByEventWithAllFields() { + TrackedEntity to = trackedEntity(); + Event from = event(enrollment(to)); + Relationship r = relationship(from, to); + + JsonList relationships = + GET("/tracker/relationships?event={uid}&fields=*", from.getUid()) + .content(HttpStatus.OK) + .getList("instances", JsonRelationship.class); + + JsonRelationship relationship = assertFirstRelationship(r, relationships); + assertEventWithinRelationshipItem(from, relationship.getFrom()); + assertTrackedEntityWithinRelationshipItem(to, relationship.getTo()); + } + @Test void getRelationshipsByEventWithFields() { TrackedEntity to = trackedEntity(); @@ -782,6 +800,7 @@ private Relationship relationship(Event from, TrackedEntity to) { r.setAutoFields(); r.getSharing().setOwner(owner); + r.setCreatedAtClient(new Date()); manager.save(r, false); return r; } diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportControllerTest.java index 825fb74e0274..c311c6392042 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntitiesExportControllerTest.java @@ -648,6 +648,7 @@ private JsonEvent assertDefaultEventResponse(JsonEnrollment enrollment, Event ev assertHasMember(jsonEvent, "createdAtClient"); assertHasMember(jsonEvent, "updatedAt"); assertHasMember(jsonEvent, "notes"); + assertHasMember(jsonEvent, "followUp"); assertHasMember(jsonEvent, "followup"); JsonDataValue dataValue = jsonEvent.getDataValues().get(0); diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/GeoJsonImportController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/GeoJsonImportController.java index be8ef3954c22..8563675ff3ac 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/GeoJsonImportController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/GeoJsonImportController.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2022, University of Oslo + * Copyright (c) 2004-2023, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -28,37 +28,33 @@ package org.hisp.dhis.webapi.controller; import static java.lang.String.format; -import static org.apache.commons.io.IOUtils.toBufferedInputStream; import static org.apache.commons.io.IOUtils.toInputStream; import static org.hisp.dhis.common.IdentifiableProperty.UID; import static org.hisp.dhis.dxf2.webmessage.WebMessageUtils.jobConfigurationReport; import static org.hisp.dhis.dxf2.webmessage.WebMessageUtils.ok; +import static org.springframework.http.MediaType.APPLICATION_JSON; import java.io.IOException; -import java.io.InputStream; import java.nio.charset.StandardCharsets; -import javax.servlet.ServletInputStream; import javax.servlet.http.HttpServletRequest; -import lombok.AllArgsConstructor; -import org.hibernate.SessionFactory; +import lombok.RequiredArgsConstructor; import org.hisp.dhis.common.IdentifiableProperty; import org.hisp.dhis.common.OpenApi; -import org.hisp.dhis.dbms.DbmsUtils; -import org.hisp.dhis.dxf2.geojson.GeoJsonImportParams; import org.hisp.dhis.dxf2.geojson.GeoJsonImportReport; import org.hisp.dhis.dxf2.geojson.GeoJsonService; import org.hisp.dhis.dxf2.importsummary.ImportStatus; import org.hisp.dhis.dxf2.webmessage.WebMessage; +import org.hisp.dhis.feedback.ConflictException; +import org.hisp.dhis.feedback.NotFoundException; import org.hisp.dhis.feedback.Status; import org.hisp.dhis.scheduling.JobConfiguration; +import org.hisp.dhis.scheduling.JobConfigurationService; +import org.hisp.dhis.scheduling.JobSchedulerService; import org.hisp.dhis.scheduling.JobType; -import org.hisp.dhis.security.SecurityContextRunnable; -import org.hisp.dhis.system.notification.NotificationLevel; -import org.hisp.dhis.system.notification.Notifier; +import org.hisp.dhis.scheduling.parameters.GeoJsonImportJobParams; import org.hisp.dhis.user.CurrentUser; +import org.hisp.dhis.user.CurrentUserService; import org.hisp.dhis.user.User; -import org.hisp.dhis.user.UserService; -import org.springframework.core.task.TaskExecutor; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -74,17 +70,15 @@ @OpenApi.Tags("data") @RequestMapping("/organisationUnits") @RestController -@AllArgsConstructor +@RequiredArgsConstructor public class GeoJsonImportController { private final GeoJsonService geoJsonService; - private final Notifier notifier; + private final JobSchedulerService jobSchedulerService; - private final TaskExecutor taskExecutor; + private final JobConfigurationService jobConfigurationService; - private final SessionFactory sessionFactory; - - private final UserService userService; + private final CurrentUserService currentUserService; @PostMapping( value = "/geometry", @@ -96,11 +90,10 @@ public WebMessage postImport( @RequestParam(required = false) String attributeId, @RequestParam(required = false) boolean dryRun, @RequestParam(required = false, defaultValue = "false") boolean async, - HttpServletRequest request, - @CurrentUser User currentUser) - throws IOException { - GeoJsonImportParams params = - GeoJsonImportParams.builder() + HttpServletRequest request) + throws IOException, ConflictException, NotFoundException { + GeoJsonImportJobParams params = + GeoJsonImportJobParams.builder() .attributeId(attributeId) .dryRun(dryRun) .idType( @@ -108,22 +101,26 @@ public WebMessage postImport( ? UID : IdentifiableProperty.valueOf(orgUnitProperty.toUpperCase())) .orgUnitIdProperty(geoJsonId ? "id" : "properties." + geoJsonProperty) - .user(currentUser) .build(); - return runImport(async, params, request.getInputStream()); + return runImport(async, params, request); } - private WebMessage runImport(boolean async, GeoJsonImportParams params, ServletInputStream data) - throws IOException { + private WebMessage runImport( + boolean async, GeoJsonImportJobParams params, HttpServletRequest request) + throws ConflictException, NotFoundException, IOException { + User currentUser = currentUserService.getCurrentUser(); if (async) { - JobConfiguration config = - new JobConfiguration("GeoJSON import", JobType.GEOJSON_IMPORT, params.getUser().getUid()); - taskExecutor.execute(new GeoJsonAsyncImporter(params, config, toBufferedInputStream(data))); - return jobConfigurationReport(config); - } + JobConfiguration jobConfig = new JobConfiguration(JobType.GEOJSON_IMPORT); + jobConfig.setJobParameters(params); + jobConfig.setExecutedBy(currentUser.getUid()); + jobSchedulerService.executeNow( + jobConfigurationService.create(jobConfig, APPLICATION_JSON, request.getInputStream())); - return toWebMessage(geoJsonService.importGeoData(params, data)); + return jobConfigurationReport(jobConfig); + } + params.setUser(currentUser); + return toWebMessage(geoJsonService.importGeoData(params, request.getInputStream())); } @PreAuthorize("hasRole('ALL') or hasRole('F_PERFORM_MAINTENANCE')") @@ -141,8 +138,8 @@ public WebMessage postImportSingle( @RequestParam(required = false) boolean dryRun, @RequestBody String geometry, @CurrentUser User currentUser) { - GeoJsonImportParams params = - GeoJsonImportParams.builder() + GeoJsonImportJobParams params = + GeoJsonImportJobParams.builder() .user(currentUser) .attributeId(attributeId) .dryRun(dryRun) @@ -179,51 +176,4 @@ private WebMessage toWebMessage(GeoJsonImportReport report) { } return ok(msg).setStatus(status).setResponse(report); } - - @AllArgsConstructor - private class GeoJsonAsyncImporter extends SecurityContextRunnable { - - private final GeoJsonImportParams params; - - private final JobConfiguration config; - - private final InputStream data; - - @Override - public void before() { - DbmsUtils.bindSessionToThread(sessionFactory); - } - - @Override - public void after() { - DbmsUtils.unbindSessionFromThread(sessionFactory); - } - - @Override - public void call() { - notifier.clear(config); - notifier.notify(config, NotificationLevel.INFO, "GeoJSON import stared", false); - GeoJsonImportReport report = geoJsonService.importGeoData(reattachedParams(), data); - notifier.notify( - config, - NotificationLevel.INFO, - "GeoJSON import complete. " + report.getImportCount(), - true); - notifier.addJobSummary(config, report, GeoJsonImportReport.class); - } - - @Override - public void handleError(Throwable ex) { - notifier.notify( - config, NotificationLevel.ERROR, "GeoJSON import failed: " + ex.getMessage(), true); - } - - /** - * This is a work-around for the time being because the user otherwise is not attached to the - * session. What we should be using here instead is the CurrentUserDetails - */ - private GeoJsonImportParams reattachedParams() { - return params.toBuilder().user(userService.getUser(params.getUser().getUid())).build(); - } - } } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SystemSettingController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SystemSettingController.java index 9de28c5c7b1a..e1d67a45df88 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SystemSettingController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SystemSettingController.java @@ -89,6 +89,9 @@ public class SystemSettingController { private final UserSettingService userSettingService; + private static final String SETTING_DOESNT_EXIST_OR_CONFIDENTIAL = + "Setting does not exist or is marked as confidential"; + // ------------------------------------------------------------------------- // Create // ------------------------------------------------------------------------- @@ -191,20 +194,24 @@ public WebMessage setSystemSettingV29(@RequestBody Map settings) // ------------------------------------------------------------------------- @GetMapping(value = "/{key}", produces = ContextUtils.CONTENT_TYPE_TEXT) - public @ResponseBody Serializable getSystemSettingOrTranslationAsPlainText( + public @ResponseBody ResponseEntity getSystemSettingOrTranslationAsPlainText( @PathVariable("key") String key, @RequestParam(value = "locale", required = false) String locale, HttpServletResponse response, @CurrentUser User currentUser) { + if (!settingExistsAndIsNotConfidential(key)) { + return ResponseEntity.status(404).body(SETTING_DOESNT_EXIST_OR_CONFIDENTIAL); + } response.setHeader( ContextUtils.HEADER_CACHE_CONTROL, CacheControl.noCache().cachePrivate().getHeaderValue()); - return String.valueOf(getSystemSettingOrTranslation(key, locale, currentUser)); + return ResponseEntity.ok() + .body(String.valueOf(getSystemSettingOrTranslation(key, locale, currentUser))); } @GetMapping( value = "/{key}", - produces = {ContextUtils.CONTENT_TYPE_JSON, ContextUtils.CONTENT_TYPE_HTML}) + produces = {ContextUtils.CONTENT_TYPE_JSON}) public @ResponseBody ResponseEntity> getSystemSettingOrTranslationAsJson( @PathVariable("key") String key, @@ -212,7 +219,7 @@ public WebMessage setSystemSettingV29(@RequestBody Map settings) @CurrentUser User currentUser) throws NotFoundException { if (!settingExistsAndIsNotConfidential(key)) { - throw new NotFoundException("Setting does not exist or is marked as confidential"); + throw new NotFoundException(SETTING_DOESNT_EXIST_OR_CONFIDENTIAL); } return ResponseEntity.ok() @@ -271,10 +278,16 @@ private Optional getLocaleToFetch(String locale, String key, User curren @GetMapping(produces = APPLICATION_JSON_VALUE) public ResponseEntity> getSystemSettingsJson( - @RequestParam(value = "key", required = false) Set keys) { + @RequestParam(value = "key", required = false) Set keys) throws NotFoundException { + Set settingKeysToFetch = getSettingKeysToFetch(keys); + + if (settingKeysToFetch.isEmpty()) { + throw new NotFoundException(SETTING_DOESNT_EXIST_OR_CONFIDENTIAL); + } + return ResponseEntity.ok() .cacheControl(CacheControl.noCache().cachePrivate()) - .body(systemSettingManager.getSystemSettings(getSettingKeysToFetch(keys))); + .body(systemSettingManager.getSystemSettings(settingKeysToFetch)); } private Set getSettingKeysToFetch(Set keys) { diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/deprecated/tracker/EventUtils.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/deprecated/tracker/EventUtils.java index 2ba8e4f3c245..19eb6490ffc7 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/deprecated/tracker/EventUtils.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/deprecated/tracker/EventUtils.java @@ -30,18 +30,9 @@ import static org.hisp.dhis.common.OrganisationUnitSelectionMode.ACCESSIBLE; import static org.hisp.dhis.common.OrganisationUnitSelectionMode.SELECTED; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.function.Function; -import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.common.OrganisationUnitSelectionMode; -import org.hisp.dhis.feedback.ForbiddenException; import org.hisp.dhis.organisationunit.OrganisationUnit; -import org.hisp.dhis.program.Program; -import org.hisp.dhis.trackedentity.TrackerAccessManager; -import org.hisp.dhis.user.User; @Slf4j public class EventUtils { @@ -49,122 +40,6 @@ private EventUtils() { throw new UnsupportedOperationException("Utility class"); } - public static List validateAccessibleOrgUnits( - User user, - OrganisationUnit orgUnit, - OrganisationUnitSelectionMode orgUnitMode, - Program program, - Function> orgUnitDescendants, - TrackerAccessManager trackerAccessManager) - throws ForbiddenException { - List accessibleOrgUnits = - getUserAccessibleOrgUnits( - user, orgUnit, orgUnitMode, program, orgUnitDescendants, trackerAccessManager); - - if (orgUnit != null && accessibleOrgUnits.isEmpty()) { - throw new ForbiddenException("User does not have access to orgUnit: " + orgUnit.getUid()); - } - - return accessibleOrgUnits; - } - - /** - * Returns a list of all the org units the user has access to - * - * @param user the user to check the access of - * @param orgUnit parent org unit to get descendants/children of - * @param orgUnitDescendants function to retrieve org units, in case ou mode is descendants - * @param program the program the user wants to access to - * @return a list containing the user accessible organisation units - */ - private static List getUserAccessibleOrgUnits( - User user, - OrganisationUnit orgUnit, - OrganisationUnitSelectionMode orgUnitMode, - Program program, - Function> orgUnitDescendants, - TrackerAccessManager trackerAccessManager) { - - switch (orgUnitMode) { - case DESCENDANTS: - return orgUnit != null - ? getAccessibleDescendants(user, program, orgUnitDescendants.apply(orgUnit.getUid())) - : Collections.emptyList(); - case CHILDREN: - return orgUnit != null - ? getAccessibleDescendants( - user, - program, - Stream.concat(Stream.of(orgUnit), orgUnit.getChildren().stream()).toList()) - : Collections.emptyList(); - case CAPTURE: - return new ArrayList<>(user.getOrganisationUnits()); - case ACCESSIBLE: - return getAccessibleOrgUnits(user, program); - case SELECTED: - return getSelectedOrgUnits(user, program, orgUnit, trackerAccessManager); - default: - return Collections.emptyList(); - } - } - - private static List getSelectedOrgUnits( - User user, - Program program, - OrganisationUnit orgUnit, - TrackerAccessManager trackerAccessManager) { - return trackerAccessManager.canAccess(user, program, orgUnit) - ? List.of(orgUnit) - : Collections.emptyList(); - } - - private static List getAccessibleOrgUnits(User user, Program program) { - return isProgramAccessRestricted(program) - ? new ArrayList<>(user.getOrganisationUnits()) - : new ArrayList<>(user.getTeiSearchOrganisationUnitsWithFallback()); - } - - /** - * Returns the org units whose path is contained in the user search or capture scope org unit. If - * there's a match, it means the user org unit is at the same level or above the supplied org - * unit. - * - * @param user the user to check the access of - * @param program the program the user wants to access to - * @param orgUnits the org units to check if the user has access to - * @return a list with the org units the user has access to - */ - private static List getAccessibleDescendants( - User user, Program program, List orgUnits) { - if (orgUnits.isEmpty()) { - return Collections.emptyList(); - } - - if (isProgramAccessRestricted(program)) { - return orgUnits.stream() - .filter( - availableOrgUnit -> - user.getOrganisationUnits().stream() - .anyMatch( - captureScopeOrgUnit -> - availableOrgUnit.getPath().contains(captureScopeOrgUnit.getPath()))) - .toList(); - } else { - return orgUnits.stream() - .filter( - availableOrgUnit -> - user.getTeiSearchOrganisationUnits().stream() - .anyMatch( - searchScopeOrgUnit -> - availableOrgUnit.getPath().contains(searchScopeOrgUnit.getPath()))) - .toList(); - } - } - - private static boolean isProgramAccessRestricted(Program program) { - return program != null && (program.isClosed() || program.isProtected()); - } - /** * Returns the same org unit mode if not null. If null, and an org unit is present, SELECT mode is * used by default, mode ACCESSIBLE is used otherwise. diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/EventChartController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/EventChartController.java index d1407aeb726e..42c02b5f5b8f 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/EventChartController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/EventChartController.java @@ -133,8 +133,7 @@ public void getChart( @RequestParam(value = "attachment", required = false) boolean attachment, HttpServletResponse response) throws IOException, WebMessageException { - EventChart chart = eventChartService.getEventChart(uid); // TODO no - // acl? + EventChart chart = eventChartService.getEventChart(uid); if (chart == null) { throw new WebMessageException(notFound("Event chart does not exist: " + uid)); @@ -164,7 +163,7 @@ public void getChart( /** * @deprecated This is a temporary workaround to keep EventChart backward compatible with the new * EventVisualization entity. Only legacy and chart related types can be returned by this - * endpoint. + * endpoint. Also, multi-program charts cannot be generated, so they are filtered out. * @param filters */ @Deprecated diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/EventReportController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/EventReportController.java index 2a3ff5d60b73..0ba7e3b61985 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/EventReportController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/EventReportController.java @@ -75,8 +75,6 @@ public class EventReportController extends AbstractCrudController { // CRUD // -------------------------------------------------------------------------- - // TODO: ONLY allow querying LINE_LIST and PIVOT_TABLE type. - @Override protected EventReport deserializeJsonEntity(HttpServletRequest request) throws IOException { EventReport eventReport = super.deserializeJsonEntity(request); diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/EventVisualizationController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/EventVisualizationController.java index 2868adb1df74..d77e4c940f9e 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/EventVisualizationController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/EventVisualizationController.java @@ -29,8 +29,9 @@ import static org.hisp.dhis.common.DhisApiVersion.ALL; import static org.hisp.dhis.common.DhisApiVersion.DEFAULT; -import static org.hisp.dhis.common.DimensionalObjectUtils.getDimensions; +import static org.hisp.dhis.common.DimensionalObjectUtils.getQualifiedDimensions; import static org.hisp.dhis.common.cache.CacheStrategy.RESPECT_SYSTEM_SETTING; +import static org.hisp.dhis.dxf2.webmessage.WebMessageUtils.conflict; import static org.hisp.dhis.dxf2.webmessage.WebMessageUtils.notFound; import static org.hisp.dhis.eventvisualization.EventVisualizationType.LINE_LIST; import static org.hisp.dhis.eventvisualization.EventVisualizationType.PIVOT_TABLE; @@ -111,14 +112,13 @@ void generateChart( @RequestParam(value = "attachment", required = false) boolean attachment, HttpServletResponse response) throws IOException, WebMessageException { - // TODO no acl? EventVisualization eventVisualization = eventVisualizationService.getEventVisualization(uid); if (eventVisualization == null) { throw new WebMessageException(notFound("Event visualization does not exist: " + uid)); } - doesNotAllowPivotAndReportChart(eventVisualization); + checkChartGenerationConditions(eventVisualization); OrganisationUnit unit = ou != null ? organisationUnitService.getOrganisationUnit(ou) : null; @@ -243,9 +243,15 @@ private void prepare(EventVisualization eventVisualization) { eventVisualization.getFilterDimensions().clear(); eventVisualization.getSimpleDimensions().clear(); - eventVisualization.getColumnDimensions().addAll(getDimensions(eventVisualization.getColumns())); - eventVisualization.getRowDimensions().addAll(getDimensions(eventVisualization.getRows())); - eventVisualization.getFilterDimensions().addAll(getDimensions(eventVisualization.getFilters())); + eventVisualization + .getColumnDimensions() + .addAll(getQualifiedDimensions(eventVisualization.getColumns())); + eventVisualization + .getRowDimensions() + .addAll(getQualifiedDimensions(eventVisualization.getRows())); + eventVisualization + .getFilterDimensions() + .addAll(getQualifiedDimensions(eventVisualization.getFilters())); eventVisualization.associateSimpleDimensions(); maybeLoadLegendSetInto(eventVisualization); @@ -268,11 +274,18 @@ private void maybeLoadLegendSetInto(EventVisualization eventVisualization) { } } - private void doesNotAllowPivotAndReportChart(EventVisualization eventVisualization) + private void checkChartGenerationConditions(EventVisualization eventVisualization) throws WebMessageException { if (eventVisualization.getType() == PIVOT_TABLE || eventVisualization.getType() == LINE_LIST) { throw new WebMessageException( - notFound("Cannot generate chart for " + eventVisualization.getType())); + conflict("Cannot generate chart for " + eventVisualization.getType())); + } + + if (eventVisualization.isMultiProgram()) { + throw new WebMessageException( + conflict( + "Cannot generate chart for multi-program visualization " + + eventVisualization.getUid())); } } } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/ProgramMessageController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/ProgramMessageController.java index be30c25c4d77..3feb7ef28a6b 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/ProgramMessageController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/ProgramMessageController.java @@ -43,6 +43,8 @@ import org.hisp.dhis.feedback.BadRequestException; import org.hisp.dhis.feedback.ConflictException; import org.hisp.dhis.outboundmessage.BatchResponseStatus; +import org.hisp.dhis.program.Enrollment; +import org.hisp.dhis.program.Event; import org.hisp.dhis.program.message.ProgramMessage; import org.hisp.dhis.program.message.ProgramMessageBatch; import org.hisp.dhis.program.message.ProgramMessageQueryParams; @@ -126,10 +128,16 @@ public List getProgramMessages( @GetMapping(value = "/scheduled/sent", produces = APPLICATION_JSON_VALUE) @ResponseBody public List getScheduledSentMessage( - @Deprecated(since = "2.41") @RequestParam(required = false) UID programInstance, - @RequestParam(required = false) UID enrollment, - @Deprecated(since = "2.41") @RequestParam(required = false) UID programStageInstance, - @RequestParam(required = false) UID event, + @OpenApi.Param(value = Enrollment.class) + @Deprecated(since = "2.41") + @RequestParam(required = false) + UID programInstance, + @OpenApi.Param({UID.class, Enrollment.class}) @RequestParam(required = false) UID enrollment, + @OpenApi.Param(value = Event.class) + @Deprecated(since = "2.41") + @RequestParam(required = false) + UID programStageInstance, + @OpenApi.Param({UID.class, Event.class}) @RequestParam(required = false) UID event, @RequestParam(required = false) Date afterDate, @RequestParam(required = false) Integer page, @RequestParam(required = false) Integer pageSize) @@ -160,7 +168,7 @@ public List getScheduledSentMessage( @PreAuthorize("hasRole('ALL') or hasRole('F_MOBILE_SENDSMS') or hasRole('F_SEND_EMAIL')") @PostMapping(consumes = APPLICATION_JSON_VALUE, produces = APPLICATION_JSON_VALUE) @ResponseBody - public BatchResponseStatus saveMessages(HttpServletRequest request, HttpServletResponse response) + public BatchResponseStatus sendMessages(HttpServletRequest request, HttpServletResponse response) throws IOException { ProgramMessageBatch batch = renderService.fromJson(request.getInputStream(), ProgramMessageBatch.class); diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/scheduling/JobConfigurationController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/scheduling/JobConfigurationController.java index d6343591afdb..f1eb38f81765 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/scheduling/JobConfigurationController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/scheduling/JobConfigurationController.java @@ -74,7 +74,7 @@ public class JobConfigurationController extends AbstractCrudController getDueJobConfigurations( @RequestParam int seconds, @RequestParam(required = false, defaultValue = "false") boolean includeWaiting) { - return jobConfigurationService.getDueJobConfigurations(seconds, false, includeWaiting); + return jobConfigurationService.getDueJobConfigurations(seconds, includeWaiting); } @GetMapping("/stale") diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/RequestParamsValidator.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/RequestParamsValidator.java index 853559bb1173..722bd5c29bba 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/RequestParamsValidator.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/RequestParamsValidator.java @@ -326,6 +326,25 @@ else if (filters.length == 4) { } } + public static OrganisationUnitSelectionMode validateOrgUnitModeForTrackedEntities( + Set orgUnits, OrganisationUnitSelectionMode orgUnitMode, Set trackedEntities) + throws BadRequestException { + + orgUnitMode = validateOrgUnitMode(orgUnitMode, orgUnits); + validateOrgUnitOrTrackedEntityIsPresent(orgUnitMode, orgUnits, trackedEntities); + + return orgUnitMode; + } + + public static OrganisationUnitSelectionMode validateOrgUnitModeForEnrollmentsAndEvents( + Set orgUnits, OrganisationUnitSelectionMode orgUnitMode) throws BadRequestException { + + orgUnitMode = validateOrgUnitMode(orgUnitMode, orgUnits); + validateOrgUnitIsPresent(orgUnitMode, orgUnits); + + return orgUnitMode; + } + /** * Validates that no org unit is present if the ou mode is ACCESSIBLE or CAPTURE. If it is, an * exception will be thrown. If the org unit mode is not defined, SELECTED will be used by default @@ -335,64 +354,52 @@ else if (filters.length == 4) { * @return a valid org unit mode * @throws BadRequestException if a wrong combination of org unit and org unit mode supplied */ - public static OrganisationUnitSelectionMode validateOrgUnitMode( - Set orgUnits, OrganisationUnitSelectionMode orgUnitMode) throws BadRequestException { - + private static OrganisationUnitSelectionMode validateOrgUnitMode( + OrganisationUnitSelectionMode orgUnitMode, Set orgUnits) throws BadRequestException { if (orgUnitMode == null) { return orgUnits.isEmpty() ? ACCESSIBLE : SELECTED; } - if (!orgUnits.isEmpty() - && (orgUnitMode == ACCESSIBLE || orgUnitMode == CAPTURE || orgUnitMode == ALL)) { + if (orgUnitModeDoesNotRequireOrgUnit(orgUnitMode) && !orgUnits.isEmpty()) { throw new BadRequestException( String.format( "orgUnitMode %s cannot be used with orgUnits. Please remove the orgUnit parameter and try again.", orgUnitMode)); } - if ((orgUnitMode == CHILDREN || orgUnitMode == SELECTED || orgUnitMode == DESCENDANTS) - && orgUnits.isEmpty()) { - throw new BadRequestException( - String.format( - "At least one org unit is required for orgUnitMode: %s. Please add an orgUnit or use a different orgUnitMode.", - orgUnitMode)); - } - return orgUnitMode; } - /** - * Validates that the org unit is not present if the ou mode is ACCESSIBLE or CAPTURE. If it is, - * an exception will be thrown. If the org unit mode is not defined, SELECTED will be used by - * default if an org unit is present. Otherwise, ACCESSIBLE will be the default. - * - * @param orgUnit the org unit to validate - * @return a valid org unit mode - * @throws BadRequestException if a wrong combination of org unit and org unit mode supplied - */ - public static OrganisationUnitSelectionMode validateOrgUnitMode( - UID orgUnit, OrganisationUnitSelectionMode orgUnitMode) throws BadRequestException { - - if (orgUnitMode == null) { - orgUnitMode = orgUnit != null ? SELECTED : ACCESSIBLE; - } - - if ((orgUnitMode == ACCESSIBLE || orgUnitMode == CAPTURE) && orgUnit != null) { + private static void validateOrgUnitOrTrackedEntityIsPresent( + OrganisationUnitSelectionMode orgUnitMode, Set orgUnits, Set trackedEntities) + throws BadRequestException { + if (orgUnitModeRequiresOrgUnit(orgUnitMode) + && orgUnits.isEmpty() + && trackedEntities.isEmpty()) { throw new BadRequestException( String.format( - "orgUnitMode %s cannot be used with orgUnits. Please remove the orgUnit parameter and try again.", + "At least one org unit or tracked entity is required for orgUnitMode: %s. Please add one of the two or use a different orgUnitMode.", orgUnitMode)); } + } - if ((orgUnitMode == CHILDREN || orgUnitMode == SELECTED || orgUnitMode == DESCENDANTS) - && orgUnit == null) { + private static void validateOrgUnitIsPresent( + OrganisationUnitSelectionMode orgUnitMode, Set orgUnits) throws BadRequestException { + if (orgUnitModeRequiresOrgUnit(orgUnitMode) && orgUnits.isEmpty()) { throw new BadRequestException( String.format( - "orgUnit is required for orgUnitMode: %s. Please add an orgUnit or use a different orgUnitMode.", + "At least one org unit is required for orgUnitMode: %s. Please add one org unit or use a different orgUnitMode.", orgUnitMode)); } + } - return orgUnitMode; + private static boolean orgUnitModeRequiresOrgUnit(OrganisationUnitSelectionMode orgUnitMode) { + return orgUnitMode == CHILDREN || orgUnitMode == SELECTED || orgUnitMode == DESCENDANTS; + } + + private static boolean orgUnitModeDoesNotRequireOrgUnit( + OrganisationUnitSelectionMode orgUnitMode) { + return orgUnitMode == ACCESSIBLE || orgUnitMode == CAPTURE || orgUnitMode == ALL; } public static void validatePaginationParameters(PageRequestParams params) diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/EnrollmentRequestParamsMapper.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/EnrollmentRequestParamsMapper.java index a2eb3966725f..90fed0fea4ce 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/EnrollmentRequestParamsMapper.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/EnrollmentRequestParamsMapper.java @@ -27,16 +27,12 @@ */ package org.hisp.dhis.webapi.controller.tracker.export.enrollment; -import static org.apache.commons.lang3.BooleanUtils.toBooleanDefaultIfNull; -import static org.hisp.dhis.tracker.export.enrollment.EnrollmentOperationParams.DEFAULT_PAGE; -import static org.hisp.dhis.tracker.export.enrollment.EnrollmentOperationParams.DEFAULT_PAGE_SIZE; import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateDeprecatedParameter; import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateDeprecatedUidsParameter; import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateOrderParams; -import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateOrgUnitMode; +import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateOrgUnitModeForEnrollmentsAndEvents; import java.util.List; -import java.util.Objects; import java.util.Set; import lombok.RequiredArgsConstructor; import org.hisp.dhis.common.OrganisationUnitSelectionMode; @@ -69,10 +65,17 @@ public EnrollmentOperationParams map(RequestParams requestParams) throws BadRequ validateDeprecatedParameter( "ouMode", requestParams.getOuMode(), "orgUnitMode", requestParams.getOrgUnitMode()); - validateOrgUnitMode(orgUnits, orgUnitMode); + orgUnitMode = validateOrgUnitModeForEnrollmentsAndEvents(orgUnits, orgUnitMode); validateOrderParams(requestParams.getOrder(), ORDERABLE_FIELD_NAMES); + Set enrollmentUids = + validateDeprecatedUidsParameter( + "enrollment", + requestParams.getEnrollment(), + "enrollments", + requestParams.getEnrollments()); + EnrollmentOperationParamsBuilder builder = EnrollmentOperationParams.builder() .programUid( @@ -93,11 +96,8 @@ public EnrollmentOperationParams map(RequestParams requestParams) throws BadRequ : null) .orgUnitUids(UID.toValueSet(orgUnits)) .orgUnitMode(orgUnitMode) - .page(Objects.requireNonNullElse(requestParams.getPage(), DEFAULT_PAGE)) - .pageSize(Objects.requireNonNullElse(requestParams.getPageSize(), DEFAULT_PAGE_SIZE)) - .totalPages(toBooleanDefaultIfNull(requestParams.isTotalPages(), false)) - .skipPaging(toBooleanDefaultIfNull(requestParams.isSkipPaging(), false)) .includeDeleted(requestParams.isIncludeDeleted()) + .enrollmentUids(UID.toValueSet(enrollmentUids)) .enrollmentParams(fieldsParamMapper.map(requestParams.getFields())); mapOrderParam(builder, requestParams.getOrder()); diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/EnrollmentsExportController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/EnrollmentsExportController.java index 6dd127a6630f..7d8122e58439 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/EnrollmentsExportController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/EnrollmentsExportController.java @@ -29,14 +29,13 @@ import static org.hisp.dhis.webapi.controller.tracker.ControllerSupport.RESOURCE_PATH; import static org.hisp.dhis.webapi.controller.tracker.ControllerSupport.assertUserOrderableFieldsAreSupported; -import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateDeprecatedUidsParameter; +import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validatePaginationParameters; import static org.hisp.dhis.webapi.controller.tracker.export.enrollment.RequestParams.DEFAULT_FIELDS_PARAM; import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; import com.fasterxml.jackson.databind.node.ObjectNode; -import java.util.ArrayList; +import java.util.Collection; import java.util.List; -import java.util.Set; import org.hisp.dhis.common.DhisApiVersion; import org.hisp.dhis.common.OpenApi; import org.hisp.dhis.common.OpenApi.Response.Status; @@ -46,10 +45,11 @@ import org.hisp.dhis.feedback.NotFoundException; import org.hisp.dhis.fieldfiltering.FieldFilterService; import org.hisp.dhis.fieldfiltering.FieldPath; +import org.hisp.dhis.tracker.export.Page; +import org.hisp.dhis.tracker.export.PageParams; import org.hisp.dhis.tracker.export.enrollment.EnrollmentOperationParams; import org.hisp.dhis.tracker.export.enrollment.EnrollmentParams; import org.hisp.dhis.tracker.export.enrollment.EnrollmentService; -import org.hisp.dhis.tracker.export.enrollment.Enrollments; import org.hisp.dhis.webapi.controller.event.webrequest.PagingWrapper; import org.hisp.dhis.webapi.controller.tracker.export.OpenApiExport; import org.hisp.dhis.webapi.controller.tracker.view.Enrollment; @@ -99,43 +99,43 @@ public EnrollmentsExportController( @GetMapping(produces = APPLICATION_JSON_VALUE) PagingWrapper getEnrollments(RequestParams requestParams) throws BadRequestException, ForbiddenException, NotFoundException { - PagingWrapper pagingWrapper = new PagingWrapper<>(); + validatePaginationParameters(requestParams); + EnrollmentOperationParams operationParams = paramsMapper.map(requestParams); - List enrollmentList; + if (requestParams.isPaged()) { + PageParams pageParams = + new PageParams( + requestParams.getPage(), requestParams.getPageSize(), requestParams.getTotalPages()); - EnrollmentOperationParams operationParams = paramsMapper.map(requestParams); + Page enrollmentPage = + enrollmentService.getEnrollments(operationParams, pageParams); - Set enrollmentUids = - validateDeprecatedUidsParameter( - "enrollment", - requestParams.getEnrollment(), - "enrollments", - requestParams.getEnrollments()); - if (enrollmentUids.isEmpty()) { - Enrollments enrollments = enrollmentService.getEnrollments(operationParams); - - if (requestParams.isPagingRequest()) { - pagingWrapper = - pagingWrapper.withPager( - PagingWrapper.Pager.fromLegacy(requestParams, enrollments.getPager())); - } + PagingWrapper.Pager.PagerBuilder pagerBuilder = + PagingWrapper.Pager.builder() + .page(enrollmentPage.getPager().getPage()) + .pageSize(enrollmentPage.getPager().getPageSize()); - enrollmentList = enrollments.getEnrollments(); - } else { - List list = new ArrayList<>(); - for (UID uid : enrollmentUids) { - list.add( - enrollmentService.getEnrollment( - uid.getValue(), - operationParams.getEnrollmentParams(), - operationParams.isIncludeDeleted())); + if (requestParams.isPageTotal()) { + pagerBuilder + .pageCount(enrollmentPage.getPager().getPageCount()) + .total(enrollmentPage.getPager().getTotal()); } - enrollmentList = list; + + PagingWrapper pagingWrapper = new PagingWrapper<>(); + pagingWrapper = pagingWrapper.withPager(pagerBuilder.build()); + List objectNodes = + fieldFilterService.toObjectNodes( + ENROLLMENT_MAPPER.fromCollection(enrollmentPage.getItems()), + requestParams.getFields()); + return pagingWrapper.withInstances(objectNodes); } + Collection enrollments = + enrollmentService.getEnrollments(operationParams); List objectNodes = fieldFilterService.toObjectNodes( - ENROLLMENT_MAPPER.fromCollection(enrollmentList), requestParams.getFields()); + ENROLLMENT_MAPPER.fromCollection(enrollments), requestParams.getFields()); + PagingWrapper pagingWrapper = new PagingWrapper<>(); return pagingWrapper.withInstances(objectNodes); } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/RequestParams.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/RequestParams.java index 1eb219508dec..f2f6597a87c1 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/RequestParams.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/RequestParams.java @@ -27,6 +27,7 @@ */ package org.hisp.dhis.webapi.controller.tracker.export.enrollment; +import java.util.ArrayList; import java.util.Date; import java.util.HashSet; import java.util.List; @@ -42,7 +43,8 @@ import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramStatus; import org.hisp.dhis.trackedentity.TrackedEntityType; -import org.hisp.dhis.webapi.controller.event.webrequest.PagingAndSortingCriteriaAdapter; +import org.hisp.dhis.webapi.controller.event.webrequest.OrderCriteria; +import org.hisp.dhis.webapi.controller.tracker.export.PageRequestParams; import org.hisp.dhis.webapi.controller.tracker.view.Enrollment; import org.hisp.dhis.webapi.controller.tracker.view.TrackedEntity; @@ -51,9 +53,16 @@ @OpenApi.Property @Data @NoArgsConstructor -class RequestParams extends PagingAndSortingCriteriaAdapter { +class RequestParams implements PageRequestParams { static final String DEFAULT_FIELDS_PARAM = "*,!relationships,!events,!attributes"; + private Integer page; + private Integer pageSize; + private Boolean totalPages; + private Boolean skipPaging; + + private List order = new ArrayList<>(); + /** * Semicolon-delimited list of organisation unit UIDs. * diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventDataValue.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventDataValue.java index 7529e8f6c063..11031d871c24 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventDataValue.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventDataValue.java @@ -46,7 +46,7 @@ "geometry", "latitude", "longitude", - "followup", + "followUp", "deleted", "createdAt", "createdAtClient", @@ -83,7 +83,7 @@ class CsvEventDataValue { private String scheduledAt; - private boolean followup; + private boolean followUp; private boolean deleted; @@ -140,7 +140,7 @@ public CsvEventDataValue(CsvEventDataValue dataValue) { orgUnit = dataValue.getOrgUnit(); occurredAt = dataValue.getOccurredAt(); scheduledAt = dataValue.getScheduledAt(); - followup = dataValue.isFollowup(); + followUp = dataValue.isFollowUp(); deleted = dataValue.isDeleted(); createdAt = dataValue.getCreatedAt(); updatedAt = dataValue.getUpdatedAt(); @@ -232,12 +232,12 @@ public void setScheduledAt(String scheduledAt) { } @JsonProperty - public boolean isFollowup() { - return followup; + public boolean isFollowUp() { + return followUp; } - public void setFollowup(boolean followup) { - this.followup = followup; + public void setFollowUp(boolean followUp) { + this.followUp = followUp; } @JsonProperty diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventService.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventService.java index c02983c4650c..d34954bf38f8 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventService.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventService.java @@ -102,7 +102,7 @@ private static CsvEventDataValue map(Event event) { result.setOccurredAt(event.getOccurredAt() == null ? null : event.getOccurredAt().toString()); result.setScheduledAt( event.getScheduledAt() == null ? null : event.getScheduledAt().toString()); - result.setFollowup(event.isFollowup()); + result.setFollowUp(event.isFollowUp()); result.setDeleted(event.isDeleted()); result.setCreatedAt(event.getCreatedAt() == null ? null : event.getCreatedAt().toString()); result.setCreatedAtClient( diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventMapper.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventMapper.java index f231a74e68cf..efd1a43ec87e 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventMapper.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventMapper.java @@ -70,7 +70,8 @@ public interface EventMapper entry("enrollment", "enrollment.uid"), entry("enrollmentStatus", "enrollment.status"), entry("event", "uid"), - entry("followup", "enrollment.followup"), + entry("followUp", "enrollment.followUp"), + entry("followup", "enrollment.followUp"), // Deprecated 2.41 entry("occurredAt", "executionDate"), entry("orgUnit", "organisationUnit.uid"), entry("program", "enrollment.program.uid"), @@ -90,7 +91,8 @@ public interface EventMapper @Mapping(target = "orgUnit", source = "organisationUnit.uid") @Mapping(target = "occurredAt", source = "executionDate") @Mapping(target = "scheduledAt", source = "dueDate") - @Mapping(target = "followup", source = "enrollment.followup") + @Mapping(target = "legacyFollowUp", source = "enrollment.followup") // Deprecated 2.41 + @Mapping(target = "followUp", source = "enrollment.followup") @Mapping(target = "createdAt", source = "created") @Mapping(target = "createdAtClient", source = "createdAtClient") @Mapping(target = "updatedAt", source = "lastUpdated") diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventRequestParamsMapper.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventRequestParamsMapper.java index bc8679d8c8c3..5d35ab0a39c2 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventRequestParamsMapper.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventRequestParamsMapper.java @@ -27,11 +27,12 @@ */ package org.hisp.dhis.webapi.controller.tracker.export.event; +import static java.util.Collections.emptySet; import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.parseFilters; import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateDeprecatedParameter; import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateDeprecatedUidsParameter; import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateOrderParams; -import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateOrgUnitMode; +import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateOrgUnitModeForEnrollmentsAndEvents; import java.util.List; import java.util.Map; @@ -63,7 +64,10 @@ public EventOperationParams map(RequestParams requestParams) throws BadRequestEx validateDeprecatedParameter( "ouMode", requestParams.getOuMode(), "orgUnitMode", requestParams.getOrgUnitMode()); - orgUnitMode = validateOrgUnitMode(requestParams.getOrgUnit(), orgUnitMode); + orgUnitMode = + validateOrgUnitModeForEnrollmentsAndEvents( + requestParams.getOrgUnit() != null ? Set.of(requestParams.getOrgUnit()) : emptySet(), + orgUnitMode); UID attributeCategoryCombo = validateDeprecatedParameter( diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipItemMapper.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipItemMapper.java index 7511083c9cda..022c600fe9ba 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipItemMapper.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipItemMapper.java @@ -101,7 +101,8 @@ default EnrollmentStatus from(ProgramStatus programStatus) { @Mapping(target = "orgUnit", source = "organisationUnit.uid") @Mapping(target = "occurredAt", source = "executionDate") @Mapping(target = "scheduledAt", source = "dueDate") - @Mapping(target = "followup", source = "enrollment.followup") + @Mapping(target = "followUp", source = "enrollment.followup") + @Mapping(target = "legacyFollowUp", source = "enrollment.followup") @Mapping(target = "createdAt", source = "created") @Mapping(target = "createdAtClient", source = "createdAtClient") @Mapping(target = "updatedAt", source = "lastUpdated") diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipMapper.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipMapper.java index 502edc66390e..5488b92eae07 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipMapper.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipMapper.java @@ -46,13 +46,15 @@ public interface RelationshipMapper * Relationships can be ordered by given fields which correspond to fields on {@link * org.hisp.dhis.relationship.Relationship}. */ - Map ORDERABLE_FIELDS = Map.ofEntries(entry("createdAt", "created")); + Map ORDERABLE_FIELDS = + Map.ofEntries(entry("createdAt", "created"), entry("createdAtClient", "createdAtClient")); @Mapping(target = "relationship", source = "uid") @Mapping(target = "relationshipType", source = "relationshipType.uid") @Mapping(target = "relationshipName", source = "relationshipType.name") @Mapping(target = "bidirectional", source = "relationshipType.bidirectional") @Mapping(target = "createdAt", source = "created") + @Mapping(target = "createdAtClient", source = "createdAtClient") @Mapping(target = "updatedAt", source = "lastUpdated") @Override Relationship from(org.hisp.dhis.relationship.Relationship relationship); diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipRequestParamsMapper.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipRequestParamsMapper.java index 1aa64ca44b5a..20befd0ee4ca 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipRequestParamsMapper.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipRequestParamsMapper.java @@ -31,8 +31,6 @@ import static org.hisp.dhis.tracker.TrackerType.ENROLLMENT; import static org.hisp.dhis.tracker.TrackerType.EVENT; import static org.hisp.dhis.tracker.TrackerType.TRACKED_ENTITY; -import static org.hisp.dhis.tracker.export.enrollment.EnrollmentOperationParams.DEFAULT_PAGE; -import static org.hisp.dhis.tracker.export.enrollment.EnrollmentOperationParams.DEFAULT_PAGE_SIZE; import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateDeprecatedParameter; import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateOrderParams; @@ -59,6 +57,8 @@ @Component @RequiredArgsConstructor class RelationshipRequestParamsMapper { + private static final int DEFAULT_PAGE = 1; + private static final int DEFAULT_PAGE_SIZE = 50; private static final Set ORDERABLE_FIELD_NAMES = RelationshipMapper.ORDERABLE_FIELDS.keySet(); diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RequestParams.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RequestParams.java index cec8b1cb9822..b70f511b26b2 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RequestParams.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RequestParams.java @@ -46,7 +46,7 @@ class RequestParams extends PagingAndSortingCriteriaAdapter { static final String DEFAULT_FIELDS_PARAM = - "relationship,relationshipType,from[trackedEntity[trackedEntity],enrollment[enrollment],event[event]],to[trackedEntity[trackedEntity],enrollment[enrollment],event[event]]"; + "relationship,relationshipType,createdAtClient,from[trackedEntity[trackedEntity],enrollment[enrollment],event[event]],to[trackedEntity[trackedEntity],enrollment[enrollment],event[event]]"; /** * @deprecated use {@link #trackedEntity} instead diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntityRequestParamsMapper.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntityRequestParamsMapper.java index 1ddd49876c1d..679c9ba3e5d0 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntityRequestParamsMapper.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntityRequestParamsMapper.java @@ -28,13 +28,11 @@ package org.hisp.dhis.webapi.controller.tracker.export.trackedentity; import static org.apache.commons.lang3.BooleanUtils.toBooleanDefaultIfNull; -import static org.hisp.dhis.tracker.export.enrollment.EnrollmentOperationParams.DEFAULT_PAGE; -import static org.hisp.dhis.tracker.export.enrollment.EnrollmentOperationParams.DEFAULT_PAGE_SIZE; import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.parseFilters; import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateDeprecatedParameter; import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateDeprecatedUidsParameter; import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateOrderParams; -import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateOrgUnitMode; +import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateOrgUnitModeForTrackedEntities; import java.util.List; import java.util.Map; @@ -62,6 +60,9 @@ @RequiredArgsConstructor class TrackedEntityRequestParamsMapper { + private static final int DEFAULT_PAGE = 1; + private static final int DEFAULT_PAGE_SIZE = 50; + private static final Set ORDERABLE_FIELD_NAMES = TrackedEntityMapper.ORDERABLE_FIELDS.keySet(); @@ -91,7 +92,9 @@ public TrackedEntityOperationParams map( validateDeprecatedParameter( "ouMode", requestParams.getOuMode(), "orgUnitMode", requestParams.getOrgUnitMode()); - orgUnitMode = validateOrgUnitMode(orgUnitUids, orgUnitMode); + orgUnitMode = + validateOrgUnitModeForTrackedEntities( + orgUnitUids, orgUnitMode, requestParams.getTrackedEntities()); Set trackedEntities = validateDeprecatedUidsParameter( diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/view/Event.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/view/Event.java index f347f8a1a531..8a84af9d3d03 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/view/Event.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/view/Event.java @@ -80,7 +80,11 @@ public class Event { @JsonProperty private String storedBy; - @JsonProperty private boolean followup; + @JsonProperty private boolean followUp; + + @Deprecated(since = "2.41", forRemoval = true) + @JsonProperty("followup") + private boolean legacyFollowUp; @JsonProperty private boolean deleted; diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/view/Relationship.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/view/Relationship.java index 53af8aa5b4b0..8ed547ae46d8 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/view/Relationship.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/view/Relationship.java @@ -59,6 +59,8 @@ public class Relationship { @JsonProperty private Instant createdAt; + @JsonProperty private Instant createdAtClient; + @JsonProperty private Instant updatedAt; @JsonProperty private boolean bidirectional; diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/view/RelationshipItem.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/view/RelationshipItem.java index f4f4415b04be..72e7eb2bd250 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/view/RelationshipItem.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/view/RelationshipItem.java @@ -174,7 +174,11 @@ public static class Event { @JsonProperty private String storedBy; - @JsonProperty private boolean followup; + @JsonProperty private boolean followUp; + + @Deprecated(since = "2.41", forRemoval = true) + @JsonProperty("followup") + private boolean legacyFollowUp; @JsonProperty private boolean deleted; diff --git a/dhis-2/dhis-web-api/src/main/resources/openapi/EventsExportController.md b/dhis-2/dhis-web-api/src/main/resources/openapi/EventsExportController.md index 0d01f8d07105..2fb1b0349b81 100644 --- a/dhis-2/dhis-web-api/src/main/resources/openapi/EventsExportController.md +++ b/dhis-2/dhis-web-api/src/main/resources/openapi/EventsExportController.md @@ -166,7 +166,8 @@ case-sensitive properties * `enrollment` * `enrollmentStatus` * `event` -* `followup` +* `followup` (deprecated) +* `followUp` * `occurredAt` * `orgUnit` * `program` diff --git a/dhis-2/dhis-web-api/src/main/resources/openapi/ProgramMessageController.md b/dhis-2/dhis-web-api/src/main/resources/openapi/ProgramMessageController.md new file mode 100644 index 000000000000..140f7224d084 --- /dev/null +++ b/dhis-2/dhis-web-api/src/main/resources/openapi/ProgramMessageController.md @@ -0,0 +1,65 @@ + # Send Message + +## Specific endpoints + +### `sendMessages` + +Send event or enrollment messages. + +### `sendMessages.request.description` + +Send batch of program messages + +### `getScheduledSentMessage` + +Get all of those scheduled messages which were sent successfully. + +### `getProgramMessages` + +Get program messages matching given query criteria. + +### `getProgramMessages.parameter.ou` + +Get program messages for given set of OrganisationUnits + +### `getProgramMessages.parameter.messageStatus` + +Get program messages based on message status + +### `getProgramMessages.parameter.beforeDate` + +Get program messages before given date + +## Common for all endpoints + +### `*.parameter.enrollment` + +Get program messages for given enrollment + +### `*.parameter.event` + +Get program messages for given event + +### `*.parameter.programInstace` + +**DEPRECATED as of 2.41:** Use parameter enrollment instead + +Get program messages for given enrollment + +### `*.parameter.programStageInstace` + +**DEPRECATED as of 2.41:** Use parameter event instead + +Get program messages for given event + +### `*.parameter.afterDate` + +Get program messages after given date + +### `*.parameter.page` + +Defines which page number to return. + +### `*.parameter.pageSize` + +Defines the number of elements to return for each page. \ No newline at end of file diff --git a/dhis-2/dhis-web-api/src/main/resources/openapi/RelationshipsExportController.md b/dhis-2/dhis-web-api/src/main/resources/openapi/RelationshipsExportController.md index eea0308c6ee9..e9da51e22581 100644 --- a/dhis-2/dhis-web-api/src/main/resources/openapi/RelationshipsExportController.md +++ b/dhis-2/dhis-web-api/src/main/resources/openapi/RelationshipsExportController.md @@ -52,6 +52,7 @@ Get relationships in given order. Relationships can be ordered by the following properties * `createdAt` +* `createdAtClient` Valid `sortDirection`s are `asc` and `desc`. `sortDirection` is case-insensitive. `sortDirection` defaults to `asc` for properties without explicit `sortDirection` as in `order=createdAt`. diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/RequestParamsValidatorTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/RequestParamsValidatorTest.java index 82b172f29eed..f6ca0c66d740 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/RequestParamsValidatorTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/RequestParamsValidatorTest.java @@ -33,7 +33,8 @@ import static org.hisp.dhis.webapi.controller.event.webrequest.OrderCriteria.fromOrderString; import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.parseFilters; import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateOrderParams; -import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateOrgUnitMode; +import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateOrgUnitModeForEnrollmentsAndEvents; +import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateOrgUnitModeForTrackedEntities; import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validatePaginationParameters; import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; @@ -304,7 +305,7 @@ void shouldFailWhenOrgUnitSuppliedAndOrgUnitModeDoesNotRequireOrgUnit( Exception exception = assertThrows( BadRequestException.class, - () -> validateOrgUnitMode(Set.of(UID.of(orgUnit)), orgUnitMode)); + () -> validateOrgUnitModeForEnrollmentsAndEvents(Set.of(UID.of(orgUnit)), orgUnitMode)); assertStartsWith( String.format("orgUnitMode %s cannot be used with orgUnits.", orgUnitMode), @@ -317,7 +318,7 @@ void shouldFailWhenOrgUnitSuppliedAndOrgUnitModeDoesNotRequireOrgUnit( names = {"CAPTURE", "ACCESSIBLE", "ALL"}) void shouldPassWhenNoOrgUnitSuppliedAndOrgUnitModeDoesNotRequireOrgUnit( OrganisationUnitSelectionMode orgUnitMode) { - assertDoesNotThrow(() -> validateOrgUnitMode(emptySet(), orgUnitMode)); + assertDoesNotThrow(() -> validateOrgUnitModeForEnrollmentsAndEvents(emptySet(), orgUnitMode)); } @ParameterizedTest @@ -326,7 +327,20 @@ void shouldPassWhenNoOrgUnitSuppliedAndOrgUnitModeDoesNotRequireOrgUnit( names = {"SELECTED", "DESCENDANTS", "CHILDREN"}) void shouldPassWhenOrgUnitSuppliedAndOrgUnitModeRequiresOrgUnit( OrganisationUnitSelectionMode orgUnitMode) { - assertDoesNotThrow(() -> validateOrgUnitMode(Set.of(UID.of(orgUnit)), orgUnitMode)); + assertDoesNotThrow( + () -> validateOrgUnitModeForEnrollmentsAndEvents(Set.of(UID.of(orgUnit)), orgUnitMode)); + } + + @ParameterizedTest + @EnumSource( + value = OrganisationUnitSelectionMode.class, + names = {"SELECTED", "DESCENDANTS", "CHILDREN"}) + void shouldPassWhenTrackedEntitySuppliedAndOrgUnitModeRequiresOrgUnit( + OrganisationUnitSelectionMode orgUnitMode) { + assertDoesNotThrow( + () -> + validateOrgUnitModeForTrackedEntities( + emptySet(), orgUnitMode, Set.of(UID.of(TEA_1_UID)))); } @ParameterizedTest @@ -336,13 +350,32 @@ void shouldPassWhenOrgUnitSuppliedAndOrgUnitModeRequiresOrgUnit( void shouldFailWhenNoOrgUnitSuppliedAndOrgUnitModeRequiresOrgUnit( OrganisationUnitSelectionMode orgUnitMode) { Exception exception = - assertThrows(BadRequestException.class, () -> validateOrgUnitMode(emptySet(), orgUnitMode)); + assertThrows( + BadRequestException.class, + () -> validateOrgUnitModeForEnrollmentsAndEvents(emptySet(), orgUnitMode)); assertStartsWith( String.format("At least one org unit is required for orgUnitMode: %s", orgUnitMode), exception.getMessage()); } + @ParameterizedTest + @EnumSource( + value = OrganisationUnitSelectionMode.class, + names = {"SELECTED", "DESCENDANTS", "CHILDREN"}) + void shouldFailWhenNoOrgUnitNorTrackedEntitySuppliedAndOrgUnitModeRequiresOrgUnit( + OrganisationUnitSelectionMode orgUnitMode) { + Exception exception = + assertThrows( + BadRequestException.class, + () -> validateOrgUnitModeForTrackedEntities(emptySet(), orgUnitMode, emptySet())); + + assertStartsWith( + String.format( + "At least one org unit or tracked entity is required for orgUnitMode: %s", orgUnitMode), + exception.getMessage()); + } + @Data private static class PaginationParameters implements PageRequestParams { private Integer page; diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventServiceTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventServiceTest.java index 545aac21df6c..e6c7ac15bbdc 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventServiceTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/CsvEventServiceTest.java @@ -84,7 +84,7 @@ void writeEventsWithoutDataValues() throws IOException { Event event = Event.builder() .event("BuA2R2Gr4vt") - .followup(true) + .followUp(true) .deleted(false) .status(EventStatus.ACTIVE) .build(); @@ -106,7 +106,7 @@ void writeEventsWithDataValuesIntoASingleRow() throws IOException { Event event = Event.builder() .event("BuA2R2Gr4vt") - .followup(true) + .followUp(true) .deleted(false) .status(EventStatus.ACTIVE) .dataValues(Set.of(dataValue1, dataValue2)) diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventRequestParamsMapperTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventRequestParamsMapperTest.java index 30f5d20992e3..2877d38a5d44 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventRequestParamsMapperTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventRequestParamsMapperTest.java @@ -585,6 +585,8 @@ void shouldFailWhenNoOrgUnitSuppliedAndOrgUnitModeNeedsOrgUnit( Exception exception = assertThrows(BadRequestException.class, () -> mapper.map(requestParams)); - assertStartsWith("orgUnit is required for orgUnitMode: " + orgUnitMode, exception.getMessage()); + assertStartsWith( + "At least one org unit is required for orgUnitMode: " + orgUnitMode, + exception.getMessage()); } } diff --git a/dhis-2/pom.xml b/dhis-2/pom.xml index f068274253fd..f6da20607ded 100644 --- a/dhis-2/pom.xml +++ b/dhis-2/pom.xml @@ -94,19 +94,19 @@ 0.10.1 - 5.8.7 + 5.8.8 1.1.1.RELEASE 1.70 1.9.3 - 9.36 + 9.37 3.5.2 3.5.2 5.3.30 - 2.7.16 - 2.7.3 + 2.7.17 + 2.7.4 1.1.5.RELEASE 1.3.4 2.0.7.RELEASE @@ -134,7 +134,7 @@ 2.0 - 2.15.2 + 2.15.3 2.14 1.14 @@ -166,7 +166,7 @@ 2.31.0 4.1.100.Final - 4.8.162 + 4.8.163 0.2.1.1 @@ -174,7 +174,7 @@ 1.0.0 3.0.0 2.8.0 - 1.16.1 + 1.16.2 10.0.17 @@ -193,7 +193,7 @@ 1.5 3.6.1 1.7 - 2.14.0 + 2.15.0 1.16.0 2.1.1 1.5 @@ -206,7 +206,7 @@ 1.11.5 - 2.20.0 + 2.21.1 1.7.36 @@ -218,27 +218,27 @@ 1.19.1 1.5.1 4.2.0 - 2.1.16 + 2.1.18 5.0.0 0.2.5 2.2.224 2.2 - 3.1.2 + 3.2.1 3.11.0 3.3.1 - 3.1.2 + 3.2.1 3.6.0 3.1.0 3.4.1 3.5.0 2.4.0 2.16.1 - 8.4.0 + 8.4.2 4.7.3.6 2.40.0 - 0.8.10 + 0.8.11 1.2.0 1.0.34 1.1.1