diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheck.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheck.java index 8b520145d6fe..71feb0af1b6e 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheck.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheck.java @@ -27,9 +27,16 @@ */ package org.hisp.dhis.dataintegrity; +import static java.util.Comparator.comparingLong; +import static java.util.stream.Collectors.joining; + +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import java.io.Serializable; +import java.util.Comparator; import java.util.function.Function; +import java.util.stream.Stream; +import javax.annotation.Nonnull; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; @@ -50,23 +57,58 @@ @Builder @AllArgsConstructor(access = AccessLevel.PRIVATE) public final class DataIntegrityCheck implements Serializable { - @JsonProperty private final String name; - @JsonProperty private final String displayName; + public static final Comparator FAST_TO_SLOW = + comparingLong(DataIntegrityCheck::getExecutionTimeIndicator); + @JsonProperty private final String name; + @JsonProperty private final String displayName; @JsonProperty private final String section; - + @JsonProperty private final int sectionOrder; @JsonProperty private final DataIntegritySeverity severity; - @JsonProperty private final String description; - @JsonProperty private final String introduction; - @JsonProperty private final String recommendation; - @JsonProperty private final String issuesIdType; + @JsonProperty private final boolean isSlow; + @JsonProperty private final boolean isProgrammatic; + + private long executionTime; + private int executionCount; + + public @JsonProperty Long getAverageExecutionTime() { + return executionTime <= 0L ? null : executionTime / executionCount; + } + + @JsonIgnore + long getExecutionTimeIndicator() { + Long time = getAverageExecutionTime(); + if (time != null) return time; + return isSlow ? Long.MAX_VALUE : 1000; + } + + @JsonProperty + public String getCode() { + return getCodeFromName(name); + } private final transient Function runSummaryCheck; private final transient Function runDetailsCheck; + + public DataIntegrityCheck addExecution(long time) { + executionCount++; + executionTime += time; + return this; + } + + /** + * Method that takes in a name of a {@link DataIntegrityCheck} and converts it to an acronym of + * its name using the first letter from each word e.g. my_data_integrity_check -> MDIC + */ + public static String getCodeFromName(@Nonnull String fullName) { + return Stream.of(fullName.split("_")) + .map(word -> String.valueOf(word.charAt(0)).toUpperCase()) + .collect(joining()); + } } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheckType.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheckType.java index 506dbf3e6f1b..ceec5e3e8fba 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheckType.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheckType.java @@ -91,7 +91,7 @@ public enum DataIntegrityCheckType { PROGRAM_RULE_ACTIONS_WITHOUT_DATA_OBJECT, PROGRAM_RULE_ACTIONS_WITHOUT_NOTIFICATION, PROGRAM_RULE_ACTIONS_WITHOUT_SECTION, - PROGRAM_RULE_ACTIONS_WITHOUT_STAGE; + PROGRAM_RULE_ACTIONS_WITHOUT_STAGE_ID; public String getName() { return name().toLowerCase(); diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityDetails.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityDetails.java index 616221fab9f9..040170dac0cb 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityDetails.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityDetails.java @@ -50,6 +50,8 @@ public class DataIntegrityDetails implements Serializable { @JsonUnwrapped private final DataIntegrityCheck check; + @JsonProperty private final Date startTime; + @JsonProperty private final Date finishedTime; @JsonProperty private final String error; diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityService.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityService.java index ae41cd147454..a3049d059080 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityService.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityService.java @@ -30,6 +30,7 @@ import java.util.Collection; import java.util.Map; import java.util.Set; +import javax.annotation.Nonnull; import org.hisp.dhis.scheduling.JobProgress; /** @@ -46,23 +47,39 @@ public interface DataIntegrityService { * kept for backwards compatibility until new UI exists */ @Deprecated(since = "2.38", forRemoval = true) + @Nonnull FlattenedDataIntegrityReport getReport(Set checks, JobProgress progress); /* * New generic API */ - default Collection getDataIntegrityChecks() { + default @Nonnull Collection getDataIntegrityChecks() { return getDataIntegrityChecks(Set.of()); } + @Nonnull Collection getDataIntegrityChecks(Set checks); - Map getSummaries(Set checks, long timeout); + @Nonnull + Map getSummaries(@Nonnull Set checks, long timeout); - Map getDetails(Set checks, long timeout); + @Nonnull + Map getDetails(@Nonnull Set checks, long timeout); - void runSummaryChecks(Set checks, JobProgress progress); + void runSummaryChecks(@Nonnull Set checks, JobProgress progress); - void runDetailsChecks(Set checks, JobProgress progress); + void runDetailsChecks(@Nonnull Set checks, JobProgress progress); + + @Nonnull + Set getRunningSummaryChecks(); + + @Nonnull + Set getRunningDetailsChecks(); + + @Nonnull + Set getCompletedSummaryChecks(); + + @Nonnull + Set getCompletedDetailsChecks(); } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegritySummary.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegritySummary.java index 026f5a0f287f..c4c81a7084ba 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegritySummary.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegritySummary.java @@ -44,6 +44,8 @@ public class DataIntegritySummary implements Serializable { @JsonUnwrapped private final DataIntegrityCheck check; + @JsonProperty private final Date startTime; + @JsonProperty private final Date finishedTime; @JsonProperty private final String error; diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java index d417149858f6..0b6fa378afff 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java @@ -243,7 +243,8 @@ public FlattenedDataIntegrityReport(Map detailsByN DataIntegrityCheckType.PROGRAM_RULE_ACTIONS_WITHOUT_SECTION.getName())); this.programRuleActionsWithNoStageId = mapOfRefsByDisplayNameOrUid( - detailsByName.get(DataIntegrityCheckType.PROGRAM_RULE_ACTIONS_WITHOUT_STAGE.getName())); + detailsByName.get( + DataIntegrityCheckType.PROGRAM_RULE_ACTIONS_WITHOUT_STAGE_ID.getName())); } private List listOfDisplayNameOrUid(DataIntegrityDetails details) { diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfiguration.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfiguration.java index db15fb32a66c..4af84aa4eab2 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfiguration.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfiguration.java @@ -45,6 +45,7 @@ import org.hisp.dhis.scheduling.parameters.AggregateDataExchangeJobParameters; import org.hisp.dhis.scheduling.parameters.AnalyticsJobParameters; import org.hisp.dhis.scheduling.parameters.ContinuousAnalyticsJobParameters; +import org.hisp.dhis.scheduling.parameters.DataIntegrityDetailsJobParameters; import org.hisp.dhis.scheduling.parameters.DataIntegrityJobParameters; import org.hisp.dhis.scheduling.parameters.DataSynchronizationJobParameters; import org.hisp.dhis.scheduling.parameters.DisableInactiveUsersJobParameters; @@ -341,6 +342,9 @@ public void setDelay(Integer delay) { value = TrackerTrigramIndexJobParameters.class, name = "TRACKER_SEARCH_OPTIMIZATION"), @JsonSubTypes.Type(value = DataIntegrityJobParameters.class, name = "DATA_INTEGRITY"), + @JsonSubTypes.Type( + value = DataIntegrityDetailsJobParameters.class, + name = "DATA_INTEGRITY_DETAILS"), @JsonSubTypes.Type( value = AggregateDataExchangeJobParameters.class, name = "AGGREGATE_DATA_EXCHANGE"), diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java index 9675b3767e03..1d55765264d6 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java @@ -31,6 +31,7 @@ import org.hisp.dhis.scheduling.parameters.AggregateDataExchangeJobParameters; import org.hisp.dhis.scheduling.parameters.AnalyticsJobParameters; import org.hisp.dhis.scheduling.parameters.ContinuousAnalyticsJobParameters; +import org.hisp.dhis.scheduling.parameters.DataIntegrityDetailsJobParameters; import org.hisp.dhis.scheduling.parameters.DataIntegrityJobParameters; import org.hisp.dhis.scheduling.parameters.DataSynchronizationJobParameters; import org.hisp.dhis.scheduling.parameters.DisableInactiveUsersJobParameters; @@ -61,6 +62,11 @@ public enum JobType { SchedulingType.CRON, DataIntegrityJobParameters.class, Map.of("checks", "/api/dataIntegrity")), + DATA_INTEGRITY_DETAILS( + true, + SchedulingType.CRON, + DataIntegrityDetailsJobParameters.class, + Map.of("checks", "/api/dataIntegrity")), RESOURCE_TABLE(true), ANALYTICS_TABLE( true, diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityDetailsJobParameters.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityDetailsJobParameters.java new file mode 100644 index 000000000000..fbd9b5010f4d --- /dev/null +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityDetailsJobParameters.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.scheduling.parameters; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import java.util.Set; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.hisp.dhis.common.DxfNamespaces; +import org.hisp.dhis.scheduling.JobParameters; + +/** + * @author Jan Bernitt + */ +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class DataIntegrityDetailsJobParameters implements JobParameters { + + @JsonProperty(required = false) + @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0) + private Set checks; +} diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityJobParameters.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityJobParameters.java index a9261e991f28..3241a55639f6 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityJobParameters.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityJobParameters.java @@ -30,7 +30,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import java.util.Set; +import lombok.AllArgsConstructor; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; import org.hisp.dhis.common.DxfNamespaces; import org.hisp.dhis.scheduling.JobParameters; @@ -40,6 +42,8 @@ */ @Getter @Setter +@NoArgsConstructor +@AllArgsConstructor public class DataIntegrityJobParameters implements JobParameters { public enum DataIntegrityReportType { REPORT, @@ -49,9 +53,9 @@ public enum DataIntegrityReportType { @JsonProperty(required = false) @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0) - private Set checks; + private DataIntegrityReportType type; @JsonProperty(required = false) @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0) - private DataIntegrityReportType type; + private Set checks; } diff --git a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityCheckTest.java b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityCheckTest.java new file mode 100644 index 000000000000..b37394df49a4 --- /dev/null +++ b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityCheckTest.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.dataintegrity; + +import static java.util.stream.Collectors.toList; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import org.junit.jupiter.api.Test; + +/** + * @author Jan Bernitt + */ +class DataIntegrityCheckTest { + + @Test + void testSortingFastToSlow_NoExecutionTime() { + List checks = + List.of( + DataIntegrityCheck.builder().name("a").isSlow(true).build(), + DataIntegrityCheck.builder().name("b").isSlow(false).build(), + DataIntegrityCheck.builder().name("c").isSlow(true).build()); + List actual = + checks.stream().sorted(DataIntegrityCheck.FAST_TO_SLOW).collect(toList()); + assertEquals( + List.of("b", "a", "c"), actual.stream().map(DataIntegrityCheck::getName).collect(toList())); + } + + @Test + void testSortingFastToSlow() { + List checks = + List.of( + DataIntegrityCheck.builder().name("a").isSlow(true).build().addExecution(12345), + DataIntegrityCheck.builder().name("b").isSlow(false).build().addExecution(20), + DataIntegrityCheck.builder().name("c").isSlow(true).build().addExecution(500)); + List actual = + checks.stream().sorted(DataIntegrityCheck.FAST_TO_SLOW).collect(toList()); + assertEquals( + List.of("b", "c", "a"), actual.stream().map(DataIntegrityCheck::getName).collect(toList())); + } + + @Test + void testSortingFastToSlow_MixedExecutionTime() { + List checks = + List.of( + DataIntegrityCheck.builder().name("a").isSlow(true).build().addExecution(12345), + DataIntegrityCheck.builder().name("b").isSlow(false).build(), + DataIntegrityCheck.builder().name("c").isSlow(true).build().addExecution(500)); + List actual = + checks.stream().sorted(DataIntegrityCheck.FAST_TO_SLOW).collect(toList()); + assertEquals( + List.of("c", "b", "a"), actual.stream().map(DataIntegrityCheck::getName).collect(toList())); + } + + @Test + void testAddExecution() { + DataIntegrityCheck check = DataIntegrityCheck.builder().name("a").build(); + + check.addExecution(1000).addExecution(1000); // sum is 2000, 2 times + + assertEquals(1000L, check.getAverageExecutionTime()); + + check.addExecution(4000); // sum is now 6000, 3 times + + assertEquals(2000L, check.getAverageExecutionTime()); + } +} diff --git a/dhis-2/dhis-services/dhis-service-administration/pom.xml b/dhis-2/dhis-services/dhis-service-administration/pom.xml index 9b51db467c4e..d50c8de20882 100644 --- a/dhis-2/dhis-services/dhis-service-administration/pom.xml +++ b/dhis-2/dhis-services/dhis-service-administration/pom.xml @@ -139,6 +139,25 @@ org.apache.commons commons-lang3 + + com.google.code.findbugs + jsr305 + + + com.networknt + json-schema-validator + 1.0.87 + + + org.apache.commons + commons-lang3 + + + + + org.apache.commons + commons-collections4 + diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReader.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReader.java index 4da6475b0452..c1a6ced00264 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReader.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReader.java @@ -29,15 +29,20 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.ValidationMessage; +import java.io.InputStream; import java.nio.file.Path; import java.util.List; -import java.util.function.BinaryOperator; -import java.util.function.Consumer; -import java.util.function.Function; +import java.util.Set; import lombok.extern.slf4j.Slf4j; +import org.hisp.dhis.jsonschema.JsonSchemaValidator; +import org.springframework.core.io.AbstractFileResolvingResource; import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.UrlResource; /** * Reads {@link DataIntegrityCheck}s from YAML files. @@ -62,6 +67,9 @@ static class CheckYamlFile { @JsonProperty String section; + @JsonProperty("section_order") + int sectionOrder; + @JsonProperty("summary_sql") String summarySql; @@ -71,6 +79,9 @@ static class CheckYamlFile { @JsonProperty("details_id_type") String detailsIdType; + @JsonProperty("is_slow") + boolean isSlow; + @JsonProperty String introduction; @JsonProperty String recommendation; @@ -79,49 +90,151 @@ static class CheckYamlFile { } public static void readDataIntegrityYaml( - String listFile, - Consumer adder, - BinaryOperator info, - Function> sqlToSummary, - Function> sqlToDetails) { - + DefaultDataIntegrityService.DataIntegrityRecord dataIntegrityRecord) { ObjectMapper yaml = new ObjectMapper(new YAMLFactory()); ListYamlFile file; - try { - file = yaml.readValue(new ClassPathResource(listFile).getInputStream(), ListYamlFile.class); + + AbstractFileResolvingResource resource = + getResourceFromType( + dataIntegrityRecord.resourceLocation(), dataIntegrityRecord.yamlFileChecks()); + + if (resource == null) { + log.warn( + "Failed to get resource from location `{}` and with file`{}`", + dataIntegrityRecord.resourceLocation(), + dataIntegrityRecord.yamlFileChecks()); + return; + } + + try (InputStream is = resource.getInputStream()) { + file = yaml.readValue(is, ListYamlFile.class); } catch (Exception ex) { - log.warn("Failed to load data integrity checks from YAML", ex); + log.warn( + "Failed to load data integrity check from YAML. Error message `{}`", ex.getMessage()); return; } - for (String checkFile : file.checks) { - try { - String path = - Path.of(listFile) - .resolve("..") - .resolve("data-integrity-checks") - .resolve(checkFile) - .toString(); - CheckYamlFile e = - yaml.readValue(new ClassPathResource(path).getInputStream(), CheckYamlFile.class); - - String name = e.name.trim(); - adder.accept( + + file.checks.forEach( + dataIntegrityCheckFile -> processFile(dataIntegrityCheckFile, yaml, dataIntegrityRecord)); + } + + /** + * This method processes a {@link DataIntegrityCheck} file by: + * + *
    + *
  1. resolving the file path + *
  2. converting file to {@link JsonNode} + *
  3. validate JsonNode against {@link JsonSchema} + *
  4. Then either + *
      + *
    1. add {@link DataIntegrityCheck} to map if no validation errors or + *
    2. log a warning if any validation errors and do not add {@link DataIntegrityCheck} to + * map + *
    + *
+ * + * @param dataIntegrityCheckFile the yaml file version of a {@link DataIntegrityCheck} + * @param yaml object mapper + * @param dataIntegrityRecord record used for storing {@link DataIntegrityCheck} details used for + * processing + */ + private static void processFile( + String dataIntegrityCheckFile, + ObjectMapper yaml, + DefaultDataIntegrityService.DataIntegrityRecord dataIntegrityRecord) { + + try { + String path = + Path.of(dataIntegrityRecord.yamlFileChecks()) + .resolve("..") + .resolve(dataIntegrityRecord.checksDir()) + .resolve(dataIntegrityCheckFile) + .toString(); + + AbstractFileResolvingResource resource = + getResourceFromType(dataIntegrityRecord.resourceLocation(), path); + + if (resource == null) { + log.warn( + "Failed to get resource from location `{}` and with file`{}`", + dataIntegrityRecord.resourceLocation(), + path); + return; + } + + try (InputStream is = resource.getInputStream()) { + + JsonNode jsonNode = yaml.readValue(is, JsonNode.class); + Set validationMessages = + JsonSchemaValidator.validateDataIntegrityCheck(jsonNode); + + if (validationMessages.isEmpty()) { + CheckYamlFile yamlFile = yaml.convertValue(jsonNode, CheckYamlFile.class); + acceptDataIntegrityCheck(dataIntegrityRecord, yamlFile); + } else { + log.warn( + "JsonSchema validation errors found for Data Integrity Check `{}`. Errors: {}", + dataIntegrityCheckFile, + validationMessages); + } + } + } catch (Exception ex) { + log.error( + "Failed to load data integrity check `{}` with error message {}", + dataIntegrityCheckFile, + ex.getMessage()); + } + } + + private static void acceptDataIntegrityCheck( + DefaultDataIntegrityService.DataIntegrityRecord dataIntegrityRecord, CheckYamlFile yamlFile) { + String name = yamlFile.name.trim(); + dataIntegrityRecord + .adder() + .accept( DataIntegrityCheck.builder() .name(name) - .displayName(info.apply(name + ".name", name.replace('_', ' '))) - .description(info.apply(name + ".description", trim(e.description))) - .introduction(info.apply(name + ".introduction", trim(e.introduction))) - .recommendation(info.apply(name + ".recommendation", trim(e.recommendation))) - .issuesIdType(trim(e.detailsIdType)) - .section(trim(e.section)) - .severity(e.severity) - .runSummaryCheck(sqlToSummary.apply(sanitiseSQL(e.summarySql))) - .runDetailsCheck(sqlToDetails.apply(sanitiseSQL(e.detailsSql))) + .displayName( + dataIntegrityRecord.info().apply(name + ".name", name.replace('_', ' '))) + .description( + dataIntegrityRecord + .info() + .apply(name + ".description", trim(yamlFile.description))) + .introduction( + dataIntegrityRecord + .info() + .apply(name + ".introduction", trim(yamlFile.introduction))) + .recommendation( + dataIntegrityRecord + .info() + .apply(name + ".recommendation", trim(yamlFile.recommendation))) + .issuesIdType(trim(yamlFile.detailsIdType)) + .section(trim(yamlFile.section)) + .sectionOrder(yamlFile.sectionOrder) + .severity(yamlFile.severity) + .isSlow(yamlFile.isSlow) + .runSummaryCheck( + dataIntegrityRecord.sqlToSummary().apply(sanitiseSQL(yamlFile.summarySql))) + .runDetailsCheck( + dataIntegrityRecord.sqlToDetails().apply(sanitiseSQL(yamlFile.detailsSql))) .build()); - } catch (Exception ex) { - log.error("Failed to load data integrity check " + checkFile, ex); - } + } + + private static AbstractFileResolvingResource getResourceFromType( + ResourceLocation resourceLocation, String filePath) { + AbstractFileResolvingResource resource = null; + try { + resource = + resourceLocation == ResourceLocation.CLASS_PATH + ? new ClassPathResource(filePath) + : new UrlResource("file://" + filePath); + } catch (Exception ex) { + log.warn( + "Failed to load data integrity checks from YAML file path `{}`. Error message: {}", + filePath, + ex.getMessage()); } + return resource; } private static String trim(String str) { @@ -140,4 +253,9 @@ private static String sanitiseSQL(String sql) { .replace("::varchar", "") .replace("|| '%'", "")); } + + enum ResourceLocation { + CLASS_PATH, + FILE_SYSTEM + } } diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java index aa2e5e876346..d52e878348ef 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2022, University of Oslo + * Copyright (c) 2004-2023, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,14 +30,16 @@ import static java.lang.String.format; import static java.lang.System.currentTimeMillis; import static java.util.Collections.unmodifiableCollection; -import static java.util.Collections.unmodifiableSet; +import static java.util.function.Predicate.not; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toUnmodifiableList; import static java.util.stream.Collectors.toUnmodifiableSet; +import static java.util.stream.StreamSupport.stream; import static org.hisp.dhis.commons.collection.ListUtils.getDuplicates; import static org.hisp.dhis.dataintegrity.DataIntegrityDetails.DataIntegrityIssue.toIssue; import static org.hisp.dhis.dataintegrity.DataIntegrityDetails.DataIntegrityIssue.toRefsList; +import static org.hisp.dhis.dataintegrity.DataIntegrityYamlReader.ResourceLocation.CLASS_PATH; +import static org.hisp.dhis.dataintegrity.DataIntegrityYamlReader.ResourceLocation.FILE_SYSTEM; import static org.hisp.dhis.dataintegrity.DataIntegrityYamlReader.readDataIntegrityYaml; import static org.hisp.dhis.expression.ParseType.INDICATOR_EXPRESSION; import static org.hisp.dhis.expression.ParseType.VALIDATION_RULE_EXPRESSION; @@ -57,15 +59,20 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.BiFunction; import java.util.function.BinaryOperator; +import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; +import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.annotation.Nonnull; import javax.annotation.PostConstruct; import lombok.RequiredArgsConstructor; +import lombok.Value; +import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.hisp.dhis.antlr.ParserException; import org.hisp.dhis.cache.Cache; import org.hisp.dhis.cache.CacheProvider; @@ -82,6 +89,8 @@ import org.hisp.dhis.expression.Expression; import org.hisp.dhis.expression.ExpressionService; import org.hisp.dhis.expression.ExpressionValidationOutcome; +import org.hisp.dhis.external.location.LocationManager; +import org.hisp.dhis.external.location.LocationManagerException; import org.hisp.dhis.i18n.I18n; import org.hisp.dhis.i18n.I18nManager; import org.hisp.dhis.indicator.Indicator; @@ -123,6 +132,8 @@ public class DefaultDataIntegrityService implements DataIntegrityService { private final I18nManager i18nManager; + private final LocationManager locationManager; + private final ProgramRuleService programRuleService; private final ProgramRuleActionService programRuleActionService; @@ -161,6 +172,10 @@ public class DefaultDataIntegrityService implements DataIntegrityService { private Cache detailsCache; + private final Set runningSummaryChecks = ConcurrentHashMap.newKeySet(); + + private final Set runningDetailsChecks = ConcurrentHashMap.newKeySet(); + @PostConstruct public void init() { summaryCache = cacheProvider.createDataIntegritySummaryCache(); @@ -176,7 +191,7 @@ private static List toSimpleIssueList( return items .map(DataIntegrityIssue::toIssue) .sorted(DefaultDataIntegrityService::alphabeticalOrder) - .collect(toUnmodifiableList()); + .collect(toList()); } private static List toIssueList( @@ -184,7 +199,31 @@ private static List toIssueLi return items .map(e -> DataIntegrityIssue.toIssue(e, toRefs.apply(e))) .sorted(DefaultDataIntegrityService::alphabeticalOrder) - .collect(toUnmodifiableList()); + .collect(toList()); + } + + @Nonnull + @Override + public Set getRunningSummaryChecks() { + return Set.copyOf(runningSummaryChecks); + } + + @Nonnull + @Override + public Set getRunningDetailsChecks() { + return Set.copyOf(runningDetailsChecks); + } + + @Nonnull + @Override + public Set getCompletedSummaryChecks() { + return stream(summaryCache.keys().spliterator(), false).collect(toUnmodifiableSet()); + } + + @Nonnull + @Override + public Set getCompletedDetailsChecks() { + return stream(detailsCache.keys().spliterator(), false).collect(toUnmodifiableSet()); } // ------------------------------------------------------------------------- @@ -391,7 +430,7 @@ List getDuplicatePeriods() { null, group.getValue().stream() .map(p -> p.toString() + ":" + p.getUid()) - .collect(toUnmodifiableList()))); + .collect(toList()))); } } return issues; @@ -457,7 +496,6 @@ private List getInvalidValidationRuleExpressions( issues.add(toIssue(rule, i18n.getString(result.getKey()))); } } - return issues; } @@ -474,11 +512,13 @@ private void registerNonDatabaseIntegrityCheck( try { Schema issueSchema = issueIdType == null ? null : schemaService.getDynamicSchema(issueIdType); String issueIdTypeName = issueSchema == null ? null : issueSchema.getPlural(); - checksByName.put( + checksByName.putIfAbsent( name, DataIntegrityCheck.builder() .name(name) .displayName(info.apply("name", name.replace('_', ' '))) + .isSlow(true) + .isProgrammatic(true) .severity( DataIntegritySeverity.valueOf( info.apply("severity", DataIntegritySeverity.WARNING.name()).toUpperCase())) @@ -487,9 +527,19 @@ private void registerNonDatabaseIntegrityCheck( .introduction(info.apply("introduction", null)) .recommendation(info.apply("recommendation", null)) .issuesIdType(issueIdTypeName) - .runDetailsCheck(c -> new DataIntegrityDetails(c, new Date(), null, check.get())) + .runDetailsCheck( + c -> { + Date startTime = new Date(); + List issues = check.get(); + return new DataIntegrityDetails(c, startTime, new Date(), null, issues); + }) .runSummaryCheck( - c -> new DataIntegritySummary(c, new Date(), null, check.get().size(), null)) + c -> { + Date startTime = new Date(); + List issues = check.get(); + return new DataIntegritySummary( + c, startTime, new Date(), null, issues.size(), null); + }) .build()); } catch (Exception ex) { log.error("Failed to register data integrity check " + type, ex); @@ -589,11 +639,11 @@ public void initIntegrityChecks() { DataIntegrityCheckType.VALIDATION_RULES_WITH_INVALID_RIGHT_SIDE_EXPRESSION, ValidationRule.class, this::getInvalidValidationRuleRightSideExpressions); - registerNonDatabaseIntegrityCheck( DataIntegrityCheckType.PROGRAM_INDICATORS_WITH_INVALID_EXPRESSIONS, ProgramIndicator.class, this::getInvalidProgramIndicatorExpressions); + registerNonDatabaseIntegrityCheck( DataIntegrityCheckType.PROGRAM_INDICATORS_WITH_INVALID_FILTERS, ProgramIndicator.class, @@ -638,11 +688,12 @@ public void initIntegrityChecks() { ProgramRule.class, this::getProgramRuleActionsWithNoSectionId); registerNonDatabaseIntegrityCheck( - DataIntegrityCheckType.PROGRAM_RULE_ACTIONS_WITHOUT_STAGE, + DataIntegrityCheckType.PROGRAM_RULE_ACTIONS_WITHOUT_STAGE_ID, ProgramRule.class, this::getProgramRuleActionsWithNoProgramStageId); } + @Nonnull @Override @Transactional(readOnly = true) public FlattenedDataIntegrityReport getReport(Set checks, JobProgress progress) { @@ -785,7 +836,7 @@ private static List groupActionsByProgramRule( List groupBy(Function property, Collection values) { return values.stream().collect(groupingBy(property)).entrySet().stream() .map(e -> DataIntegrityIssue.toIssue(e.getKey(), e.getValue())) - .collect(toUnmodifiableList()); + .collect(toList()); } /* @@ -796,6 +847,7 @@ List groupBy(Function property, Collection values) private final AtomicBoolean configurationsAreLoaded = new AtomicBoolean(false); + @Nonnull @Override public Collection getDataIntegrityChecks(Set checks) { ensureConfigurationsAreLoaded(); @@ -804,40 +856,54 @@ public Collection getDataIntegrityChecks(Set checks) : expandChecks(checks).stream().map(checksByName::get).collect(toList()); } + @Nonnull @Override - public Map getSummaries(Set checks, long timeout) { + public Map getSummaries(@Nonnull Set checks, long timeout) { return getCached(checks, timeout, summaryCache); } // OBS! We intentionally do not open the transaction here to have each check // be independent @Override - public void runSummaryChecks(Set checks, JobProgress progress) { + public void runSummaryChecks(@Nonnull Set checks, JobProgress progress) { runDataIntegrityChecks( "Data Integrity summary checks", expandChecks(checks), progress, summaryCache, + runningSummaryChecks, check -> check.getRunSummaryCheck().apply(check), - (check, ex) -> new DataIntegritySummary(check, new Date(), ex.getMessage(), -1, null)); + (check, startTime, ex) -> + new DataIntegritySummary( + check, startTime, new Date(), errorMessage(check, ex), -1, null)); } + @Nonnull @Override - public Map getDetails(Set checks, long timeout) { + public Map getDetails(@Nonnull Set checks, long timeout) { return getCached(checks, timeout, detailsCache); } // OBS! We intentionally do not open the transaction here to have each check // be independent @Override - public void runDetailsChecks(Set checks, JobProgress progress) { + public void runDetailsChecks(@Nonnull Set checks, JobProgress progress) { runDataIntegrityChecks( "Data Integrity details checks", expandChecks(checks), progress, detailsCache, + runningDetailsChecks, check -> check.getRunDetailsCheck().apply(check), - (check, ex) -> new DataIntegrityDetails(check, new Date(), ex.getMessage(), List.of())); + (check, startTime, ex) -> + new DataIntegrityDetails( + check, startTime, new Date(), errorMessage(check, ex), List.of())); + } + + private static String errorMessage(DataIntegrityCheck check, RuntimeException ex) { + String message = "Check failed because an exception was thrown: " + ex.getMessage(); + log.error("Check " + check.getName() + " failed because an exception was thrown", ex); + return message; } private Map getCached(Set checks, long timeout, Cache cache) { @@ -864,69 +930,171 @@ private Map getCached(Set checks, long timeout, Cache return resByName; } + @FunctionalInterface + private interface DataIntegrityCheckErrorHandler { + T createErrorReport(DataIntegrityCheck check, Date startTime, RuntimeException ex); + } + private void runDataIntegrityChecks( String stageDesc, Set checks, JobProgress progress, Cache cache, + Set running, Function runCheck, - BiFunction createErrorReport) { - progress.startingProcess("Data Integrity check"); - progress.startingStage(stageDesc, checks.size(), SKIP_ITEM); - progress.runStage( - checks.stream().map(checksByName::get).filter(Objects::nonNull), - DataIntegrityCheck::getDescription, - check -> { - T res = null; - try { - res = runCheck.apply(check); - } catch (RuntimeException ex) { - cache.put(check.getName(), createErrorReport.apply(check, ex)); - throw ex; - } - if (res != null) { - cache.put(check.getName(), res); - } - }); - progress.completedProcess(null); + DataIntegrityCheckErrorHandler createErrorReport) { + try { + running.addAll(checks); + progress.startingProcess("Data Integrity check"); + progress.startingStage(stageDesc, checks.size(), SKIP_ITEM); + progress.runStage( + checks.stream() + .map(checksByName::get) + .filter(Objects::nonNull) + .sorted(DataIntegrityCheck.FAST_TO_SLOW), + DataIntegrityCheck::getDescription, + check -> { + Date startTime = new Date(); + T res; + try { + res = runCheck.apply(check); + } catch (RuntimeException ex) { + cache.put(check.getName(), createErrorReport.createErrorReport(check, startTime, ex)); + throw ex; + } finally { + running.remove(check.getName()); + } + if (res != null) { + check.addExecution(currentTimeMillis() - startTime.getTime()); + cache.put(check.getName(), res); + } + }); + progress.completedProcess(null); + } finally { + running.removeAll(checks); + } } private Set expandChecks(Set names) { ensureConfigurationsAreLoaded(); - if (names == null || names.isEmpty()) { - return unmodifiableSet(checksByName.keySet()); + + if (CollectionUtils.isEmpty(names)) { + return getDefaultChecks(); } Set expanded = new LinkedHashSet<>(); + for (String name : names) { - String uniformName = name.toLowerCase().replace('-', '_'); - if (uniformName.contains("*")) { - String pattern = uniformName.replace("*", ".*"); - for (String existingName : checksByName.keySet()) { - if (existingName.matches(pattern)) { - expanded.add(existingName); + if (name.toUpperCase().equals(name) && name.indexOf('_') < 0) { + // assume it is a code + checksByName.values().stream() + .filter(check -> check.getCode().equals(name)) + .map(DataIntegrityCheck::getName) + .forEach(expanded::add); + } else if (name.contains("*")) { + String pattern = + name.toLowerCase() + .replace('-', '_') // make uniform + .replaceAll("[^*_a-z0-9]+", "") // sanitise against regex attacks + .replace("*", ".*"); // expand regex wildcard match + for (DataIntegrityCheck check : checksByName.values()) { + if (check.getName().matches(pattern)) { + expanded.add(check.getName()); } } } else { - expanded.add(uniformName); + expanded.add(name.toLowerCase().replace('-', '_')); } } return expanded; } + private Set getDefaultChecks() { + ensureConfigurationsAreLoaded(); + + return checksByName.values().stream() + .filter(not(DataIntegrityCheck::isSlow)) + .map(DataIntegrityCheck::getName) + .collect(Collectors.toUnmodifiableSet()); + } + private void ensureConfigurationsAreLoaded() { if (configurationsAreLoaded.compareAndSet(false, true)) { + // load system-packaged data integrity checks + loadChecks(CLASS_PATH, "data-integrity-checks.yaml", "data-integrity-checks"); + // programmatic checks initIntegrityChecks(); - // YAML based checks - I18n i18n = i18nManager.getI18n(DataIntegrityService.class); - readDataIntegrityYaml( - "data-integrity-checks.yaml", - check -> checksByName.put(check.getName(), check), - (property, defaultValue) -> - i18n.getString(format("data_integrity.%s", property), defaultValue), - sql -> check -> dataIntegrityStore.querySummary(check, sql), - sql -> check -> dataIntegrityStore.queryDetails(check, sql)); + // load user-packaged custom data integrity checks + try { + String dhis2Home = locationManager.getExternalDirectoryPath(); + loadChecks( + FILE_SYSTEM, + dhis2Home + "/custom-data-integrity-checks.yaml", + dhis2Home + "/custom-data-integrity-checks"); + } catch (LocationManagerException ex) { + log.warn( + "Could not get DHIS2_HOME external directory. No custom data integrity checks loaded."); + } } } + + private void loadChecks( + DataIntegrityYamlReader.ResourceLocation resourceLocation, + String yamlFileChecks, + String checksDir) { + I18n i18n = i18nManager.getI18n(DataIntegrityService.class); + readDataIntegrityYaml( + new DataIntegrityRecord( + resourceLocation, + yamlFileChecks, + checksDir, + addToChecks, + (property, defaultValue) -> + i18n.getString(format("data_integrity.%s", property), defaultValue), + sql -> check -> dataIntegrityStore.querySummary(check, sql), + sql -> check -> dataIntegrityStore.queryDetails(check, sql))); + } + + /** + * Consumer that adds a {@link DataIntegrityCheck} to a map. It only adds the {@link + * DataIntegrityCheck} to the map if: + * + *
    + *
  1. the {@link DataIntegrityCheck} code is unique + *
  2. the map does not already have a key with the same {@link DataIntegrityCheck} name + *
+ */ + private final Consumer addToChecks = + check -> { + String checkCode = DataIntegrityCheck.getCodeFromName(check.getName()); + Set checkCodes = + checksByName.keySet().stream() + .map(DataIntegrityCheck::getCodeFromName) + .collect(Collectors.toSet()); + if (!checkCodes.contains(checkCode)) { + DataIntegrityCheck dataIntegrityCheck = checksByName.putIfAbsent(check.getName(), check); + if (dataIntegrityCheck != null) { + log.warn( + "Data Integrity Check `{}` not added as a check with that name already exists", + check.getName()); + } + } else + log.warn( + "Data Integrity Check `{}` not added as a check with the code `{}` already exists", + check.getName(), + check.getCode()); + }; + + @Value + @Accessors(fluent = true) + static class DataIntegrityRecord { + DataIntegrityYamlReader.ResourceLocation resourceLocation; + String yamlFileChecks; + String checksDir; + Consumer adder; + BinaryOperator info; + Function> sqlToSummary; + Function> sqlToDetails; + } } diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/hibernate/HibernateDataIntegrityStore.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/hibernate/HibernateDataIntegrityStore.java index 8b985a71f69d..dc98614ed971 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/hibernate/HibernateDataIntegrityStore.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/hibernate/HibernateDataIntegrityStore.java @@ -31,7 +31,7 @@ import java.util.Date; import java.util.List; -import lombok.AllArgsConstructor; +import lombok.RequiredArgsConstructor; import org.hibernate.SessionFactory; import org.hisp.dhis.dataintegrity.DataIntegrityCheck; import org.hisp.dhis.dataintegrity.DataIntegrityDetails; @@ -48,25 +48,28 @@ * @author Jan Bernitt */ @Repository -@AllArgsConstructor +@RequiredArgsConstructor public class HibernateDataIntegrityStore implements DataIntegrityStore { private final SessionFactory sessionFactory; @Override @Transactional(readOnly = true) public DataIntegritySummary querySummary(DataIntegrityCheck check, String sql) { + Date startTime = new Date(); Object summary = sessionFactory.getCurrentSession().createNativeQuery(sql).getSingleResult(); return new DataIntegritySummary( - check, new Date(), null, parseCount(summary), parsePercentage(summary)); + check, startTime, new Date(), null, parseCount(summary), parsePercentage(summary)); } @Override @Transactional(readOnly = true) public DataIntegrityDetails queryDetails(DataIntegrityCheck check, String sql) { + Date startTime = new Date(); @SuppressWarnings("unchecked") List rows = sessionFactory.getCurrentSession().createNativeQuery(sql).getResultList(); return new DataIntegrityDetails( check, + startTime, new Date(), null, rows.stream() diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityDetailsJob.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityDetailsJob.java new file mode 100644 index 000000000000..d6ed5cfbd70b --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityDetailsJob.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.dataintegrity.jobs; + +import java.util.Set; +import lombok.AllArgsConstructor; +import org.hisp.dhis.dataintegrity.DataIntegrityService; +import org.hisp.dhis.scheduling.Job; +import org.hisp.dhis.scheduling.JobConfiguration; +import org.hisp.dhis.scheduling.JobProgress; +import org.hisp.dhis.scheduling.JobType; +import org.hisp.dhis.scheduling.parameters.DataIntegrityDetailsJobParameters; +import org.springframework.stereotype.Component; + +/** + * @author Jan Bernitt + */ +@Component +@AllArgsConstructor +public class DataIntegrityDetailsJob implements Job { + + private final DataIntegrityService dataIntegrityService; + + @Override + public JobType getJobType() { + return JobType.DATA_INTEGRITY_DETAILS; + } + + @Override + public void execute(JobConfiguration config, JobProgress progress) { + DataIntegrityDetailsJobParameters parameters = + (DataIntegrityDetailsJobParameters) config.getJobParameters(); + Set checks = parameters == null ? Set.of() : parameters.getChecks(); + + dataIntegrityService.runDetailsChecks(checks, progress); + } +} diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityJob.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityJob.java index 9b1a95c0083f..9e418a69b1cb 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityJob.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityJob.java @@ -39,12 +39,12 @@ import org.hisp.dhis.scheduling.JobType; import org.hisp.dhis.scheduling.parameters.DataIntegrityJobParameters; import org.hisp.dhis.scheduling.parameters.DataIntegrityJobParameters.DataIntegrityReportType; -import org.hisp.dhis.system.notification.NotificationLevel; import org.hisp.dhis.system.notification.Notifier; import org.springframework.stereotype.Component; /** - * @author Halvdan Hoem Grelland + * @author Halvdan Hoem Grelland (original) + * @author Jan Bernitt (refactored) */ @Component @AllArgsConstructor @@ -75,18 +75,14 @@ public void execute(JobConfiguration config, JobProgress progress) { private void runReport(JobConfiguration config, JobProgress progress, Set checks) { Timer timer = new SystemTimer().start(); - notifier.notify(config, NotificationLevel.INFO, "Starting data integrity job", false); + notifier.notify(config, "Starting data integrity job"); FlattenedDataIntegrityReport report = dataIntegrityService.getReport(checks, progress); timer.stop(); notifier - .notify( - config, - NotificationLevel.INFO, - "Data integrity checks completed in " + timer + ".", - true) + .notify(config, "Data integrity checks completed in " + timer + ".", true) .addJobSummary(config, report, FlattenedDataIntegrityReport.class); } } diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/jsonschema/JsonSchemaValidator.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/jsonschema/JsonSchemaValidator.java new file mode 100644 index 000000000000..27f6c13ad656 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/jsonschema/JsonSchemaValidator.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.jsonschema; + +import com.fasterxml.jackson.databind.JsonNode; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.JsonSchemaFactory; +import com.networknt.schema.SpecVersion; +import com.networknt.schema.ValidationMessage; +import java.io.IOException; +import java.io.InputStream; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.io.ClassPathResource; + +@Slf4j +public class JsonSchemaValidator { + + private static final String DATA_INTEGRITY_CHECK_DIR = "data-integrity-checks/"; + private static final String DATA_INTEGRITY_CHECK_SCHEMA_FILE = "integrity_check_schema.json"; + private static final String DATA_INTEGRITY_CHECK_SCHEMA = + DATA_INTEGRITY_CHECK_DIR + DATA_INTEGRITY_CHECK_SCHEMA_FILE; + private static JsonSchema dataIntegritySchema; + + static { + JsonSchemaFactory schemaFactory = + JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012); + try (InputStream is = new ClassPathResource(DATA_INTEGRITY_CHECK_SCHEMA).getInputStream()) { + dataIntegritySchema = schemaFactory.getSchema(is); + } catch (IOException e) { + log.error( + "Error loading data integrity check schema at class path location {}. Error message: {}", + DATA_INTEGRITY_CHECK_SCHEMA, + e.getMessage()); + } + } + + private JsonSchemaValidator() { + throw new UnsupportedOperationException("util"); + } + + public static Set validateDataIntegrityCheck(JsonNode json) { + return dataIntegritySchema.validate(json); + } +} diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks.yaml index 38990c2824cf..42908a8c8a71 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks.yaml @@ -5,3 +5,60 @@ checks: - categories/categories_one_default_category_combo.yaml - categories/categories_one_default_category_option_combo.yaml - categories/categories_unique_category_combo.yaml + - categories/category_combos_unused.yaml + - categories/categories_same_category_options.yaml + - categories/category_option_combinations_cardinality.yaml + - categories/categories_shared_category_options_in_combo.yaml + - categories/category_option_combinations_disjoint.yaml + - categories/category_options_no_categories.yaml + - categories/coc_no_category_combo.yaml + - categories/category_option_groups_excess_members.yaml + - categories/category_option_groups_sets_incomplete.yaml + - datasets/datasets_empty.yaml + - orgunits/compulsory_orgunit_groups.yaml + - orgunits/orgunit_open_date_gt_closed_date.yaml + - orgunits/orgunits_multiple_spaces.yaml + - orgunits/orgunit_null_island.yaml + - orgunits/orgunits_multiple_roots.yaml + - orgunits/orgunits_orphaned.yaml + - orgunits/orgunits_trailing_spaces.yaml + - orgunits/orgunits_excess_group_memberships.yaml + - orgunits/orgunits_not_contained_by_parent.yaml + - orgunits/orgunits_invalid_geometry.yaml + - orgunits/orgunits_no_geometry.yaml + - orgunits/orgunits_same_name_and_parent.yaml + - option_sets/option_sets_empty.yaml + - option_sets/unused_option_sets.yaml + - option_sets/option_sets_wrong_sort_order.yaml + - program_rules/program_rules_no_action.yaml + - program_rules/program_rules_no_expression.yaml + - program_rules/program_rules_message_no_template.yaml + - analytical_objects/visualizations_not_used_1year.yaml + - analytical_objects/maps_not_used_1year.yaml + - analytical_objects/dashboards_not_used_1year.yaml + - analytical_objects/dashboards_empty.yaml + - data_elements/aggregate_des_no_groups.yaml + - data_elements/aggregate_des_abandoned.yaml + - data_elements/aggregate_des_inconsistent_agg_operator.yaml + - data_elements/aggregate_des_excess_groupset_membership.yaml + - data_elements/aggregate_des_no_analysis.yaml + - data_elements/aggregate_des_datasets_different_period_types.yaml + - data_elements/aggregate_des_nodata.yaml + - indicators/indicator_duplicate_types.yaml + - indicators/indicator_nongrouped.yaml + - indicators/indicator_noanalysis.yaml + - indicators/indicator_duplicated_terms.yaml + - indicators/indicator_exact_duplicates.yaml + - periods/periods_same_start_end_date.yaml + - periods/periods_3y_future.yaml + - periods/periods_distant_past.yaml + - validation_rules/validation_rules_undefined_missing_value_strategy.yaml + - groups/group_size_data_element_groups.yaml + - groups/group_size_indicator_groups.yaml + - groups/group_size_program_indicator_groups.yaml + - groups/group_size_indicator_group_sets.yaml + - groups/group_size_validation_rule_groups.yaml + - groups/group_size_organisation_unit_groups.yaml + - groups/group_size_category_option_groups.yaml + - groups/group_size_category_option_group_sets.yaml + - groups/group_size_user_groups.yaml diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/dashboards_empty.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/dashboards_empty.yaml new file mode 100644 index 000000000000..512ad4485bad --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/dashboards_empty.yaml @@ -0,0 +1,47 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- + name: dashboards_no_items + description: Dashboards with no items. + section: Dashboards + section_order: 4 + summary_sql: >- + select count(*) as value, + 100*count(*) / NULLIF( (select count(*) from dashboard),0) as percent + from dashboard where + dashboardid not in (select dashboardid from dashboard_items); + details_sql: >- + SELECT uid,name from dashboard WHERE + dashboardid not in (select dashboardid from dashboard_items); + severity: INFO + introduction: > + All dashboards should have content on them. Dashboards without any content do not + serve any purpose, and can make it more difficult to find relevant dashboards with content. + details_id_type: dashboards + recommendation: > + Dashboards without content that have not been modified in the last 14 days should + be considered for deletion. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/dashboards_not_used_1year.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/dashboards_not_used_1year.yaml new file mode 100644 index 000000000000..3d3753207b82 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/dashboards_not_used_1year.yaml @@ -0,0 +1,69 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- + name: dashboards_not_viewed_one_year + description: Dashboards which have not been actively viewed in the past 12 months + section: Visualizations + section_order: 3 + summary_sql: >- + WITH unused_dashboards as ( + SELECT c.uid,c.name, c.last_viewed from ( + SELECT a.uid,a.name,b.last_viewed + from dashboard a + LEFT OUTER JOIN (SELECT favoriteuid, + MAX(timestamp) as last_viewed FROM datastatisticsevent + where eventtype = 'DASHBOARD_VIEW' + AND favoriteuid IS NOT NULL GROUP BY favoriteuid ) b on + a.uid = b.favoriteuid ) as c + where c.last_viewed IS NULL + OR AGE(now(),c.last_viewed) >= INTERVAL '12 months' ) + SELECT COUNT(*) as value, + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM dashboard), 0) as percent + from unused_dashboards; + details_sql: >- + SELECT c.uid,c.name, CAST(c.last_viewed as text) from ( + SELECT a.uid,a.name,b.last_viewed + from dashboard a + LEFT OUTER JOIN (SELECT favoriteuid, + MAX(timestamp) as last_viewed FROM datastatisticsevent + where eventtype = 'DASHBOARD_VIEW' + AND favoriteuid IS NOT NULL GROUP BY favoriteuid ) b on + a.uid = b.favoriteuid ) as c + where c.last_viewed IS NULL + OR AGE(now(),c.last_viewed) >= INTERVAL '12 months' + ORDER BY c.name; + severity: WARNING + introduction: > + Dashboards should be regularly viewed in the system. In many cases, users may create dashboards for + temporary purposes and then never delete them. This can eventually lead to a lack of tidiness + in the system. This can lead to useful dashboards being difficult to find. This + check identifies any dashboards which have not been viewed in the past year. + details_id_type: dashboards + recommendation: > + Unused dashboards can be deleted in the entirety from the main dashboard app by clicking on the "Edit" + button and choosing "Delete". Note that this process cannot be undone! Another option would be to alter + the sharing of the dashboard so that it is not visible to any user. \ No newline at end of file diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/maps_not_used_1year.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/maps_not_used_1year.yaml new file mode 100644 index 000000000000..365184e8716d --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/maps_not_used_1year.yaml @@ -0,0 +1,68 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- + name: maps_not_viewed_one_year + description: Maps which have not been viewed in the past 12 months + section: Visualizations + section_order: 1 + summary_sql: >- + WITH unused_maps as ( + SELECT c.uid,c.name, c.last_viewed from ( + SELECT a.uid,a.name,b.last_viewed + from map a + LEFT OUTER JOIN (SELECT favoriteuid, + MAX(timestamp) as last_viewed FROM datastatisticsevent + where eventtype = 'MAP_VIEW' + AND favoriteuid IS NOT NULL GROUP BY favoriteuid ) b on + a.uid = b.favoriteuid ) as c + where c.last_viewed IS NULL + OR AGE(now(),c.last_viewed) >= INTERVAL '12 months' ) + SELECT COUNT(*) as value, + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM map), 0) as percent + from unused_maps; + details_sql: >- + SELECT c.uid,c.name, CAST(c.last_viewed as text) from ( + SELECT a.uid,a.name,b.last_viewed + from map a + LEFT OUTER JOIN (SELECT favoriteuid, + MAX(timestamp) as last_viewed FROM datastatisticsevent + where eventtype = 'MAP_VIEW' + AND favoriteuid IS NOT NULL GROUP BY favoriteuid ) b on + a.uid = b.favoriteuid ) as c + where c.last_viewed IS NULL + OR AGE(now(),c.last_viewed) >= INTERVAL '12 months' + ORDER BY c.name; + severity: WARNING + introduction: > + Maps should be regularly viewed in the system. In many cases, users may create maps for + temporary purposes and then never delete them. This can eventually lead to a lack of tidiness + in the system and being difficult to find in the Maps app. + details_id_type: maps + recommendation: > + Unused maps can be removed directly using the Maps app by a user with sufficient + authority. If maps are a part of any dashboard however, they will also need to be removed + from the dashboard first. \ No newline at end of file diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/visualizations_not_used_1year.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/visualizations_not_used_1year.yaml new file mode 100644 index 000000000000..9dcd774b2319 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/visualizations_not_used_1year.yaml @@ -0,0 +1,67 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- + name: visualizations_not_viewed_one_year + description: Visualizations which have not been viewed in the past 12 months + section: Visualizations + section_order: 1 + summary_sql: >- + WITH unused_visualizations as ( + SELECT c.uid,c.name, c.last_viewed from ( + SELECT a.uid,a.name,b.last_viewed + from visualization a + LEFT OUTER JOIN (SELECT favoriteuid, + MAX(timestamp) as last_viewed FROM datastatisticsevent + where eventtype = 'VISUALIZATION_VIEW' + AND favoriteuid IS NOT NULL GROUP BY favoriteuid ) b on + a.uid = b.favoriteuid ) as c + where c.last_viewed IS NULL + OR AGE(now(),c.last_viewed) >= INTERVAL '12 months' ) + SELECT COUNT(*) as value, + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM visualization), 0) as percent + from unused_visualizations; + details_sql: >- + SELECT c.uid,c.name, CAST(c.last_viewed as text) from ( + SELECT a.uid,a.name,b.last_viewed + from visualization a + LEFT OUTER JOIN (SELECT favoriteuid, + MAX(timestamp) as last_viewed FROM datastatisticsevent + where eventtype = 'VISUALIZATION_VIEW' + AND favoriteuid IS NOT NULL GROUP BY favoriteuid ) b on + a.uid = b.favoriteuid ) as c + where c.last_viewed IS NULL + OR AGE(now(),c.last_viewed) >= INTERVAL '12 months'; + severity: WARNING + introduction: > + Visualizations should be regularly viewed in the system. In many cases, users may create charts for + temporary purposes and then never delete them. This can eventually lead to a lack of tidiness + in the system. This can lead to charts being difficult to find in the visualization app. + details_id_type: visualizations + recommendation: > + Unused charts can be removed directly using the data visualization app by a user with sufficient + authority. If charts are a part of any dashboard however, they will also need to be removed + from the dashboard first. \ No newline at end of file diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_no_options.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_no_options.yaml index 8b025e7e1ea6..2a5960ad6148 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_no_options.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_no_options.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2021, University of Oslo +# Copyright (c) 2004-2022, University of Oslo # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -31,7 +31,7 @@ section: Categories summary_sql: >- select COUNT(*) as value, - (100 * COUNT(*) / (SELECT COUNT(*) FROM dataelementcategory)) as percent + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM dataelementcategory), 0) as percent from dataelementcategory where categoryid not in (select distinct categoryid from categories_categoryoptions); details_sql: >- @@ -42,7 +42,8 @@ details_sql: >- details_id_type: categories severity: WARNING introduction: > - Categories should always have at least a single category options. + Categories should always have at least one category option. +section_order: 1 recommendation: > Any categories without category options should either be removed from the system if they are not in use. Otherwise, appropriate category options diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category.yaml index 0edbdd0c18a2..a8ea841e03c3 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category.yaml @@ -39,6 +39,7 @@ details_id_type: categories severity: SEVERE introduction: > There should only exist one category with name and code "default". +section_order: 2 recommendation: > Only the category with UID "GLevLNI9wkl" should be named "default" and have code "default". Either rename the conflicting category or move all references to category "GLevLNI9wkl" diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_combo.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_combo.yaml index a412fdfd50b8..5bb22da2e5bf 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_combo.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_combo.yaml @@ -39,6 +39,7 @@ details_id_type: categoryCombos severity: SEVERE introduction: > There should only exist one category combo with name and code "default". +section_order: 3 recommendation: > Only the category combo with UID "bjDvmb4bfuf" should be named "default" and have code "default". Either rename the conflicting category combo or move all references to category combo "bjDvmb4bfuf" diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option.yaml index 59c3e13f298e..72465253f354 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option.yaml @@ -39,6 +39,7 @@ details_id_type: categoryOptions severity: SEVERE introduction: > There should only exist one category option with name and code "default". +section_order: 4 recommendation: > Only the category option with UID "xYerKDKCefk" should be named "default" and have code "default". Either rename the conflicting category option or move all references to category option "xYerKDKCefk" diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option_combo.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option_combo.yaml index 9f0f7b5ea5c0..8b5fbdc76d38 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option_combo.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option_combo.yaml @@ -39,6 +39,7 @@ details_id_type: categoryOptionCombos severity: SEVERE introduction: > There should only exist one category option with name and code "default". +section_order: 4 recommendation: > Only the category option combo with UID "HllvX50cXC0" should be named "default" and have code "default". Either rename the conflicting category option combo or move all references to category option combo "HllvX50cXC0" diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_same_category_options.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_same_category_options.yaml new file mode 100644 index 000000000000..09bcc3ab03ac --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_same_category_options.yaml @@ -0,0 +1,77 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- + name: categories_same_category_options + description: Categories with the same category options + section: Categories + section_order: 6 + summary_sql: >- + WITH duplicative_categories AS ( + SELECT catoptions, COUNT(*) as count + FROM ( SELECT categoryid, + array_agg(categoryoptionid ORDER BY categoryoptionid) as catoptions + FROM categories_categoryoptions group by categoryid) as foo + GROUP BY catoptions HAVING COUNT(*) > 1 + ) + SELECT COUNT(*) as value, + 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) + FROM dataelementcategory),0 ) percent + FROM duplicative_categories; + details_sql: >- + SELECT x.uid,'(' || b.rank || ') ' || x.name as name from dataelementcategory x + INNER JOIN ( + SELECT categoryid, array_agg(categoryoptionid ORDER BY categoryoptionid) as catoptions + from categories_categoryoptions GROUP BY categoryid + ) a on x.categoryid = a.categoryid + INNER JOIN ( + SELECT catoptions, COUNT(*) as count, DENSE_RANK() OVER (ORDER BY catoptions) as rank + FROM ( SELECT categoryid, + array_agg(categoryoptionid ORDER BY categoryoptionid) as catoptions + FROM categories_categoryoptions group by categoryid) as foo + GROUP BY catoptions HAVING COUNT(*) > 1 + ) b on b.catoptions = a.catoptions + ORDER BY b.rank + severity: WARNING + introduction: > + Categories with the exact same category options should be considered + to be merged. Categories with the exact same category options + may be easily confused by users in analysis. The details view will + provide a list of categories which have the exact same category options. + For each duplicated category a number in parentheses such as (1) will + indicate which categories belong to duplicated groups. + details_id_type: categories + recommendation: | + If category combinations have already been created with duplicative categories, + it is recommended that you do not take any action, but rather ensure + that users understand that there may be two category combinations which are duplicative. + + If you choose to merge the duplicative category combinations, you would need to + remap all category option combinations from the category combo which you wish to + remove, to the one which you wish to keep. + + If one of the categories is not in use in any category combination, it should + consider to be removed from the system. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_shared_category_options_in_combo.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_shared_category_options_in_combo.yaml new file mode 100644 index 000000000000..229002d24706 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_shared_category_options_in_combo.yaml @@ -0,0 +1,68 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: category_options_shared_within_category_combo +description: Category combinations with categories which share the same category options. +section: Categories +section_order: 7 +summary_sql: >- + WITH category_option_multiple_member_category as ( + select cc.name as cc_name, co.categoryoptionid, co.name as co_name from categorycombo cc + inner join categorycombos_categories ccc on cc.categorycomboid=ccc.categorycomboid + inner join categories_categoryoptions cco on ccc.categoryid=cco.categoryid + inner join dataelementcategoryoption co on cco.categoryoptionid=co.categoryoptionid + group by cc_name, co.categoryoptionid, co_name having count(*) > 1 ) + SELECT + COUNT(*)as value, + 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM categoryoptioncombo),0) as percent + FROM category_option_multiple_member_category; +details_sql: >- + WITH category_option_multiple_member_category as ( + select cc.uid, cc.name as cc_name, co.categoryoptionid, co.name as co_name from categorycombo cc + inner join categorycombos_categories ccc on cc.categorycomboid=ccc.categorycomboid + inner join categories_categoryoptions cco on ccc.categoryid=cco.categoryid + inner join dataelementcategoryoption co on cco.categoryoptionid=co.categoryoptionid + group by cc.uid, cc_name, co.categoryoptionid, co_name having count(*) > 1 ) + SELECT uid, + cc_name as name, + co_name as comment from category_option_multiple_member_category; +severity: SEVERE +introduction: > + As a general rule, category options should be reused where possible between categories. The exception + to this rule however, is when you have a category combo with multiple categories, and within those + categories, a category option is shared. As a simple example, lets say you have a category called "Sex" with + options "Male"", "Female" and "Unknown". There is also a second category called "Age" with options "<15", "15+", + and "Unknown". A category combination called "Age/Sex" is then created with these two categories, which share + the option "Unknown". This situation should be avoided, as it creates issues when analyzing data. +details_id_type: categoryCombos +recommendation: > + The recommended approach to dealing with this situation is to create a new category combination, using two new categories. + Using the example from the introduction, you should create two new category options called "Unknown age" and "Unknown sex". + A new category combination can then be created with these new categories, which do not share category options. + All data which is potentially associated with the old category combinations, would need to be reassigned + to the new category option combinations. Any analytical objects which use the categories to be removed + would also need to be updated. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_unique_category_combo.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_unique_category_combo.yaml index bd4044af9f48..45c5ee2cbece 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_unique_category_combo.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_unique_category_combo.yaml @@ -32,7 +32,7 @@ # checks if a and b are the same set by checking that a is included in b and vice-versa --- name: categories_unique_category_combo -description: Lists category combos that share a combination of categories with at least one other category combo +description: Different category combinations should not have the exact same combination of categories. section: Categories summary_sql: >- with combo_sets as ( @@ -62,6 +62,9 @@ details_sql: >- details_id_type: categoryCombos severity: SEVERE introduction: > - A category combo should be a unique combination of categories + Category combinations should be a unique combination of categories. If two or more category combinations + contain the exact same set of categories, this would be considered to be duplicative and potentially + confusing to users. +section_order: 8 recommendation: > One category combo is kept, all references are updated to use this combo and other combos are removed. \ No newline at end of file diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_combos_unused.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_combos_unused.yaml new file mode 100644 index 000000000000..205fe0720607 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_combos_unused.yaml @@ -0,0 +1,81 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- + name: category_combos_unused + description: Category combinations not used by other metadata objects + section: Categories + section_order: 6 + summary_sql: >- + WITH unused_category_combo as ( + SELECT a.uid, a.name, a.created, b.type FROM categorycombo a + LEFT JOIN ( + SELECT categorycomboid, 'ds' as type FROM dataset + UNION + SELECT categorycomboid, 'dse' as type FROM datasetelement + UNION + SELECT categorycomboid, 'de' as type FROM dataelement + UNION + SELECT categorycomboid, 'pr' as type FROM program + UNION + SELECT categorycomboid, 'daw' as type from dataapprovalworkflow + ) b + ON a.categorycomboid = b.categorycomboid + ) + SELECT COUNT(*) as value, + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM categorycombo), 0) as percent + FROM unused_category_combo where type IS NULL; + details_sql: >- + WITH unused_category_combo as ( + SELECT a.uid, a.name, a.created, b.type FROM categorycombo a + LEFT JOIN ( + SELECT categorycomboid, 'ds' as type FROM dataset + UNION + SELECT categorycomboid, 'dse' as type FROM datasetelement + UNION + SELECT categorycomboid, 'de' as type FROM dataelement + UNION + SELECT categorycomboid, 'pr' as type FROM program + UNION + SELECT categorycomboid, 'daw' as type from dataapprovalworkflow + ) b + ON a.categorycomboid = b.categorycomboid + ) + SELECT uid, name, CAST(created AS text) as comment + from unused_category_combo + where type IS NULL; + severity: WARNING + introduction: > + Category combinations which are unused in any datasets, data elements, programs or data approval workflows may be safe + to delete. In some cases, category combinations may be created, but never actually used for anything. This may lead + to situations where users and implementers are confused about which category combination should actually be used. + In general, it should be safe to delete unused category combinations, except in situations where existing data + has been associated with them. + details_id_type: categoryCombos + recommendation: > + Check to see if any data is associated with the category combination before attempting to delete it. Category combinations + which are not currently used in any metadata objects may still be valid for legacy reasons, but they should be reviewed + to be sure they are still needed. Otherwise, it should be safe to remove them from the system. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_cardinality.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_cardinality.yaml new file mode 100644 index 000000000000..40f3916a4103 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_cardinality.yaml @@ -0,0 +1,76 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#--- +name: cocs_wrong_cardinality +description: Category option combinations with incorrect cardinality. +section: Categories +section_order: 5 +summary_sql: >- + WITH cocs_wrong_cardinality as ( + SELECT foo.categorycomboid,foo.categoryoptioncomboid,foo.actual_cardnality,bar.theoretical_cardnality FROM ( + SELECT b.categorycomboid,a.categoryoptioncomboid, COUNT(*) as actual_cardnality + FROM categoryoptioncombos_categoryoptions a + INNER JOIN categorycombos_optioncombos b on a.categoryoptioncomboid = b.categoryoptioncomboid + GROUP BY b.categorycomboid,a.categoryoptioncomboid ) as foo + INNER JOIN + (SELECT categorycomboid,COUNT(*) as theoretical_cardnality FROM categorycombos_categories + GROUP BY categorycomboid) bar on foo.categorycomboid = bar.categorycomboid + WHERE foo.actual_cardnality != bar.theoretical_cardnality ) + SELECT COUNT(*) as value, + 100.0 * COUNT(*) / NULLIF( ( SELECT COUNT(*) FROM categoryoptioncombo),0) as percent + FROM cocs_wrong_cardinality; +details_sql: >- + WITH baz as ( + SELECT foo.categorycomboid,foo.categoryoptioncomboid,foo.actual_cardnality,bar.theoretical_cardnality FROM ( + SELECT b.categorycomboid,a.categoryoptioncomboid, COUNT(*) as actual_cardnality + FROM categoryoptioncombos_categoryoptions a + INNER JOIN categorycombos_optioncombos b on a.categoryoptioncomboid = b.categoryoptioncomboid + GROUP BY b.categorycomboid,a.categoryoptioncomboid ) as foo + INNER JOIN + (SELECT categorycomboid,COUNT(*) as theoretical_cardnality FROM categorycombos_categories + GROUP BY categorycomboid) bar on foo.categorycomboid = bar.categorycomboid + WHERE foo.actual_cardnality != bar.theoretical_cardnality ) + SELECT x.uid as uid,x.name,y.name as comment + FROM baz + INNER JOIN categorycombo x on baz.categorycomboid = x.categorycomboid + INNER JOIN categoryoptioncombo y on baz.categoryoptioncomboid = y.categoryoptioncomboid; +severity: SEVERE +introduction: > + All category option combinations should have exactly the same number of category option associations + as the number of categories in the category combination. If there are two categories in a category + combination, then every category option combination should have exactly two category options. + + Category option combinations with the incorrect cardinality are usually the result of a category + combination having been created, and then modified later. Suppose that two categories are used + to create a category combination. Each category option combination of this category combo will + have two category options. If an additional category is added to the category combination later, + new category option combinations with three category options will be created and associated + with this category combo. The original category option combinations would be considered to be + invalid. +details_id_type: categoryOptionCombos +recommendation: > + Category option combinations which have an incorrect cardinality will be ignored by the DHIS2 + analytics system and should be removed. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_disjoint.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_disjoint.yaml new file mode 100644 index 000000000000..2d34c79447c3 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_disjoint.yaml @@ -0,0 +1,73 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- + name: category_option_combos_disjoint + description: Category option combinations with disjoint associations. + section: Categories + section_order: 4 + summary_sql: >- + WITH cocs_disjoint_associations as ( + SELECT DISTINCT d.uid, d.name, e.name as comment from + categoryoptioncombos_categoryoptions a + INNER JOIN categorycombos_optioncombos b USING(categoryoptioncomboid) + INNER JOIN ( SELECT DISTINCT a.categorycomboid, b.categoryoptionid from categorycombos_optioncombos a + INNER JOIN categoryoptioncombos_categoryoptions b on a.categoryoptioncomboid= b.categoryoptioncomboid + EXCEPT + SELECT a.categorycomboid,b.categoryoptionid from categorycombos_categories a + INNER JOIN categories_categoryoptions b on a.categoryid = b.categoryid ) c + ON b.categorycomboid = c.categorycomboid AND a.categoryoptionid = c.categoryoptionid + INNER JOIN categoryoptioncombo d on a.categoryoptioncomboid = d.categoryoptioncomboid + INNER JOIN categorycombo e on c.categorycomboid = e.categorycomboid) + SELECT COUNT(*) as value, + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM categoryoptioncombo), 0) as percent + FROM cocs_disjoint_associations; + details_sql: >- + SELECT DISTINCT d.uid, d.name, e.name as comment from + categoryoptioncombos_categoryoptions a + INNER JOIN categorycombos_optioncombos b USING(categoryoptioncomboid) + INNER JOIN ( SELECT DISTINCT a.categorycomboid, b.categoryoptionid from categorycombos_optioncombos a + INNER JOIN categoryoptioncombos_categoryoptions b on a.categoryoptioncomboid= b.categoryoptioncomboid + EXCEPT + SELECT a.categorycomboid,b.categoryoptionid from categorycombos_categories a + INNER JOIN categories_categoryoptions b on a.categoryid = b.categoryid ) c + ON b.categorycomboid = c.categorycomboid AND a.categoryoptionid = c.categoryoptionid + INNER JOIN categoryoptioncombo d on a.categoryoptioncomboid = d.categoryoptioncomboid + INNER JOIN categorycombo e on c.categorycomboid = e.categorycomboid + details_id_type: categoryOptionCombos + severity: SEVERE + introduction: > + Under certain circumstances, category option combinations may exist in the system, but not have any direct + association with category options which are associated with their category combination. This situation + usually occurs when category options have been added to a category and then the category is added + to a category combination. New category option combinations are created in the system at this point. + If any of the category options are then removed in one of the underlying categories, a so-called disjoint + category option combination may result. This is a category option combination which has no direct + association with any category options in any of the categories associated with the category combination. + recommendation: > + The disjoint category option combinations should be removed from the system if possible. However, + if any data is associated with the category option combination, a determination will need to be + made in regards of how to deal with this data. \ No newline at end of file diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_excess_members.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_excess_members.yaml new file mode 100644 index 000000000000..7a865aab41e8 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_excess_members.yaml @@ -0,0 +1,89 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: category_options_excess_groupset_membership +description: Category options which belong to multiple groups in a category option group set. +section: Categories +section_order: 10 +summary_sql: >- + WITH cos_multiple_groups as ( + SELECT x.uid,x.name, x.cogs_name || ' :{'|| + string_agg(x.cog_name,',') || '}' as comment from ( + SELECT co.uid,co.name,cogs.name as cogs_name,cog.name as cog_name + from ( + SELECT d.categoryoptionid,d.categoryoptiongroupsetid, + unnest(d.groupids) as categoryoptiongroupid + FROM ( + SELECT c.categoryoptionid,c.categoryoptiongroupsetid, + array_agg(c.categoryoptiongroupid) as groupids + FROM ( + SELECT a.categoryoptionid, + a.categoryoptiongroupid, + b.categoryoptiongroupsetid from + categoryoptiongroupmembers a + INNER JOIN categoryoptiongroupsetmembers b on a.categoryoptiongroupid = b.categoryoptiongroupid ) c + GROUP BY c.categoryoptionid,c.categoryoptiongroupsetid + HAVING array_length(array_agg(c.categoryoptionid), 1) > 1 ) d ) e + INNER JOIN dataelementcategoryoption co on e.categoryoptionid = co.categoryoptionid + INNER JOIN categoryoptiongroupset cogs on e.categoryoptiongroupsetid = cogs.categoryoptiongroupsetid + INNER JOIN categoryoptiongroup cog on e.categoryoptiongroupid = cog.categoryoptiongroupid ) x + GROUP BY x.uid,x.name,x.cogs_name + ) + SELECT COUNT(*), + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM dataelementcategoryoption), 0) as percent + FROM cos_multiple_groups; +details_sql: >- + SELECT x.uid,x.name, x.cogs_name || ' :{'|| + string_agg(x.cog_name,',') || '}' as comment from ( + SELECT co.uid,co.name,cogs.name as cogs_name,cog.name as cog_name + from ( + SELECT d.categoryoptionid,d.categoryoptiongroupsetid, + unnest(d.groupids) as categoryoptiongroupid + FROM ( + SELECT c.categoryoptionid,c.categoryoptiongroupsetid, + array_agg(c.categoryoptiongroupid) as groupids + FROM ( + SELECT a.categoryoptionid, + a.categoryoptiongroupid, + b.categoryoptiongroupsetid from + categoryoptiongroupmembers a + INNER JOIN categoryoptiongroupsetmembers b on a.categoryoptiongroupid = b.categoryoptiongroupid ) c + GROUP BY c.categoryoptionid,c.categoryoptiongroupsetid + HAVING array_length(array_agg(c.categoryoptionid), 1) > 1 ) d ) e + INNER JOIN dataelementcategoryoption co on e.categoryoptionid = co.categoryoptionid + INNER JOIN categoryoptiongroupset cogs on e.categoryoptiongroupsetid = cogs.categoryoptiongroupsetid + INNER JOIN categoryoptiongroup cog on e.categoryoptiongroupid = cog.categoryoptiongroupid ) x + GROUP BY x.uid,x.name,x.cogs_name +details_id_type: categories +severity: SEVERE +introduction: > + Category options should belong to exactly one category option group + which are part of a category option group set. If the category option belongs to + multiple groups, this will lead to unpredictable results in analysis. +recommendation: > + Using the maintenance app, assign the category option in the details list + to exactly one category option group within each group set. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_sets_incomplete.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_sets_incomplete.yaml new file mode 100644 index 000000000000..abc7b2eeb11c --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_sets_incomplete.yaml @@ -0,0 +1,138 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: category_option_group_sets_incomplete +description: Category option group sets which which do not contain all category options. +section: Categories +section_order: 10 +summary_sql: >- + WITH cat_option_group_complete as( + SELECT cogs.uid,cogs.name , cats.name || ':{' || opt.name || '}' as comment + FROM ( + SELECT categoryoptiongroupsetid,categoryid,unnest(missing_catoptions) as categoryoptionid + FROM ( + SELECT categoryoptiongroupsetid, categoryid,array(SELECT unnest(wants) EXCEPT SELECT unnest(has)) as missing_catoptions + FROM ( + SELECT y.categoryoptiongroupsetid,y.categoryid, y.has, z.wants + FROM + (SELECT x.categoryoptiongroupsetid, + x.categoryid, + array_agg(categoryoptionid) as has FROM + ( + SELECT a.categoryoptiongroupsetid, + b.categoryoptionid, + c.categoryid from categoryoptiongroupsetmembers a + INNER JOIN categoryoptiongroupmembers b USING(categoryoptiongroupid) + INNER JOIN categories_categoryoptions c USING(categoryoptionid) ) x + GROUP BY categoryoptiongroupsetid, categoryid ) y + LEFT OUTER JOIN + (SELECT categoryid,array_agg(categoryoptionid) as wants from categories_categoryoptions + GROUP BY categoryid) as z USING (categoryid) ) as cat_option_group_check ) f) g + INNER JOIN categoryoptiongroupset cogs USING(categoryoptiongroupsetid) + INNER JOIN dataelementcategory cats USING(categoryid) + INNER JOIN dataelementcategoryoption opt USING(categoryoptionid) + ORDER BY cogs.uid, cats.name) + SELECT COUNT(*) as value, + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) from categoryoptiongroupmembers + where categoryoptiongroupid in (SELECT categoryoptiongroupid from categoryoptiongroupsetmembers)), 0) as percent + FROM cat_option_group_complete; +details_sql: >- + SELECT cogs.uid,cogs.name , cats.name || ':{' || opt.name || '}' as comment + FROM ( + SELECT categoryoptiongroupsetid,categoryid,unnest(missing_catoptions) as categoryoptionid + FROM ( + SELECT categoryoptiongroupsetid, categoryid,array(SELECT unnest(wants) EXCEPT SELECT unnest(has)) as missing_catoptions + FROM ( + SELECT y.categoryoptiongroupsetid,y.categoryid, y.has, z.wants + FROM + (SELECT x.categoryoptiongroupsetid, + x.categoryid, + array_agg(categoryoptionid) as has FROM + ( + SELECT a.categoryoptiongroupsetid, + b.categoryoptionid, + c.categoryid from categoryoptiongroupsetmembers a + INNER JOIN categoryoptiongroupmembers b USING(categoryoptiongroupid) + INNER JOIN categories_categoryoptions c USING(categoryoptionid) ) x + GROUP BY categoryoptiongroupsetid, categoryid ) y + LEFT OUTER JOIN + (SELECT categoryid,array_agg(categoryoptionid) as wants from categories_categoryoptions + GROUP BY categoryid) as z USING (categoryid) ) as cat_option_group_check ) f) g + INNER JOIN categoryoptiongroupset cogs USING(categoryoptiongroupsetid) + INNER JOIN dataelementcategory cats USING(categoryid) + INNER JOIN dataelementcategoryoption opt USING(categoryoptionid) + ORDER BY cogs.uid, cats.name; +details_id_type: categoryOptionGroupSets +severity: SEVERE +introduction: > + Category option group sets are composed of multiple category option groups, which are in turn composed of various category + options. Category option group sets are often used to group related category options together for analytical purposes. + Categories are also composed of category options. In general, but not always, category option group sets should contain + all category options of related categories. Suppose we have the following two age categories. + + Age coarse: <15, 15+ + Age fine: <1, 1-10, 10-14,15-19,20-24,25-29,30-34,35-39,40-44,45+ + + In many cases, related data elements may be disaggregated differently. This may be to differences in the way + in which the data is collected, or as is often the case, the disaggregation has changed over time. If we + wish to analyze data for two different data elements (one of which uses Age coarse and the other Age fine), we + can create a category option group set consisting of two category option groups which should consist + of the following category options from above. + + <15: <15, <1, 1-10, 10-14 + 15+: 15+, 15-19,20-24,25-29,30-34,35-39,40-44,45+ + + Suppose that we happen to omit the category option "<1" from the "<15" category option group. This would + result in potential aggregation errors in when using this category option group set in any analytical + objects. The details of this metadata check would return the UID and name of the category option group + set. The category and category option would also be provided. Using the previous example, we would + see "Age fine:{<1}" in the metadata check details section, to indicate that the <1 category option + which is part of the "Age fine" category, is missing from the category option group set. + + There may exist specific analytical reasons why specific category options are omitted from a category option group set. + However,these should usually be special cases. This metadata check will identify cases where category option group + sets appear to be incomplete, however you should carefully review any groups which appear in the details. + + This check may also produce a number of issues which may at first glance appear to be false positives. In some + cases, category options may be added to a category by mistake. When category option group sets are created, + this extraneous option may be omitted and thus appear as missing from the group set. + + Another observed situation is where an option like "Unknown" is used across unrelated categories. Suppose you + have two categories "Age (<15, 15+, Unknown)" and "Sex (Male,Female,Unknown)". Similar to above, we create + category option group sets like: + + <15: <15 + 15+: 15+ + Unknown: Unknown + + This metadata check will report that "Male" and "Female" are missing from the group set. This is because the + same "Unknown" option is shared between two different, unrelated categories. It is recommended that if this + situation occurs, that you create two separate "Unknown" category options like "Unknown sex" and "Unknown age". + +recommendation: > + Using the maintenance app, assign the missing category options to an appropriate category option group + within the affected category option group set. \ No newline at end of file diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_options_no_categories.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_options_no_categories.yaml new file mode 100644 index 000000000000..26f1ecb871b0 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_options_no_categories.yaml @@ -0,0 +1,56 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- + name: category_options_no_categories + description: Category options with no categories. + section: Categories + section_order: 3 + summary_sql: >- + WITH category_options_no_categories AS ( + SELECT uid,name FROM dataelementcategoryoption + WHERE categoryoptionid + NOT IN + (SELECT DISTINCT categoryoptionid + FROM categories_categoryoptions)) + SELECT COUNT(*) as value, + 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) + FROM dataelementcategoryoption), 0 ) as percent + FROM category_options_no_categories; + details_sql: >- + SELECT uid,name FROM dataelementcategoryoption + WHERE categoryoptionid + NOT IN + (SELECT DISTINCT categoryoptionid + FROM categories_categoryoptions) + ORDER BY name; + severity: WARNING + introduction: > + All category options should belong to at least one category. + details_id_type: categoryOptions + recommendation: > + Category options which are not part of any category should be removed + or alternatively should be added to an appropriate category. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/coc_no_category_combo.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/coc_no_category_combo.yaml new file mode 100644 index 000000000000..8b0a941450cf --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/coc_no_category_combo.yaml @@ -0,0 +1,56 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- + name: catoptioncombos_no_catcombo + description: Category options combinations with no category combination. + section: Categories + section_order: 7 + summary_sql: >- + WITH catoptioncombos_no_catcombo AS ( + SELECT uid,name FROM categoryoptioncombo + WHERE categoryoptioncomboid NOT IN (SELECT DISTINCT categoryoptioncomboid + FROM categorycombos_optioncombos)) + SELECT COUNT(*)as value, + 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) + FROM categoryoptioncombo), 0 ) percent + FROM catoptioncombos_no_catcombo; + details_sql: >- + SELECT uid,name FROM categoryoptioncombo + WHERE categoryoptioncomboid NOT IN (SELECT categoryoptioncomboid + FROM categorycombos_optioncombos) + ORDER BY name; + severity: WARNING + introduction: > + All category option combinations should be associated with a category combo. + In certain cases, when category combinations are deleted,the linkage between + a category option combination and a category combination may become corrupted. + details_id_type: categoryOptionCombos + recommendation: > + Check if any data is associated with the category option combination in question. Likely, + the data should either be deleted or migrated to a valid category option combination. + Any data which is associated with any of these category option combinations will + not be available through either the data entry modules or any of the analytical apps. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_abandoned.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_abandoned.yaml new file mode 100644 index 000000000000..37e925da78d1 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_abandoned.yaml @@ -0,0 +1,56 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- + name: data_elements_aggregate_abandoned + description: Aggregate data elements that have not been changed in last 100 days and do not have any data values. + section: Data elements (aggregate) + section_order: 5 + summary_sql: >- + SELECT COUNT(*) as value, + 100*COUNT(*) / NULLIF( ( select COUNT(*) from dataelement where domaintype = 'AGGREGATE'), 0) as percent + from dataelement where domaintype = 'AGGREGATE' + and dataelementid + not in (select dataelementid from datavalue group by dataelementid) + and AGE( now(), lastupdated) > INTERVAL '100 days'; + details_sql: >- + select uid,name,lastupdated from dataelement + where domaintype = 'AGGREGATE' + and dataelementid + not in (select DISTINCT dataelementid from datavalue ) + AND AGE( now(), lastupdated) > INTERVAL '100 days' + ORDER BY name; + severity: WARNING + introduction: > + Data elements are considered to be abandoned when they have not been edited in at least 100 days + and do not have any data values associated with them. Often, these are the result of new or changed + configurations that have been abandoned at some point. + recommendation: > + Data elements that have no data associated with them and which there are no plans + to start using for data collection should be deleted. + is_slow: true + details_id_type: dataElements + diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_datasets_different_period_types.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_datasets_different_period_types.yaml new file mode 100644 index 000000000000..207283092b0b --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_datasets_different_period_types.yaml @@ -0,0 +1,93 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- + name: data_elements_aggregate_with_different_period_types + description: Aggregate data elements which belong to datasets with different period types. + section: Data elements (aggregate) + section_order: 7 + summary_sql: >- + WITH des_different_periodtypes as ( + SELECT uid,dataelement_name as name, + STRING_AGG(dataset_name, ';') as comment from ( + SELECT e.dataelement_name, + e.uid, + h.name as dataset_name + FROM ( + SELECT dataelementid,dataelement_name,uid, + UNNEST(periodtypes) as periodtypeid FROM ( + SELECT dataelementid,dataelement_name,uid, + array_agg(periodtypeid) as periodtypes + FROM( + SELECT DISTINCT a.dataelementid, + b.periodtypeid, + c.name as dataelement_name, + c.uid + from datasetelement a + INNER JOIN dataset b USING(datasetid) + INNER JOIN dataelement c USING(dataelementid) ) as c + GROUP BY dataelementid,dataelement_name,uid + HAVING array_length(array_agg(periodtypeid),1) > 1 ) d ) as e + INNER JOIN datasetelement g USING(dataelementid) + INNER JOIN dataset h ON g.datasetid = h.datasetid AND e.periodtypeid = h.periodtypeid ) x + GROUP BY uid, dataelement_name) + SELECT COUNT(*) as value, + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM dataelement), 0) as percent + FROM des_different_periodtypes; + details_sql: >- + SELECT uid,dataelement_name as name, + STRING_AGG(dataset_name, ';') as comment from ( + SELECT e.dataelement_name, + e.uid, + h.name as dataset_name + FROM ( + SELECT dataelementid,dataelement_name,uid, + UNNEST(periodtypes) as periodtypeid FROM ( + SELECT dataelementid,dataelement_name,uid, + array_agg(periodtypeid) as periodtypes + FROM( + SELECT DISTINCT a.dataelementid, + b.periodtypeid, + c.name as dataelement_name, + c.uid + from datasetelement a + INNER JOIN dataset b USING(datasetid) + INNER JOIN dataelement c USING(dataelementid) ) as c + GROUP BY dataelementid,dataelement_name,uid + HAVING array_length(array_agg(periodtypeid),1) > 1 ) d ) as e + INNER JOIN datasetelement g USING(dataelementid) + INNER JOIN dataset h ON g.datasetid = h.datasetid AND e.periodtypeid = h.periodtypeid ) x + GROUP BY uid, dataelement_name; + severity: SEVERE + introduction: > + Data elements should not belong to datasets with different period types. + details_id_type: dataElements + recommendation: > + If you need to collect data with different period types (e.g. weekly and monthly) you should + use different data elements for each period type. In general, it is not recommended to + collect data with different frequencies, since in general, data collected at higher frequencies (e.g. weekly) + can be aggregated to data with lower frequencies (e.g yearly). + diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_excess_groupset_membership.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_excess_groupset_membership.yaml new file mode 100644 index 000000000000..7c1b1b70668d --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_excess_groupset_membership.yaml @@ -0,0 +1,89 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: data_elements_excess_groupset_membership +description: Data elements which belong to multiple groups in a group set. +section: data_elements +section_order: 10 +summary_sql: >- + WITH des_multiple_groups as ( + SELECT x.uid,x.name, x.degs_name || ' :{'|| + string_agg(x.deg_name,',') || '}' as comment from ( + SELECT de.uid,de.name,degs.name as degs_name,deg.name as deg_name + from ( + SELECT d.dataelementid,d.dataelementgroupsetid, + unnest(d.groupids) as dataelementgroupid + FROM ( + SELECT c.dataelementid,c.dataelementgroupsetid, + array_agg(c.dataelementgroupid) as groupids + FROM ( + SELECT a.dataelementid, + a.dataelementgroupid, + b.dataelementgroupsetid from + dataelementgroupmembers a + INNER JOIN dataelementgroupsetmembers b on a.dataelementgroupid = b.dataelementgroupid ) c + GROUP BY c.dataelementid,c.dataelementgroupsetid + HAVING array_length(array_agg(c.dataelementgroupid), 1) > 1 ) d ) e + INNER JOIN dataelement de on e.dataelementid = de.dataelementid + INNER JOIN dataelementgroupset degs on e.dataelementgroupsetid = degs.dataelementgroupsetid + INNER JOIN dataelementgroup deg on e.dataelementgroupid = deg.dataelementgroupid ) x + GROUP BY x.uid,x.name,x.degs_name + ) + SELECT COUNT(*), + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM dataelement), 0) as percent + FROM des_multiple_groups; +details_sql: >- + SELECT x.uid,x.name, x.degs_name || ' :{'|| + string_agg(x.deg_name,',') || '}' as comment from ( + SELECT de.uid,de.name,degs.name as degs_name,deg.name as deg_name + from ( + SELECT d.dataelementid,d.dataelementgroupsetid, + unnest(d.groupids) as dataelementgroupid + FROM ( + SELECT c.dataelementid,c.dataelementgroupsetid, + array_agg(c.dataelementgroupid) as groupids + FROM ( + SELECT a.dataelementid, + a.dataelementgroupid, + b.dataelementgroupsetid from + dataelementgroupmembers a + INNER JOIN dataelementgroupsetmembers b on a.dataelementgroupid = b.dataelementgroupid ) c + GROUP BY c.dataelementid,c.dataelementgroupsetid + HAVING array_length(array_agg(c.dataelementgroupid), 1) > 1 ) d ) e + INNER JOIN dataelement de on e.dataelementid = de.dataelementid + INNER JOIN dataelementgroupset degs on e.dataelementgroupsetid = degs.dataelementgroupsetid + INNER JOIN dataelementgroup deg on e.dataelementgroupid = deg.dataelementgroupid ) x + GROUP BY x.uid,x.name,x.degs_name +details_id_type: data_elements_aggregate +severity: SEVERE +introduction: > + Data elements should belong to exactly one group within each data element + group set of which they are a member. If the data element belongs to + multiple groups, this will lead to unpredictable results in analysis. +recommendation: > + Using the maintenance app, assign the data elements in the details list + to exactly one data element group within each group set. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_inconsistent_agg_operator.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_inconsistent_agg_operator.yaml new file mode 100644 index 000000000000..a8fe12d60682 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_inconsistent_agg_operator.yaml @@ -0,0 +1,72 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- + name: data_elements_aggregate_aggregation_operator + description: Non-numeric data elements which have an aggregation operator other than NONE. + section: Data elements (aggregate) + section_order: 8 + summary_sql: >- + with des_aggregation_op as ( + SELECT uid,name,valuetype,aggregationtype from dataelement + where valuetype IN ('TEXT', 'LONG_TEXT', 'LETTER', + 'PHONE_NUMBER','EMAIL','DATE','DATETIME','TIME','USERNAME', + 'TRACKER_ASSOCIATE','ORGANISATION_UNIT','REFERENCE','AGE','URL', + 'FILE_RESOURCE','IMAGE','GEOJSON','MULTI_TEXT') + AND aggregationtype != 'NONE' + UNION + SELECT uid,name,valuetype,aggregationtype from dataelement + where valuetype IN ('BOOLEAN', 'TRUE_ONLY', 'NUMBER', 'UNIT_INTERVAL', 'PERCENTAGE', 'INTEGER', 'INTEGER_POSITIVE', + 'INTEGER_NEGATIVE','INTEGER_ZERO_OR_POSITIVE' ) + AND aggregationtype = 'NONE' + + ) + SELECT COUNT(*) as value, + 100*COUNT(*) / NULLIF( ( SELECT COUNT(*) from dataelement), 0 ) as percent + from des_aggregation_op; + details_sql: >- + SELECT uid,name,valuetype || ' (' || aggregationtype || ')' as comment + FROM dataelement + WHERE valuetype IN ('TEXT', 'LONG_TEXT', 'LETTER', + 'PHONE_NUMBER','EMAIL','DATE','DATETIME','TIME','USERNAME', + 'TRACKER_ASSOCIATE','ORGANISATION_UNIT','REFERENCE','AGE','URL', + 'FILE_RESOURCE','IMAGE','GEOJSON','MULTI_TEXT') + AND aggregationtype != 'NONE' + UNION + SELECT uid,name,valuetype || ' (' || aggregationtype || ')' as comment from dataelement + where valuetype IN ('BOOLEAN', 'TRUE_ONLY', 'NUMBER', 'UNIT_INTERVAL', 'PERCENTAGE', 'INTEGER', 'INTEGER_POSITIVE', + 'INTEGER_NEGATIVE','INTEGER_ZERO_OR_POSITIVE' ) + AND aggregationtype = 'NONE' + ORDER BY name; + severity: WARNING + introduction: > + Data elements which are not numeric (text, dates, etc) should have an aggregation + operator set to NONE. Data elements which are able to be aggregated (numbers, integers, etc) + should have an aggregation operator set to something other than NONE, most often SUM. + recommendation: > + Open the affected data elements in the Maintenance App and change their aggregation type + to an appropriate type. Alternatively, these can be altered via the API. + details_id_type: dataElements diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_no_analysis.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_no_analysis.yaml new file mode 100644 index 000000000000..522ed12105f4 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_no_analysis.yaml @@ -0,0 +1,71 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- + name: data_elements_aggregate_no_analysis + description: Aggregate data elements not used in any favourites (directly or through indicators) + section: Data elements (aggregate) + section_order: 2 + summary_sql: >- + WITH des_no_analysis AS ( + select name,uid from dataelement where domaintype = 'AGGREGATE' and + dataelementid not in (select de.dataelementid from dataelement de + left join indicator ind on (ind.numerator like '%'||de.uid||'%' or + ind.denominator like '%'||de.uid||'%') where ind.name is not null + and ind.indicatorid in (select indicatorid from datadimensionitem where indicatorid > 0)) + and dataelementid not in (select dataelementoperand_dataelementid from datadimensionitem + where dataelementoperand_dataelementid > 0) and dataelementid + not in (select dataelementid from datadimensionitem where dataelementid > 0) order by name + ) + SELECT COUNT(*) as value, + 100 * COUNT(*) / NULLIF( ( SELECT COUNT(*) FROM dataelement WHERE domaintype = 'AGGREGATE' ),0) as percent + FROM des_no_analysis; + details_sql: >- + select uid,name from dataelement where domaintype = 'AGGREGATE' and + dataelementid not in (select de.dataelementid from dataelement de + left join indicator ind on (ind.numerator like '%'||de.uid||'%' or + ind.denominator like '%'||de.uid||'%') where ind.name is not null + and ind.indicatorid in (select indicatorid from datadimensionitem where indicatorid > 0)) + and dataelementid not in (select dataelementoperand_dataelementid from datadimensionitem + where dataelementoperand_dataelementid > 0) and dataelementid + not in (select dataelementid from datadimensionitem where dataelementid > 0) + order by name; + details_id_type: dataElements + severity: WARNING + introduction: > + All aggregate data elements that are captured in DHIS2 should be used to produce some + type of analysis output (charts, maps, tables). This can be by using them directly + in an output, or by having them contribute to an indicator calculation that is used an + output. + recommendation: > + Data elements that are not routinely being reviewed in analysis, + either directly or indirectly through indicators, should be reviewed + to determine if they still need to be collected. If these are meant to be used + in routine review, then associated outputs should be created using them. If these + data elements are not going to be used for any type of information review, + consideration should be made to either archive them or delete them. + + diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_no_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_no_groups.yaml new file mode 100644 index 000000000000..69ea171d92f4 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_no_groups.yaml @@ -0,0 +1,53 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- + name: data_elements_aggregate_no_groups + description: Aggregate data elements not in any data element groups. + section: Data elements (aggregate) + section_order: 4 + summary_sql: >- + SELECT COUNT(*) as value, + 100*COUNT(*)/ NULLIF( (SELECT COUNT(*) from dataelement where domaintype = 'AGGREGATE'), 0) as percent + from dataelement where domaintype = 'AGGREGATE' + and dataelementid not in (select dataelementid from dataelementgroupmembers); + details_sql: >- + select uid,name from dataelement + where domaintype = 'AGGREGATE' + and dataelementid not in (select dataelementid from dataelementgroupmembers) + ORDER BY name; + severity: WARNING + introduction: > + All data elements should be in a data element group. + This allows users to find the data elements more easily in analysis + apps and also contributes to having more complete data element group sets. + Maintenance operations can also be made more efficient by applying + bulk settings (ex. sharing) to all data elements within a data element group. + details_id_type: dataElements + recommendation: > + Data elements that are not in a data element group should be added to a relevant + data element group. If the data elements are not needed, they should be deleted. + diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_nodata.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_nodata.yaml new file mode 100644 index 000000000000..1449c2c88075 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_nodata.yaml @@ -0,0 +1,62 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: data_elements_aggregate_no_data +description: Aggregate data elements with NO data values. +section: Data elements (aggregate) +section_order: 6 +summary_sql: >- + WITH dataelements_no_data AS ( + SELECT uid,name from dataelement where dataelementid in ( + SELECT dataelementid from dataelement + where domaintype = 'AGGREGATE' + EXCEPT + select DISTINCT dataelementid FROM + datavalue where deleted = FALSE and value != '' )) + select count(*) as value , + 100*COUNT(*)/ NULLIF( ( select count(*) from dataelement + where domaintype = 'AGGREGATE'), 0) as percent + from dataelements_no_data; +details_sql: >- + SELECT uid,name from dataelement where dataelementid in ( + SELECT dataelementid from dataelement + where domaintype = 'AGGREGATE' + EXCEPT + select DISTINCT dataelementid FROM + datavalue where deleted = FALSE and value != '' ) + ORDER BY name; +details_id_type: dataElements +severity: WARNING +introduction: > + Data elements should generally always be associated with data values. + If data elements exist in a data set which is active, but there are no + data values associated with them, they may not be part of the data entry + screens. Alternatively, the data element may have been added but never + been associated with a dataset. +recommendation: > + Consider removing data elements with no data values. +is_slow: TRUE diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/datasets/datasets_empty.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/datasets/datasets_empty.yaml new file mode 100644 index 000000000000..f80a9d615d7c --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/datasets/datasets_empty.yaml @@ -0,0 +1,45 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: datasets_empty +description: Datasets with no data elements. +section: Data sets +section_order: 1 +summary_sql: >- + select count(*) as value, + 100*count(*) / NULLIF( (select count(*) from dataset),0) as percent + from dataset where datasetid NOT IN ( + SELECT datasetid from datasetelement); +details_sql: >- + SELECT uid,name from dataset WHERE + datasetid NOT IN (SELECT datasetid from datasetelement); +severity: WARNING +introduction: > + All datasets should have data elements assigned to them. +details_id_type: dataSets +recommendation: > + Remove empty datasets, or alternatively, assign data elements to them. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_category_option_group_sets.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_category_option_group_sets.yaml new file mode 100644 index 000000000000..60e1eb4293ba --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_category_option_group_sets.yaml @@ -0,0 +1,61 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: category_option_group_sets_scarce +description: Category option groups should have at least two members. +section: Group size +section_order: 8 +summary_sql: >- + WITH group_size as ( + SELECT uid,name, COALESCE(x.count,0) as count from + categoryoptiongroupset + LEFT OUTER JOIN ( + SELECT categoryoptiongroupsetid, COUNT(categoryoptiongroupid) as count from categoryoptiongroupsetmembers + GROUP BY categoryoptiongroupsetid) x + USING(categoryoptiongroupsetid) + ORDER BY name) + SELECT COUNT(*) as value, + 100*COUNT(*) / NULLIF( ( select COUNT(*) from categoryoptiongroupset), 0) as percent + from group_size where count < 2; +details_sql: >- + WITH group_size as ( + SELECT uid,name, COALESCE(x.count,0) as count from + categoryoptiongroupset + LEFT OUTER JOIN ( + SELECT categoryoptiongroupsetid, COUNT(categoryoptiongroupid) as count from categoryoptiongroupsetmembers + GROUP BY categoryoptiongroupsetid) x + USING(categoryoptiongroupsetid) + ORDER BY name) + SELECT uid,name,CAST(count as text) FROM + group_size where count < 2; +severity: WARNING +introduction: > + Generally, category option group sets should be composed of at least two category option groups. +recommendation: > + Considering removing groups with zero or one category option groups, or alternatively, add additional + category option groups to the group set to make it more useful. +details_id_type: categoryOptionGroupSets \ No newline at end of file diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_category_option_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_category_option_groups.yaml new file mode 100644 index 000000000000..72f545c64a80 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_category_option_groups.yaml @@ -0,0 +1,63 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: category_option_groups_scarce +description: Category option groups should have at least two members. +section: Group size +section_order: 8 +summary_sql: >- + WITH group_size as ( + SELECT uid,name, COALESCE(x.count,0) as count from + categoryoptiongroup + LEFT OUTER JOIN ( + SELECT categoryoptiongroupid, COUNT(categoryoptionid) as count from categoryoptiongroupmembers + GROUP BY categoryoptiongroupid) x + USING(categoryoptiongroupid) + ORDER BY name) + SELECT COUNT(*) as value, + 100*COUNT(*) / NULLIF( ( select COUNT(*) from categoryoptiongroup), 0) as percent + from group_size where count < 2; +details_sql: >- + WITH group_size as ( + SELECT uid,name, COALESCE(x.count,0) as count from + categoryoptiongroup + LEFT OUTER JOIN ( + SELECT categoryoptiongroupid, COUNT(categoryoptionid) as count from categoryoptiongroupmembers + GROUP BY categoryoptiongroupid) x + USING(categoryoptiongroupid) + ORDER BY name) + SELECT uid,name, CAST(count as text) as comment + from group_size where count < 2; +severity: WARNING +introduction: > + Generally, category option groups should be composed of at least two category options. There can however + be legitimate cases when a single category option is part of a group, especially in cases where + there is a direct mapping between age bands and category options. +recommendation: > + Considering removing groups with zero or one category options, or alternatively, add additional + category options to the group to make it more useful. +details_id_type: categoryOptionGroups \ No newline at end of file diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_data_element_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_data_element_groups.yaml new file mode 100644 index 000000000000..2de76cb19d99 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_data_element_groups.yaml @@ -0,0 +1,61 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: data_element_groups_scarce +description: Data element groups should have at least two members. +section: Group size +section_order: 1 +summary_sql: >- + WITH group_size as ( + SELECT uid,name, 'dataElementGroups' as comment, COALESCE(x.count,0) as count from + dataelementgroup + LEFT OUTER JOIN ( + SELECT dataelementgroupid, COUNT(dataelementid) as count from dataelementgroupmembers + GROUP BY dataelementgroupid) x + USING(dataelementgroupid) + ORDER BY name) + SELECT COUNT(*) as value, + 100*COUNT(*) / NULLIF( ( select COUNT(*) from dataelementgroup), 0) as percent + from group_size WHERE count < 2; +details_sql: >- + WITH group_size as ( + SELECT uid,name, 'dataElementGroups' as comment, COALESCE(x.count,0) as count from + dataelementgroup + LEFT OUTER JOIN ( + SELECT dataelementgroupid, COUNT(dataelementid) as count from dataelementgroupmembers + GROUP BY dataelementgroupid) x + USING(dataelementgroupid) + ORDER BY name) + SELECT uid,name, CAST(count as text) as comment from group_size + where count < 2; +severity: WARNING +introduction: > + All data element groups should be composed of at least two data elements. +recommendation: > + Considering removing groups with a single member, or alternatively, add additional + members to the group to make it more useful. +details_id_type: dataElementGroups diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_indicator_group_sets.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_indicator_group_sets.yaml new file mode 100644 index 000000000000..758f5baaccd9 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_indicator_group_sets.yaml @@ -0,0 +1,61 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: indicator_group_sets_scarce +description: Indicator groups sets should have at least two members. +section: Group size +section_order: 2 +summary_sql: >- + WITH group_size as ( + SELECT uid,name, COALESCE(x.count,0) as count from + indicatorgroupset + LEFT OUTER JOIN ( + SELECT indicatorgroupsetid, COUNT(indicatorgroupid) as count from indicatorgroupsetmembers + GROUP BY indicatorgroupsetid ) x + USING(indicatorgroupsetid) + ORDER BY name) + SELECT COUNT(*) as value, + 100*COUNT(*) / NULLIF( ( select COUNT(*) from indicatorgroupset), 0) as percent + from group_size where count < 2; +details_sql: >- + WITH group_size as ( + SELECT uid,name, COALESCE(x.count,0) as count from + indicatorgroupset + LEFT OUTER JOIN ( + SELECT indicatorgroupsetid, COUNT(indicatorgroupid) as count from indicatorgroupsetmembers + GROUP BY indicatorgroupsetid ) x + USING(indicatorgroupsetid) + ORDER BY name) + SELECT uid,name,CAST(count as text) FROM group_size + where count < 2; +severity: WARNING +introduction: > + All indicator group set should be composed of at least two indicators groups. +recommendation: > + Considering removing indicator group sets with less than two indicator groups, or alternatively, add additional + indicator groups to the indicator group set to make it more useful. +details_id_type: indicatorGroupSets diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_indicator_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_indicator_groups.yaml new file mode 100644 index 000000000000..323c02f812ea --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_indicator_groups.yaml @@ -0,0 +1,61 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: indicator_groups_scarce +description: Indicator groups should have at least two members. +section: Group size +section_order: 2 +summary_sql: >- + WITH group_size as ( + SELECT uid,name, COALESCE(x.count,0) as count from + indicatorgroup + LEFT OUTER JOIN ( + SELECT indicatorgroupid, COUNT(indicatorid) as count from indicatorgroupmembers + GROUP BY indicatorgroupid) x + USING(indicatorgroupid) + ORDER BY name) + SELECT COUNT(*) as value, + 100*COUNT(*) / NULLIF( ( select COUNT(*) from indicatorgroup), 0) as percent + from group_size where count < 2; +details_sql: >- + WITH group_size as ( + SELECT uid,name, COALESCE(x.count,0) as count from + indicatorgroup + LEFT OUTER JOIN ( + SELECT indicatorgroupid, COUNT(indicatorid) as count from indicatorgroupmembers + GROUP BY indicatorgroupid) x + USING(indicatorgroupid) + ORDER BY name) + SELECT uid,name,CAST(count as text) + FROM group_size where count < 2; +severity: WARNING +introduction: > + All indicator groups should be composed of at least two indicators. +recommendation: > + Considering removing groups with zero or one groups, or alternatively, add additional + members to the group to make it more useful. +details_id_type: indicatorGroups diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_organisation_unit_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_organisation_unit_groups.yaml new file mode 100644 index 000000000000..44da9b02f4cf --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_organisation_unit_groups.yaml @@ -0,0 +1,61 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: orgunit_groups_scarce +description: Organisation unit groups should have at least two members. +section: Group size +section_order: 1 +summary_sql: >- + WITH group_size as ( + SELECT uid,name, COALESCE(x.count,0) as count from + orgunitgroup + LEFT OUTER JOIN ( + SELECT orgunitgroupid, COUNT(organisationunitid) as count from orgunitgroupmembers + GROUP BY orgunitgroupid) x + USING(orgunitgroupid) + ORDER BY name) + SELECT COUNT(*) as value, + 100*COUNT(*) / NULLIF( ( select COUNT(*) from orgunitgroup), 0) as percent + from group_size where count < 2; +details_sql: >- + WITH group_size as ( + SELECT uid,name, COALESCE(x.count,0) as count from + orgunitgroup + LEFT OUTER JOIN ( + SELECT orgunitgroupid, COUNT(organisationunitid) as count from orgunitgroupmembers + GROUP BY orgunitgroupid) x + USING(orgunitgroupid) + ORDER BY name) + SELECT uid,name, CAST(count as text) as comment + from group_size where count < 2; +severity: WARNING +introduction: > + Generally, organisation unit groups should be composed of multiple organisation units. +recommendation: > + Considering removing groups with zero or one organisation units, or alternatively, add additional + members to the group to make it more useful. +details_id_type: organisationUnitGroups diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_program_indicator_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_program_indicator_groups.yaml new file mode 100644 index 000000000000..023d3df66d32 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_program_indicator_groups.yaml @@ -0,0 +1,63 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: program_indicator_groups_scarce +description: Program indicator groups should have at least two members. +section: Group size +section_order: 3 +summary_sql: >- + WITH group_size as ( + SELECT uid,name, COALESCE(x.count,0) as count from + programindicatorgroup + LEFT OUTER JOIN ( + SELECT programindicatorgroupid, COUNT(programindicatorid) as count + FROM programindicatorgroupmembers + GROUP BY programindicatorgroupid ) x + USING(programindicatorgroupid) + ORDER BY name) + SELECT COUNT(*) as value, + 100*COUNT(*) / NULLIF( ( select COUNT(*) from programindicatorgroup), 0) as percent + from group_size where count < 2; +details_sql: >- + WITH group_size as ( + SELECT uid,name, COALESCE(x.count,0) as count from + programindicatorgroup + LEFT OUTER JOIN ( + SELECT programindicatorgroupid, COUNT(programindicatorid) as count + FROM programindicatorgroupmembers + GROUP BY programindicatorgroupid ) x + USING(programindicatorgroupid) + ORDER BY name) + SELECT uid,name,CAST(count as text) as comment + FROM group_size where count < 2; +severity: WARNING +introduction: > + Program indicator groups should generally be composed of two or more program indicators. +recommendation: > + Considering removing groups with a single member, or alternatively, add additional + members to the group to make it more useful. +details_id_type: programIndicatorGroups diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_user_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_user_groups.yaml new file mode 100644 index 000000000000..de6dd47ad955 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_user_groups.yaml @@ -0,0 +1,61 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: user_groups_scarce +description: User groups should have at least two members. +section: Group size +section_order: 9 +summary_sql: >- + WITH group_size as ( + SELECT uid,name, COALESCE(x.count,0) as count from + usergroup + LEFT OUTER JOIN ( + SELECT usergroupid, COUNT(userid) as count from usergroupmembers + GROUP BY usergroupid) x + USING(usergroupid) + ORDER BY name) + SELECT COUNT(*) as value, + 100*COUNT(*) / NULLIF( ( select COUNT(*) from usergroup), 0) as percent + from group_size where count < 2; +details_sql: >- + WITH group_size as ( + SELECT uid,name, COALESCE(x.count,0) as count from + usergroup + LEFT OUTER JOIN ( + SELECT usergroupid, COUNT(userid) as count from usergroupmembers + GROUP BY usergroupid) x + USING(usergroupid) + ORDER BY name) + SELECT uid,name,CAST(count as text) + from group_size where count < 2; +severity: WARNING +introduction: > + Generally, user groups should contain two or more users. +recommendation: > + Considering removing user groups with less than two users, or alternatively, add additional + users to the group to make it more useful. +details_id_type: userGroups diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_validation_rule_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_validation_rule_groups.yaml new file mode 100644 index 000000000000..115b7e684b78 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_validation_rule_groups.yaml @@ -0,0 +1,63 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: validation_rule_groups_scarce +description: Validation rule should have at least two members. +section: Group size +section_order: 2 +summary_sql: >- + WITH group_size as ( + SELECT uid,name, COALESCE(x.count,0) as count from + validationrulegroup + LEFT OUTER JOIN ( + SELECT validationgroupid as validationrulegroupid, + COUNT(validationruleid) as count from validationrulegroupmembers + GROUP BY validationgroupid ) x + USING(validationrulegroupid) + ORDER BY name) + SELECT COUNT(*) as value, + 100*COUNT(*) / NULLIF( ( select COUNT(*) from validationrulegroup), 0) as percent + from group_size where count < 2; +details_sql: >- + WITH group_size as ( + SELECT uid,name, COALESCE(x.count,0) as count from + validationrulegroup + LEFT OUTER JOIN ( + SELECT validationgroupid as validationrulegroupid, + COUNT(validationruleid) as count from validationrulegroupmembers + GROUP BY validationgroupid ) x + USING(validationrulegroupid) + ORDER BY name) + SELECT uid,name,CAST(count as text) as comment + FROM group_size where count < 2; +severity: WARNING +introduction: > + Generally validation rule groups should be composed of multiple validation rules. +recommendation: > + Considering removing groups which are empty or which have a single member. Alternatively, add additional + members to the group to make it more useful. +details_id_type: validationRuleGroups diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_duplicate_types.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_duplicate_types.yaml new file mode 100644 index 000000000000..748a83a01d8a --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_duplicate_types.yaml @@ -0,0 +1,62 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,/home/jason/Desktop EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE./home/jason/Desktop +# +--- +name: indicator_types_duplicated +description: Indicator types with the same factor. +section: Indicators +section_order: 5 +summary_sql: >- + WITH duplicate_indicator_factors AS ( + SELECT uid,name from indicatortype where + indicatorfactor IN ( + SELECT indicatorfactor + from indicatortype + GROUP BY indicatorfactor + HAVING COUNT(indicatorfactor) > 1)) + select count(*) as value, + 100 * COUNT(*) / NULLIF( ( SELECT COUNT(*) FROM indicatortype) ,0) as percent + FROM duplicate_indicator_factors; +details_sql: >- + SELECT uid,name from indicatortype where + indicatorfactor IN ( + SELECT indicatorfactor + from indicatortype + GROUP BY indicatorfactor + HAVING COUNT(indicatorfactor) > 1) +severity: WARNING +introduction: Indicators can be assigned a factor which is multiplied by the + indicator's value. For example, if you create a percentage + based indicator, you can assign the indicator a factor of 100, which would + be multiplied by the actual value of the indicator calculated by DHIS2. + In general, having multiple indicator types with the same factor is not + recommended. It is duplicative, and may lead to confusion. +details_id_type: indicatorTypes +recommendation: Duplicated indicator types should consider to be removed + from the system. Consider choosing one of the duplicated indicator types + as the one to keep, and then update all indicators which share the same + factor to this indicator type. The duplicated indicator types can then + be removed from the system. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_duplicated_terms.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_duplicated_terms.yaml new file mode 100644 index 000000000000..c3197ad6e250 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_duplicated_terms.yaml @@ -0,0 +1,83 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: indicators_duplicated_terms +description: Indicators with the same terms. +section: Indicators +section_order: 7 +summary_sql: >- + WITH duplicated_indicators as ( + SELECT unnest(indicatorids) from ( + SELECT sorted_expression, + array_agg(indicatorid) as indicatorids from ( + SELECT a.indicatorid, + replace(array_to_string(b.sorted_vals,''),' ','') as sorted_expression from ( + SELECT indicatorid, numerator, denominator from indicator ) a + LEFT JOIN LATERAL ( + SELECT array_agg(vals.x) as sorted_vals FROM ( + SELECT unnest(regexp_split_to_array(numerator || denominator,'')) as x + where indicatorid = a.indicatorid ORDER BY x ASC ) vals ) b ON TRUE) c + GROUP BY sorted_expression) d + where array_length(indicatorids,1) > 1) + SELECT COUNT(*) as count, + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM indicator), 0) as percent + from duplicated_indicators; +details_sql: >- + WITH duplicated_indicators as ( + SELECT a.indicatorid, + replace(array_to_string(b.sorted_vals,''),' ','') as sorted_expression from ( + SELECT indicatorid, numerator, denominator from indicator ) a + LEFT JOIN LATERAL ( + SELECT array_agg(vals.x) as sorted_vals FROM ( + SELECT unnest(regexp_split_to_array(numerator || denominator,'')) as x + where indicatorid = a.indicatorid ORDER BY x ASC ) vals ) b ON TRUE) + SELECT f.uid,f.name, f.numerator || '/' || f.denominator as comment + FROM indicator f + INNER JOIN ( + SELECT unnest(indicatorids) as indicatorid FROM ( + SELECT indicatorids,array_length(indicatorids,1) indicator_count FROM ( + SELECT sorted_expression,array_agg(indicatorid) as indicatorids + FROM duplicated_indicators + GROUP BY sorted_expression) as c ) as d + WHERE indicator_count > 1 ) e + on e.indicatorid = f.indicatorid; +details_id_type: indicators +severity: WARNING +introduction: > + Indicators should generally have unique formulas. This metadata check attempts + to identify indicators with the same formulas, regardless of the order of their + their terms. Suppose you have an indicator which calculates total ANC attendance: + ANC Total = ANC1 + ANC2 + ANC3 + Suppose then another indicator exists in the system, which has been defined as: + ANC Totals = ANC3 + ANC2 + ANC1 + These two indicators are equivalent, and will produce the same result, and are thus + considered to be duplicated. +recommendation: > + Duplicative indicators should be considered to be removed from the system, since + they may cause confusion on the part of users as to which one should be used. + Considered deleting all but one of the duplicated indicators using the maintenance + app. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_exact_duplicates.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_exact_duplicates.yaml new file mode 100644 index 000000000000..75050df4ed00 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_exact_duplicates.yaml @@ -0,0 +1,64 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: indicators_exact_duplicates +description: Indicators with the same formula. +section: Indicators +section_order: 8 +summary_sql: >- + WITH duplicated_indicators as ( + SELECT uid,name, numerator || '/' || denominator as comment from indicator where indicatorid in ( + SELECT unnest(indicators) as indicatorid from ( + SELECT expression,array_agg(indicatorid) as indicators FROM( + SELECT indicatorid,replace(numerator || denominator, ' ','') as expression + from indicator ) a + GROUP BY expression + HAVING array_length(array_agg(indicatorid),1) > 1 ) b ) + ) + SELECT COUNT(*) as count, + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM indicator), 0) as percent + from duplicated_indicators; +details_sql: >- + SELECT uid,name, numerator || '/' || denominator as comment from indicator where indicatorid in ( + SELECT unnest(indicators) as indicatorid from ( + SELECT expression,array_agg(indicatorid) as indicators FROM( + SELECT indicatorid,replace(numerator || denominator, ' ','') as expression + from indicator ) a + GROUP BY expression + HAVING array_length(array_agg(indicatorid),1) > 1 ) b ) + ORDER BY comment; +details_id_type: indicators +severity: WARNING +introduction: > + Indicators should generally have unique formulas. This metadata check shows + indicators which have the exact same formulas. Spaces which are present are + removed prior to comparison. +recommendation: > + Duplicative indicators should be considered to be removed from the system, since + they may cause confusion on the part of users as to which one should be used. + Consider deleting all but one of the duplicated indicators using the maintenance + app. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_noanalysis.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_noanalysis.yaml new file mode 100644 index 000000000000..44ac880a69e3 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_noanalysis.yaml @@ -0,0 +1,57 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: indicator_no_analysis +description: Indicators not used in analytical objects. +section: Indicators +section_order: 4 +summary_sql: >- + select count(*) as value, + 100*count(*)/ NULLIF( (select count(*) from indicator),0) as percent + FROM indicator + where indicatorid + not in (SELECT DISTINCT indicatorid from datadimensionitem + where indicatorid IS NOT NULL); +details_sql: >- + SELECT uid,name + FROM indicator + where indicatorid + not in (SELECT DISTINCT indicatorid from datadimensionitem + where indicatorid IS NOT NULL) + ORDER BY name; +severity: WARNING +introduction: > + Indicators should be used to produce some type of analytical output (charts, maps, pivot tables). + Note: indicators used in datasets to provide feedback during data entry are not counted + as being used in analytical objects. +details_id_type: indicators +recommendation: > + Indicators that are not routinely being used for analysis should be reviewed to + determine if they are useful and needed. If these are meant to be used for routine + review, then associated outputs should be created using them. If these indicators are + not going to be used for any type of information review, and are not used in data sets + for feedback during data entry, consideration should be made to delete them. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_nongrouped.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_nongrouped.yaml new file mode 100644 index 000000000000..a49078552bda --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_nongrouped.yaml @@ -0,0 +1,51 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: indicators_not_grouped +description: Indicators not in any groups. +section: Indicators +section_order: 2 +summary_sql: >- + select count(*) as value, + 100*count(*)/ NULLIF( (select count(*) from indicator), 0) as percent + from indicator where indicatorid + not in (select indicatorid from indicatorgroupmembers); +details_sql: >- + SELECT uid,name + FROM indicator + where indicatorid not in (select indicatorid from indicatorgroupmembers) + ORDER BY name; +severity: WARNING +introduction: > + All indicators should be in an indicator group. This allows users to find the indicators + more easily in analysis apps and also contributes to having more complete indicators + group sets. Maintenance operations can also be made more efficient by applying bulk + settings (ex. sharing, filtering) to all indicators within an indicator group. +details_id_type: indicators +recommendation: > + Indicators that are not in a indicator group should be added to a relevant indicator group. + If the indicators are not needed, they should be deleted. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/integrity_check_schema.json b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/integrity_check_schema.json new file mode 100644 index 000000000000..f855c7a684e8 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/integrity_check_schema.json @@ -0,0 +1,76 @@ +{ + "$schema":"http://json-schema.org/draft/2020-12/schema", + "$id": "https://dhis2.org/metadata-assessment.schema.json", + "title": "Metadata Assessment", + "description": "A DHIS2 metadata assessment object", + "type": "object", + "properties": { + "name": { + "description": "A short and unique name which will represent the name of the set of queries in DHIS2", + "type": "string" + }, + "description": { + "description": "A description giving information about the data integrity check", + "type": "string" + }, + "section" : { + "description": "A name which can be used to group related queries.", + "type" : "string" + }, + "section_order" : { + "description": "A number which can be used to order queries within sections", + "type": "integer", + "minimum": 1 + }, + "summary_sql": { + "description": "An SQL query which should return a single line and consist of four character fields: indicator,value,percent and description", + "type": "string" + }, + "details_sql": { + "type" : "string", + "description": "A DHIS2 database query which should return all identified objects from this particular issue. Should return at least uid and name" + }, + "details_id_type": { + "type": "string", + "description": "A short string which identifies the section of the details SQL." + }, + "severity": { + "description": "Level of severity of the issue", + "type" : "string", + "enum": ["INFO", + "WARNING", + "SEVERE", + "CRITICAL"] + }, + "introduction": { + "description": "This should field describe the issue the queries are seeking to identify.", + "type": "string" + }, + "recommendation": { + "description": "This field should describe the procedure to rectify the problems", + "type": "string" + }, + "is_protected" : { + "description": "Flag to indicate whether the query accesses protected tables. These queries cannot be accessed via the API", + "type" : "boolean", + "default" : false + }, + "is_slow" : { + "description": "Flag to indicate whether the query has the potential to take a long time to execute. By default, these will be excluded.", + "type" : "boolean", + "default" : false + } + }, + "required": [ + "description", + "details_id_type", + "details_sql", + "introduction", + "name", + "recommendation", + "section", + "section_order", + "severity", + "summary_sql" + ] +} \ No newline at end of file diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/option_sets_empty.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/option_sets_empty.yaml new file mode 100644 index 000000000000..7e63f394b8a4 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/option_sets_empty.yaml @@ -0,0 +1,54 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: options_sets_empty +description: Empty option sets +section: Option sets +section_order: 3 +summary_sql: >- + WITH options_sets_empty as ( + SELECT uid,name from + optionset where optionsetid + NOT IN (SELECT DISTINCT optionsetid + from optionvalue WHERE optionsetid IS NOT NULL) + ) + SELECT + COUNT(*) as value, + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM optionset), 0) as percent + from options_sets_empty; +details_sql: >- + SELECT uid,name from + optionset where optionsetid + NOT IN (SELECT DISTINCT optionsetid + from optionvalue WHERE optionsetid IS NOT NULL) + ORDER BY name; +details_id_type: optionSets +severity: WARNING +introduction: > + All option sets should generally include at least two items. Empty option sets serve no purpose. +recommendation: > + Options should either be added to the option set, or the option set should be deleted. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/option_sets_wrong_sort_order.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/option_sets_wrong_sort_order.yaml new file mode 100644 index 000000000000..b87a14d6f658 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/option_sets_wrong_sort_order.yaml @@ -0,0 +1,72 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: option_sets_wrong_sort_order +description: Option sets with possibly wrong sort order. +section: Option sets +section_order: 3 +summary_sql: >- + WITH option_sets_wrong_sort_order as ( + SELECT DISTINCT optionsetid, sort_order, row_number FROM ( + SELECT optionsetid,sort_order, row_number() + over ( PARTITION BY optionsetid ORDER BY sort_order) as row_number + from optionvalue + ) as foo where sort_order != row_number + ORDER BY optionsetid, sort_order + ) + SELECT COUNT(*) as value, + 100 * (SELECT COUNT(*) from (SELECT DISTINCT optionsetid + from option_sets_wrong_sort_order ) as bar) / + NULLIF( (SELECT COUNT(*) from optionset), 0) as percent + FROM option_sets_wrong_sort_order; +details_sql: >- + SELECT a.uid, a.name, b.sort_order || ' != ' || b.row_number as comment from + optionset a + INNER JOIN ( + SELECT DISTINCT optionsetid, sort_order, row_number FROM ( + SELECT optionsetid,sort_order, row_number() + over ( PARTITION BY optionsetid ORDER BY sort_order) as row_number + from optionvalue + ) as foo where sort_order != row_number + ORDER BY optionsetid, sort_order + ) b on a.optionsetid = b.optionsetid; +details_id_type: optionSets +severity: SEVERE +introduction: > + Option sets contain options which should be ordered sequentially. The sort_order property should always start + with 1 and have a sequential sequence. If there are three options in the option set, then the sort order + should be 1,2,3. + + In certain circumstances, options may be deleted from an option set, and the sort + order may become corrupted. This may lead to a situation where it becomes impossible to update the + option set from the maintenance app, and may lead to problems when attempting to using the option + set in the data entry app. +recommendation: > + If it is possible to open the option set in the maintenance app, you can resort the option set, + which should correct the problem. Another possible solution is to directly update the sort_order + property of in the `optionset` table in the database, ensuring that a valid sequence is present + for all options in the option set. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/unused_option_sets.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/unused_option_sets.yaml new file mode 100644 index 000000000000..814e4df89856 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/unused_option_sets.yaml @@ -0,0 +1,56 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: options_sets_unused +description: Option sets which are not used +section: Option sets +section_order: 2 +summary_sql: >- + WITH options_sets_unused as ( + SELECT DISTINCT uid,name FROM optionset where optionsetid NOT IN ( + SELECT DISTINCT optionsetid from attribute where optionsetid IS NOT NULL UNION + SELECT DISTINCT optionsetid from dataelement where optionsetid IS NOT NULL UNION + SELECT DISTINCT commentoptionsetid from dataelement where commentoptionsetid IS NOT NULL UNION + SELECT DISTINCT optionsetid FROM trackedentityattribute WHERE optionsetid IS NOT NULL + )) + SELECT COUNT(*) as value, + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM optionset), 0) as percent + from options_sets_unused; +details_sql: >- + SELECT DISTINCT uid,name FROM optionset where optionsetid NOT IN ( + SELECT DISTINCT optionsetid from attribute where optionsetid IS NOT NULL UNION + SELECT DISTINCT optionsetid from dataelement where optionsetid IS NOT NULL UNION + SELECT DISTINCT commentoptionsetid from dataelement where commentoptionsetid IS NOT NULL UNION + SELECT DISTINCT optionsetid FROM trackedentityattribute WHERE optionsetid IS NOT NULL) + ORDER BY name; +details_id_type: optionSets +severity: WARNING +introduction: > + Option sets should be used for some purpose. The may be used by data elements, comments, attributes, or tracked entity + attributes. +recommendation: > + Consider deleting unused option sets, or alternatively, ensure that they have been properly assigned. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/compulsory_orgunit_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/compulsory_orgunit_groups.yaml new file mode 100644 index 000000000000..79400adc9b81 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/compulsory_orgunit_groups.yaml @@ -0,0 +1,82 @@ +# Copyright (c) 2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: orgunits_compulsory_group_count +description: Orgunits that are not in all compulsory orgunit group sets +section: Organisation units +section_order: 2 +summary_sql: >- + WITH ougs_compulsory_members as ( + SELECT ou.uid,ou.name, + array_agg(DISTINCT(ougs.orgunitgroupsetid)) as ougs_members, + array_agg(DISTINCT(ougs.orgunitgroupsetid)) @> ( + SELECT array_agg(DISTINCT(orgunitgroupsetid)) from orgunitgroupset where compulsory = TRUE) as has_compulsory + FROM organisationunit ou + LEFT OUTER JOIN orgunitgroupmembers oug USING(organisationunitid) + LEFT OUTER JOIN ( + SELECT orgunitgroupsetid,orgunitgroupid from + orgunitgroupsetmembers where orgunitgroupsetid IN ( + SELECT orgunitgroupsetid from orgunitgroupset where compulsory = TRUE) + ) ougs USING(orgunitgroupid) + GROUP BY ou.uid,ou.name + ORDER BY name + ) + SELECT COUNT(*), + 100 * COUNT(*)/NULLIF( ( SELECT COUNT(*) FROM organisationunit), 0) as percent + FROM ougs_compulsory_members + where has_compulsory IS FALSE; +details_sql: >- + WITH orgunits_not_in_compulsory_groups as ( + SELECT ou.uid,ou.name, + array_agg(DISTINCT(ougs.orgunitgroupsetid)) as ougs_members, + array_agg(DISTINCT(ougs.orgunitgroupsetid)) @> ( + SELECT array_agg(DISTINCT(orgunitgroupsetid)) from orgunitgroupset where compulsory = TRUE) as has_compulsory + FROM organisationunit ou + LEFT OUTER JOIN orgunitgroupmembers oug USING(organisationunitid) + LEFT OUTER JOIN ( + SELECT orgunitgroupsetid,orgunitgroupid from + orgunitgroupsetmembers where orgunitgroupsetid IN ( + SELECT orgunitgroupsetid from orgunitgroupset where compulsory = TRUE) + ) ougs USING(orgunitgroupid) + GROUP BY ou.uid,ou.name + ORDER BY name) + SELECT uid, name from orgunits_not_in_compulsory_groups + where has_compulsory = FALSE + ORDER BY name; +details_id_type: organisationUnits +severity: SEVERE +introduction: > + If any organisation unit groups have been marked as compulsory, each and every organisation unit should + belong to exactly one group within each compulsory organisation unit group set. Assume we have created a + group set called "Ownership" with two groups "Public" and "Private". Each and every organisation + unit contained in the hierarchy must belong to either Public or Private (but not both). When organisation + unit are not part of a group set, results in the analytics apps will not be correct if the organisation + unit group set is used for aggregation. +recommendation: > + For each of the organisation units identified in the details query, you should assign the appropriate + organisation unit group within the group set. Alternatively, if the group set should not be compulsory, + you should change this attribute. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunit_null_island.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunit_null_island.yaml new file mode 100644 index 000000000000..8755a978cab6 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunit_null_island.yaml @@ -0,0 +1,71 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: orgunits_null_island +description: Organisation units located within 100 km of Null Island (0,0). +section: Organisation units +section_order: 6 +summary_sql: >- + WITH orgunits_null_island as ( + SELECT a.uid,a.name,ST_AsText(a.geometry) + from organisationunit a + INNER JOIN( + SELECT uid, ST_DistanceSphere(ST_SetSRID(ST_MakePoint(0.0,0.0),4326), + geometry) / 1000.0 as distance_km + from organisationunit + WHERE + ST_GeometryType(geometry) = 'ST_Point' + ) b ON a.uid = b.uid + WHERE distance_km < 100 + ) + SELECT + COUNT(*) as count, + 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM organisationunit), 0) as percent + FROM orgunits_null_island; +details_sql: >- + SELECT a.uid,a.name,ST_AsText(a.geometry) as comment + from organisationunit a + INNER JOIN( + SELECT uid, ST_DistanceSphere(ST_SetSRID(ST_MakePoint(0.0,0.0),4326), + geometry) / 1000.0 as distance_km + from organisationunit + WHERE + ST_GeometryType(geometry) = 'ST_Point' + ) b ON a.uid = b.uid + WHERE distance_km < 100 + ORDER BY name; +details_id_type: organisationUnits +severity: SEVERE +introduction: > + A common problem when importing coordinates is the inclusion of coordinates situated around + the point of [Null Island](https://en.wikipedia.org/wiki/Null_Island). This is the point + on the Earth's surface where the Prime Meridian and Equator intersect with a latitude of 0 + and a longitude of 0. The point also happens to be situated currently in the middle of the + ocean. This query identifies any points located within 100 km of the point having latitude + and longitude equal to zero. +recommendation: + Update the coordinates of the affected organization unit to the correct location. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunit_open_date_gt_closed_date.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunit_open_date_gt_closed_date.yaml new file mode 100644 index 000000000000..cbfed6aaf5e8 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunit_open_date_gt_closed_date.yaml @@ -0,0 +1,50 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: orgunits_openingdate_gt_closeddate +description: Organisation units which have an opening date later than the closed date. +section: Organisation units +summary_sql: >- + with orgunit_future_opening_date as (select uid,name,openingdate,closeddate + from organisationunit where openingdate > closeddate ) + select + count(*) as count, + 100*count(*)/NULLIF((select count(*) from organisationunit), 0) as percent + from orgunit_future_opening_date; +details_sql: >- + select uid,name + from organisationunit where openingdate > closeddate + ORDER BY name; +details_id_type: organisationUnits +severity: SEVERE +introduction: > + If a closing date has been defined for an organisation unit, it should always be + after the opening date (if one has been defined). +section_order: 5 +recommendation: > + Alter either the opening or closing date of all affected organisation units + so that the closing date is after the opening date. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_excess_group_memberships.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_excess_group_memberships.yaml new file mode 100644 index 000000000000..fe87a64289b9 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_excess_group_memberships.yaml @@ -0,0 +1,85 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: orgunit_group_sets_excess_groups +description: Organisation units which belong to multiple groups in a group set. +section: Organisation units +section_order: 10 +summary_sql: >- + WITH ougs_multiple_groups as ( + SELECT ou_uid,ou_name,ougs_name, + array_agg(oug_name) as ougs_groups + FROM ( + SELECT ougm.organisationunitid, + ougm.orgunitgroupid, + ougsm.orgunitgroupsetid, + ou.uid as ou_uid, + ou.name as ou_name, + oug.name as oug_name, + ougs.name as ougs_name + FROM orgunitgroupmembers ougm + INNER JOIN orgunitgroupsetmembers ougsm USING(orgunitgroupid) + INNER JOIN orgunitgroup oug USING(orgunitgroupid) + INNER JOIN orgunitgroupset ougs USING(orgunitgroupsetid) + INNER JOIN organisationunit ou USING(organisationunitid) ) a + GROUP BY ou_uid,ou_name, ougs_name ) + SELECT COUNT(*), + (100 * (SELECT COUNT(*) from ougs_multiple_groups WHERE array_length(ougs_groups,1) > 1) / + NULLIF((SELECT COUNT(*) from ougs_multiple_groups),0)) as percent + from ougs_multiple_groups + WHERE array_length(ougs_groups, 1) > 1; +details_sql: >- + WITH ougs_multiple_groups as ( + SELECT ou_uid,ou_name,ougs_name, + array_agg(oug_name) as ougs_groups + FROM ( + SELECT ougm.organisationunitid, + ougm.orgunitgroupid, + ougsm.orgunitgroupsetid, + ou.uid as ou_uid, + ou.name as ou_name, + oug.name as oug_name, + ougs.name as ougs_name + FROM orgunitgroupmembers ougm + INNER JOIN orgunitgroupsetmembers ougsm USING(orgunitgroupid) + INNER JOIN orgunitgroup oug USING(orgunitgroupid) + INNER JOIN orgunitgroupset ougs USING(orgunitgroupsetid) + INNER JOIN organisationunit ou USING(organisationunitid) ) a + GROUP BY ou_uid,ou_name, ougs_name ) + SELECT ou_uid as uid, ou_name as name, + ougs_name || ' :(' || array_to_string(ougs_groups, '; ') || ')' as comment + from ougs_multiple_groups + WHERE array_length(ougs_groups, 1) > 1; +details_id_type: organisationUnits +severity: SEVERE +introduction: > + Organisation units should belong to exactly one group within each organisation + unit group set of which they are a member. If the organisation unit belongs to + multiple groups, this will lead to unpredictable results in analysis. +recommendation: > + Using the maintenance app, assign the organisation units in the details list + to exactly one group within each group set membership. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_invalid_geometry.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_invalid_geometry.yaml new file mode 100644 index 000000000000..6ed9972de4a6 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_invalid_geometry.yaml @@ -0,0 +1,60 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: orgunits_invalid_geometry +description: Organisation units with invalid geometry. +section: Organisation units +section_order: 7 +summary_sql: >- + WITH orgunits_invalid_geometry as ( + SELECT uid,name,ST_IsValidReason(geometry) from organisationunit + where ST_IsValid(geometry) = FALSE) + SELECT + COUNT(*) as value, + 100.0 * COUNT(*) / NULLIF(( SELECT COUNT(*) FROM organisationunit where geometry IS NOT NULL),0) as percent + FROM orgunits_invalid_geometry; +details_sql: >- + SELECT uid,name,ST_IsValidReason(geometry) as comment from organisationunit + where ST_IsValid(geometry) = FALSE + ORDER BY name; +details_id_type: organisationUnits +severity: WARNING +introduction: | + DHIS2 uses the PostGIS database extension to manage the geographical information associated + with organisation units. There are various reasons why geometries may be considered to be + invalid including self-inclusions, self-intersections, and sliver polygons. Please see the + PostGIS documentation for a more in-depth discussion on this topic. + + Invalid geometry is not always a problem and may be able to be ignored, however, it has been observed in certain systems + that this can lead to problems in analytics. If you are experiencing issues when generating analytics + tables, and see errors related to invalid geometries, you will need to either remove the invalid geometry, + or fix it. +recommendation: > + Update the geometry of the affected organisation units to a valid geometry. It may be possible + to use the PostGIS function `ST_MakeValid` to automatically fix the problem. However, in other + cases the geometry may need to be edited in a GIS tool, and then updated again in DHIS2. + diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_multiple_roots.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_multiple_roots.yaml new file mode 100644 index 000000000000..d0df20dd1da2 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_multiple_roots.yaml @@ -0,0 +1,61 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: orgunits_multiple_roots +description: The organisation unit hierarchy should have a single root. +section: Organisation units +section_order: 8 +summary_sql: >- + WITH orgunit_multiple_roots as ( + SELECT uid,name, + (SELECT array_agg(name) from organisationunit where parentid IS NULL) as comment + FROM organisationunit + where parentid IS NULL ) + SELECT + COUNT(*) as count, + 100.0 * COUNT(*) / NULLIF( ( SELECT COUNT(*) FROM organisationunit), 0) as percent + FROM orgunit_multiple_roots + WHERE array_length(comment,1) > 1; +details_sql: >- + WITH duplicate_roots as ( + SELECT uid,name, + (SELECT array_agg(name) from organisationunit where parentid IS NULL) as comment + FROM organisationunit + where parentid IS NULL + ) + SELECT uid,name, + array_to_string(comment, ' ; ') as comment from duplicate_roots + where array_length(comment, 1) > 1; +details_id_type: organisationUnits +severity: CRITICAL +introduction: > + Every DHIS2 system should have a single root organisation unit. This means a single organisation + unit from which all other branches of the hierarchy are descendants. +recommendation: > + Once you have decided which organisation unit should be the real root of the organisation + unit hierarchy, you should update the parent organisation unit. This can be done by using + the DHIS2 API or my updating the value directly in the `organisationunit` table. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_multiple_spaces.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_multiple_spaces.yaml new file mode 100644 index 000000000000..4bae1195a97b --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_multiple_spaces.yaml @@ -0,0 +1,52 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: orgunits_multiple_spaces +description: Organisation units should not have multiple spaces in their names or shortnames. +section: Organisation units +section_order: 6 +summary_sql: >- + WITH orgunits_multiple_spaces as ( + SELECT uid, name from organisationunit + where name ~('\s{2,}') OR shortname ~('\s{2,}') + ) + SELECT + COUNT(*) as value, + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM organisationunit), 0) as percent + from orgunits_multiple_spaces; +details_sql: >- + SELECT uid, name, shortname as comment from organisationunit + where name ~('\s{2,}') OR shortname ~('\s{2,}'); +details_id_type: organisationUnits +severity: WARNING +introduction: > + Names and/or shortnames of organisation units should not contain multiple spaces. They are superfluous + and may complicate the location of organisation units when they are searched. +recommendation: > + If the number of affected organisation units is small, the easiest remedy is to correct + them directly from the user interface. Another possible option would be to replace + all of the multiple spaces using SQL. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_no_geometry.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_no_geometry.yaml new file mode 100644 index 000000000000..da3e2b984e6d --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_no_geometry.yaml @@ -0,0 +1,69 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: orgunits_no_coordinates +description: Organisation units with no coordinates. +section: Organisation units +section_order: 9 +summary_sql: >- + WITH orgunits_no_coordinates as ( + SELECT uid,name,path,hierarchylevel from organisationunit + where geometry IS NULL) + SELECT + COUNT(*)as value, + 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM organisationunit), 0) as percent + FROM orgunits_no_coordinates; +details_sql: >- + SELECT uid,name, CAST(hierarchylevel as text) as comment from organisationunit + where geometry IS NULL + ORDER BY name; +details_id_type: organisationUnits +severity: WARNING +introduction: | + Ideally, all organisation units contained in the DHIS2 hierarchy should have a valid + set of coordinates. Usually for all organisation units above the facility level, + these coordinates should be a polygon which provides the boundary of the organisation + unit. For facilities, these are usually represented as point coordinates. + + There can obviously be exceptions to this rule. Mobile health facilities may not have + a fixed location. Community health workers or wards below the facility level + may also not have a defined or definable coordinate. + + This check is intended to allow you to review all organisation units which do + not have any coordinates and make a determination as to whether they should be updated. + +recommendation: | + Where appropriate, update the geometry of each organisation unit with a valid geometry. + You may need to contact the appropriate local government office to obtain a copy + of district boundaries, commonly referred to as "shape files". Another possibility + is to use freely available boundary files from GADM (https://gadm.org) + + If facilities are missing coordinates, it may be possible to obtain these from + the facility staff using their smart phone to get the coordinates. Images + from Google Maps can also often be used to estimate the position of a facility, + assuming that you have good enough resolution and local knowledge of where + it is located. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_not_contained_by_parent.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_not_contained_by_parent.yaml new file mode 100644 index 000000000000..c927a4c914ee --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_not_contained_by_parent.yaml @@ -0,0 +1,70 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: orgunits_not_contained_by_parent +description: Organisation units with point coordinates should be contained by their parent. +section: Organisation units +section_order: 5 +summary_sql: >- + WITH parent_contains_point AS ( + SELECT a.uid,a.name from organisationunit a + INNER JOIN organisationunit b on a.parentid = b.organisationunitid + WHERE ST_GeometryType(a.geometry) = 'ST_Point' + AND ST_GeometryType(b.geometry) IN ('ST_Polygon','ST_MultiPolygon') + AND ST_Contains(b.geometry,a.geometry) = FALSE + ORDER BY a.name + ) + SELECT + COUNT(*) as value, + 100.0 * COUNT(*) / NULLIF((SELECT COUNT(*) + FROM organisationunit WHERE ST_GeometryType(geometry) = 'ST_Point'), 0) as percent + FROM parent_contains_point; +details_sql: >- + SELECT a.uid,a.name from organisationunit a + INNER JOIN organisationunit b on a.parentid = b.organisationunitid + WHERE ST_GeometryType(a.geometry) = 'ST_Point' + AND ST_GeometryType(b.geometry) IN ('ST_Polygon','ST_MultiPolygon') + AND ST_Contains(b.geometry,a.geometry) = FALSE + ORDER BY a.name; +details_id_type: organisationUnits +severity: WARNING +introduction: > + Facilities are often represented as points in the DHIS2 hierarchy. Their parent organisation units + geometry should contain all facilities which have been associated with them. + +recommendation: | + Often boundary files are simplified when they are uploaded into DHIS2. This process may result in + facilities which are located close to the border of a given district to fall outside of the district + when the boundary is simplified. This is considered to be more of a cosmetic problem for most DHIS2 + installations, but could become an issue if any geospatial analysis is attempted using the + boundaries and point coordinates. + + In cases where the facility falls outside of its parent's boundary + you should confirm that the coordinates are correct. If the location is close to the boundary, you + may want to reconsider how the boundary files have been simplified. Otherwise, if the location of + the facility is completely incorrect, it should be rectified. + diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_orphaned.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_orphaned.yaml new file mode 100644 index 000000000000..b03f7e8bb43a --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_orphaned.yaml @@ -0,0 +1,69 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: orgunits_orphaned +description: Orphaned organisation units. +section: Organisation units +section_order: 9 +summary_sql: >- + WITH orgunits_orphaned as ( + SELECT organisationunitid, children_count + FROM ( + SELECT organisationunitid,parentid + from organisationunit WHERE parentid IS NULL ) a + LEFT JOIN LATERAL ( + SELECT COUNT(*) as children_count + from organisationunit + WHERE a.organisationunitid = parentid ) b on TRUE + WHERE children_count = 0 + ) + SELECT + COUNT(*) as value, + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM organisationunit), 0) as percent + from orgunits_orphaned; +details_sql: >- + SELECT uid, name from organisationunit where + organisationunitid IN ( + SELECT organisationunitid + FROM ( + SELECT organisationunitid,parentid + from organisationunit WHERE parentid IS NULL ) a + LEFT JOIN LATERAL ( + SELECT COUNT(*) as children_count + from organisationunit + WHERE a.organisationunitid = parentid ) b on TRUE + WHERE children_count = 0 ); +details_id_type: organisationUnits +severity: CRITICAL +introduction: > + Orphaned organisation units are those which have neither parents nor any children. + This means that they have no relationship to the main organisation unit hierarchy. + These may be created by faulty metadata imports or direct manipulation of the database. +recommendation: > + The orphaned organisation units should be assigned a parent or removed from the system. + It is recommended to use the DHIS2 API for this task if possible. If this is not possible, + then they may need to be removed through direct SQL on the DHIS2 database. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_same_name_and_parent.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_same_name_and_parent.yaml new file mode 100644 index 000000000000..965c96accdec --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_same_name_and_parent.yaml @@ -0,0 +1,75 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: orgunits_same_name_and_parent +description: Organisation units should not have the same name and parent. +section: Organisation units +section_order: 12 +summary_sql: >- + WITH orgunits_same_name_and_parent as ( + SELECT d.uid,d.name,e.name as comment + from ( + SELECT parentid,unnest(ous) as organisationunitid FROM ( + SELECT parentid, trimmed_name, + array_agg(organisationunitid) as ous + FROM ( + SELECT organisationunitid,parentid, + regexp_replace(lower(name), '\s', '', 'g') as trimmed_name + from organisationunit) as a + GROUP BY parentid,trimmed_name + HAVING array_length(array_agg(organisationunitid),1) > 1 ) b ) c + INNER JOIN organisationunit d on c.organisationunitid = d.organisationunitid + INNER JOIN organisationunit e on c.parentid = e.organisationunitid ) + SELECT + COUNT(*) as value, + 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM organisationunit), 0) as percent + from orgunits_same_name_and_parent; +details_sql: >- + SELECT d.uid,d.name,e.name as comment + from ( + SELECT parentid,unnest(ous) as organisationunitid FROM ( + SELECT parentid, trimmed_name, + array_agg(organisationunitid) as ous + FROM ( + SELECT organisationunitid,parentid, + regexp_replace(lower(name), '\s', '', 'g') as trimmed_name + from organisationunit) as a + GROUP BY parentid,trimmed_name + HAVING array_length(array_agg(organisationunitid),1) > 1 ) b ) c + INNER JOIN organisationunit d on c.organisationunitid = d.organisationunitid + INNER JOIN organisationunit e on c.parentid = e.organisationunitid; +details_id_type: organisationUnits +severity: WARNING +introduction: > + Organisation units may have exactly the same name. This may become confusing to users, + particularly if the organisation unit has the same name and same parent. This can easily + lead to data entry or analysis mistakes as users have no way to distinguish between the + two organisation units. +recommendation: > + Rename identically named organisation units which share the same parent using the + maintenance app. + diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_trailing_spaces.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_trailing_spaces.yaml new file mode 100644 index 000000000000..809b25017def --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_trailing_spaces.yaml @@ -0,0 +1,50 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: orgunits_trailing_spaces +description: Organisation units should not have trailing spaces. +section: Organisation units +section_order: 4 +summary_sql: >- + WITH orgunits_trailing_spaces as ( + SELECT uid, name from organisationunit where name ~('\s+$') OR shortname ~('\s+$') + ) + SELECT + COUNT(*) as value, + 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM organisationunit), 0) as percent + from orgunits_trailing_spaces; +details_sql: >- + SELECT uid, name, shortname as comment from organisationunit where name ~('\s+$') or shortname ~('\s+$') + ORDER BY name; +details_id_type: organisationUnits +severity: WARNING +introduction: > + Trailing spaces in organisation units are superfluous. +recommendation: > + If the number of affected organisation units is small, the easiest remedy is to correct + them directly from the user interface. Another possible option would be to replace + all of the trailing spaces using SQL. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_3y_future.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_3y_future.yaml new file mode 100644 index 000000000000..0ffedb1ffdf8 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_3y_future.yaml @@ -0,0 +1,59 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- + name: periods_3y_future + description: Periods which are more than three years in the future. + section: Periods + section_order: 2 + summary_sql: >- + WITH periods_3y_future AS (SELECT periodid,startdate, + enddate from period where AGE(enddate,now()) >= INTERVAL '3 year' ) + SELECT COUNT(*) as value, + 100.0* COUNT(*) / NULLIF( (SELECT COUNT(*) FROM period), 0) as percent + FROM periods_3y_future; + details_sql: >- + SELECT NULL as uid, to_char(startdate, 'YYYY-MM-DD') || '-' || to_char(enddate,'YYYY-MM-DD') as name + from period where AGE(enddate,now()) >= INTERVAL '3 year' + details_id_type: periods + severity: WARNING + introduction: | + Periods in DHIS2 are automatically generated by the system. As new data is entered + into the system, new periods are automatically created. In some cases, periods + may mistakenly be created when data is sent to DHIS2 for periods which are in the + far future. Different data entry clients may not properly validate for periods + which are in the future, and thus any periods in the future should be reviewed. + In some cases, data may be valid for future dates, e.g. targets which are set for the + next fiscal year. + recommendation: | + If any periods exist in the system in the future, you should review the raw data + either directly in the datavalue table, or alternatively though the pivot tables + to ensure that this data is correct. + + In many cases, clients may mean to transmit + data for January 2021, but due to data entry errors, January 2031 is selected. Thus, + any data in the far future should be investigated to ensure it does not result + from data entry errors. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_distant_past.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_distant_past.yaml new file mode 100644 index 000000000000..8be93651b374 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_distant_past.yaml @@ -0,0 +1,53 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- + name: periods_distant_past + description: Periods which are in the distant past. + section: Periods + section_order: 3 + summary_sql: >- + WITH periods_distant_past AS (SELECT periodid,startdate, + enddate from period where AGE(now(),enddate) >= INTERVAL '30 year' ) + SELECT COUNT(*) as value, + 100.0* COUNT(*) / NULLIF( (SELECT COUNT(*) FROM period) , 0) as percent + FROM periods_distant_past; + details_sql: >- + SELECT NULL as uid, to_char(startdate, 'YYYY-MM-DD') || '-' || to_char(enddate, 'YYYY-MM-DD') as name + from period where AGE(now(),enddate) >= INTERVAL '30 year'; + severity: WARNING + introduction: | + Periods in DHIS2 are automatically generated by the system. As new data is entered + into the system, new periods are automatically created. In some cases, periods + may mistakenly be created when data is sent to DHIS2 for periods which are in the + distant past. Different data entry clients may not properly validate for periods + which are in the distant past, and thus these periods should be triaged to ensure + that data has not been entered against them by mistake. + details_id_type: periods + recommendation: | + If any periods exist in the system in the distant past, you should review the raw data + either directly in the datavalue table, or alternatively though the pivot tables + to ensure that this data is correct. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_same_start_end_date.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_same_start_end_date.yaml new file mode 100644 index 000000000000..a69d658f8f44 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_same_start_end_date.yaml @@ -0,0 +1,60 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- + name: periods_same_start_end_date + description: Periods with the same start and end dates + section: Periods + section_order: 1 + summary_sql: >- + WITH bad_periods AS ( + select p1.periodtypeid, p1.startdate, p1.enddate + from period p1 + inner join period p2 + on p1.periodtypeid = p2.periodtypeid + and p1.startdate = p2.startdate + where p1.periodid != p2.periodid + order by p1.periodtypeid, p1.startdate, p1.enddate ) + SELECT COUNT(*) as value, + 100.0* COUNT(*) / NULLIF( (SELECT COUNT(*) FROM period), 0) as percent + FROM bad_periods; + details_sql: >- + select pt.name as uid, + CAST( p1.startdate as text) as name + from period p1 + inner join period p2 + on p1.periodtypeid = p2.periodtypeid + and p1.startdate = p2.startdate + INNER JOIN periodtype pt on p1.periodtypeid = pt.periodtypeid + where p1.periodid != p2.periodid + order by p1.periodtypeid, p1.startdate; + severity: CRITICAL + introduction: > + Different periods should not have exactly the same start and end date. + details_id_type: periods + recommendation: > + All references to the duplicate periods should be removed from the system and + reassigned. It is recommended to use the period with the lower periodid. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_message_no_template.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_message_no_template.yaml new file mode 100644 index 000000000000..9272c62afff4 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_message_no_template.yaml @@ -0,0 +1,54 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: program_rules_message_no_template +description: Program rules actions which should send or schedule a message without a message template. +section: Program rules +section_order: 4 +summary_sql: >- + WITH program_rules_no_message_template as ( + SELECT uid,name from programrule where programruleid IN ( + SELECT programruleid from programruleaction WHERE + actiontype IN ('SENDMESSAGE','SCHEDULEMESSAGE') + AND templateuid IS NULL )) + SELECT COUNT(*) as value, + 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM programrule),0) as percent + FROM program_rules_no_message_template; +details_sql: >- + SELECT uid,name from programrule where programruleid IN ( + SELECT programruleid from programruleaction WHERE + actiontype IN ('SENDMESSAGE','SCHEDULEMESSAGE') + AND templateuid IS NULL ); +severity: SEVERE +introduction: > + Program rule actions of type "Send message" or "Schedule message" + should have an associated message template. +details_id_type: programRules +recommendation: > + Using the DHIS2 user interface, assign a message template to each + of the program rule actions which send or schedule messages but + which does not have an association with a message template. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_no_action.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_no_action.yaml new file mode 100644 index 000000000000..b29f7648ed3f --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_no_action.yaml @@ -0,0 +1,49 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: program_rules_no_action +description: Program rules with no action. +section: Program rules +section_order: 2 +summary_sql: >- + WITH program_rules_no_action as ( + SELECT uid from programrule where programruleid + NOT IN (SELECT DISTINCT programruleid from programruleaction) ) + SELECT COUNT(*) as value, + 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM programrule),0) as percent + FROM program_rules_no_action; +details_sql: >- + SELECT uid,name from programrule where programruleid + NOT IN (SELECT DISTINCT programruleid from programruleaction); +details_id_type: programRules +severity: SEVERE +introduction: > + All program rules should have an action. +recommendation: > + Using the DHIS2 user interface, assign an action to each of the program rules + which is missing one. Alternatively, if the program rule is not in use, then + consider removing it. \ No newline at end of file diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_no_expression.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_no_expression.yaml new file mode 100644 index 000000000000..e467596d7632 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_no_expression.yaml @@ -0,0 +1,49 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- + name: program_rules_no_expression + description: Program rules with no expression. + section: Program rules + section_order: 2 + summary_sql: >- + WITH program_rules_no_expression as ( + SELECT uid,name from programrule where rulecondition IS NULL ) + SELECT COUNT(*) as value, + 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM programrule), 0) as percent + FROM program_rules_no_expression; + details_sql: >- + SELECT uid,name from programrule where rulecondition IS NULL; + severity: WARNING + introduction: > + All program rules should have an expression. Expressions are used by program rules + to determine when they should be triggered. Program rules with no expression have + no purpose. + details_id_type: programRules + recommendation: > + Using the DHIS2 user interface, assign an expression to each of the program rules + which is missing one. Otherwise, if it is safe to delete the program rule with + no expression, it is recommended to remove it. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/validation_rules/validation_rules_undefined_missing_value_strategy.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/validation_rules/validation_rules_undefined_missing_value_strategy.yaml new file mode 100644 index 000000000000..9eed218c9784 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/validation_rules/validation_rules_undefined_missing_value_strategy.yaml @@ -0,0 +1,56 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: validation_rules_missing_value_strategy_null +description: All validation rule expressions should have a missing value strategy. +section: Validation rules +section_order: 1 +summary_sql: >- + SELECT COUNT(*) as value, + 100.0 * count(*) / NULLIF( (select count(*) from expression), 0) as percent + from expression where missingvaluestrategy IS NULL; +details_sql: >- + SELECT a.uid,a.name,'RHS' as comment + from validationrule a + INNER JOIN expression b on a.rightexpressionid = b.expressionid + where b.missingvaluestrategy IS NULL + UNION + SELECT a.uid,a.name,'LHS' as comment + from validationrule a + INNER JOIN expression b on a.leftexpressionid = b.expressionid + where b.missingvaluestrategy IS NULL; +details_id_type: validationRules +severity: SEVERE +introduction: > + Validation rules are composed of a left and right side expression. In certain systems + the missing value strategy may not be defined. This may lead to an exception during + validation rule analysis. The affected validation rules should be corrected to + with an appropriate missing value strategy. +recommendation: > + Using the results of the the details SQL view, identify the affected validation rules + and which side of the rule the missing value strategy has not been specified. Using the + maintenance app, make the appropriate corrections and save the rule. \ No newline at end of file diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityServiceTest.java b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityServiceTest.java index e751f0b76fa9..7d5a28ee7126 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityServiceTest.java @@ -43,18 +43,21 @@ import static org.hisp.dhis.dataintegrity.DataIntegrityDetails.DataIntegrityIssue.issueName; import static org.hisp.dhis.utils.Assertions.assertContainsOnly; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.clearInvocations; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -72,6 +75,8 @@ import org.hisp.dhis.dataset.DataSet; import org.hisp.dhis.dataset.DataSetService; import org.hisp.dhis.expression.ExpressionService; +import org.hisp.dhis.external.location.DefaultLocationManager; +import org.hisp.dhis.i18n.I18n; import org.hisp.dhis.i18n.I18nManager; import org.hisp.dhis.indicator.Indicator; import org.hisp.dhis.indicator.IndicatorGroup; @@ -100,6 +105,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -113,6 +119,15 @@ class DataIntegrityServiceTest { @Mock private I18nManager i18nManager; + @Mock private I18n i18n; + @Mock private DefaultLocationManager locationManager; + + @Mock private SchemaService schemaService; + + @Mock private CacheProvider cacheProvider; + + @Mock private DataIntegrityStore dataIntegrityStore; + @Mock private DataElementService dataElementService; @Mock private IndicatorService indicatorService; @@ -145,7 +160,7 @@ class DataIntegrityServiceTest { @Mock private ProgramRuleActionService programRuleActionService; - private DefaultDataIntegrityService subject; + @InjectMocks private DefaultDataIntegrityService subject; private DataElementGroup elementGroupA; @@ -197,26 +212,6 @@ class DataIntegrityServiceTest { @BeforeEach public void setUp() { - subject = - new DefaultDataIntegrityService( - i18nManager, - programRuleService, - programRuleActionService, - programRuleVariableService, - dataElementService, - indicatorService, - dataSetService, - organisationUnitService, - organisationUnitGroupService, - validationRuleService, - expressionService, - dataEntryFormService, - categoryService, - periodService, - programIndicatorService, - mock(CacheProvider.class), - mock(DataIntegrityStore.class), - mock(SchemaService.class)); setUpFixtures(); } @@ -608,6 +603,15 @@ void testValidProgramIndicatorFilter() { assertTrue(issues.isEmpty()); } + @Test + void testGetDataIntegrityChecks_ReadDataIntegrityYamlFilesOnClassPath() { + when(i18nManager.getI18n(DataIntegrityService.class)).thenReturn(i18n); + when(i18n.getString(anyString(), anyString())).thenReturn("default"); + when(i18n.getString(contains("severity"), eq("WARNING"))).thenReturn("WARNING"); + Collection dataIntegrityChecks = subject.getDataIntegrityChecks(); + assertFalse(dataIntegrityChecks.isEmpty()); + } + private Map createRandomDataElements(int quantity, String uidSeed) { return IntStream.range(1, quantity + 1) diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java index 58e7dcad3306..86f31d6d5499 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java @@ -27,13 +27,21 @@ */ package org.hisp.dhis.dataintegrity; +import static java.util.stream.Collectors.toList; +import static org.hisp.dhis.dataintegrity.DataIntegrityYamlReader.ResourceLocation.CLASS_PATH; +import static org.hisp.dhis.dataintegrity.DataIntegrityYamlReader.ResourceLocation.FILE_SYSTEM; import static org.hisp.dhis.dataintegrity.DataIntegrityYamlReader.readDataIntegrityYaml; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Set; +import java.util.function.Predicate; +import java.util.regex.Pattern; +import java.util.stream.Stream; import org.hisp.dhis.dataintegrity.DataIntegrityDetails.DataIntegrityIssue; import org.junit.jupiter.api.Test; @@ -45,20 +53,35 @@ class DataIntegrityYamlReaderTest { @Test void testReadDataIntegrityYaml() { + List checks = new ArrayList<>(); - readDataIntegrityYaml( - "data-integrity-checks.yaml", - checks::add, - (property, defaultValue) -> defaultValue, - sql -> check -> new DataIntegritySummary(check, new Date(), null, 1, 100d), - sql -> - check -> - new DataIntegrityDetails( - check, - new Date(), - null, - List.of(new DataIntegrityIssue("id", "name", sql, List.of())))); - assertEquals(6, checks.size()); + readYaml(checks, "data-integrity-checks.yaml", "data-integrity-checks", CLASS_PATH); + assertEquals(63, checks.size()); + + // Names should be unique + List allNames = checks.stream().map(DataIntegrityCheck::getName).collect(toList()); + assertEquals(allNames.size(), Set.copyOf(allNames).size()); + + // Config checks and Java checks should not have any of the same names + List nonYamlChecks = + Stream.of(DataIntegrityCheckType.values()) + .map(e -> e.getName().toLowerCase()) + .collect(toList()); + assertTrue(nonYamlChecks.size() > 0); + nonYamlChecks.retainAll(allNames); + assertEquals(0, nonYamlChecks.size()); + + // Assert that all "codes" are unique. + List codeList = + checks.stream().map(DataIntegrityCheck::getCode).sorted().collect(toList()); + assertEquals(codeList.size(), Set.copyOf(codeList).size()); + + // Assert that codes consist of upper case letter and numbers only + String regEx = "^[A-Z0-9]+$"; + Predicate IS_NOT_CAPS = Pattern.compile(regEx).asPredicate().negate(); + List badCodes = codeList.stream().filter(IS_NOT_CAPS).collect(toList()); + assertEquals(0, badCodes.size()); + DataIntegrityCheck check = checks.get(0); assertEquals("categories_no_options", check.getName()); assertEquals("Categories with no category options", check.getDescription()); @@ -66,13 +89,13 @@ void testReadDataIntegrityYaml() { assertEquals("categories", check.getIssuesIdType()); assertEquals(DataIntegritySeverity.WARNING, check.getSeverity()); assertEquals( - "Categories should always have at least a single category options.", - check.getIntroduction()); + "Categories should always have at least one category option.", check.getIntroduction()); assertEquals( "Any categories without category options should either be removed from the" + " system if they are not in use. Otherwise, appropriate category options" + " should be added to the category.", check.getRecommendation()); + assertFalse(check.isSlow()); assertTrue( check .getRunDetailsCheck() @@ -82,4 +105,63 @@ void testReadDataIntegrityYaml() { .getComment() .startsWith("SELECT uid,name from dataelementcategory")); } + + @Test + void testWithValidChecksFile() { + List checks = new ArrayList<>(); + readYaml(checks, "test-data-integrity-checks.yaml", "test-data-integrity-checks", CLASS_PATH); + assertEquals(1, checks.size()); + } + + @Test + void testWithInvalidChecksFile() { + List checks = new ArrayList<>(); + readYaml(checks, "invalid-file-checks.yaml", "test-data-integrity-checks", CLASS_PATH); + assertEquals(0, checks.size()); + } + + @Test + void testWithInvalidChecksDirectory() { + List checks = new ArrayList<>(); + readYaml(checks, "test-data-integrity-checks.yaml", "invalid-integrity-checks", CLASS_PATH); + assertEquals(0, checks.size()); + } + + @Test + void testWithInvalidChecksDirectoryFromFileSystem() { + List checks = new ArrayList<>(); + readYaml(checks, "data-integrity-checks.yaml", "invalid-integrity-checks", FILE_SYSTEM); + assertEquals(0, checks.size()); + } + + @Test + void testWithInvalidYamlFormat() { + List checks = new ArrayList<>(); + readYaml( + checks, "test-data-integrity-checks.yaml", "test-data-integrity-checks.yaml", FILE_SYSTEM); + assertEquals(0, checks.size()); + } + + private void readYaml( + List checks, + String fileChecks, + String checksDirectory, + DataIntegrityYamlReader.ResourceLocation resourceLocation) { + readDataIntegrityYaml( + new DefaultDataIntegrityService.DataIntegrityRecord( + resourceLocation, + fileChecks, + checksDirectory, + checks::add, + (property, defaultValue) -> defaultValue, + sql -> check -> new DataIntegritySummary(check, new Date(), new Date(), null, 1, 100d), + sql -> + check -> + new DataIntegrityDetails( + check, + new Date(), + new Date(), + null, + List.of(new DataIntegrityIssue("id", "name", sql, List.of()))))); + } } diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/jsonschema/JsonSchemaValidatorTest.java b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/jsonschema/JsonSchemaValidatorTest.java new file mode 100644 index 000000000000..7abfeb20950e --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/jsonschema/JsonSchemaValidatorTest.java @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.jsonschema; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.networknt.schema.ValidationMessage; +import java.io.IOException; +import java.io.InputStream; +import java.util.Locale; +import java.util.Set; +import java.util.stream.Collectors; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.springframework.core.io.ClassPathResource; + +class JsonSchemaValidatorTest { + + private final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + + @BeforeAll() + static void setup() { + Locale.setDefault(new Locale("en", "US")); + } + + @Test + void validateCheck_ValidFile() { + JsonNode jsonNode = getJsonNodeFromFile("check_with_all_required_fields.yaml"); + Set validationMessages = + JsonSchemaValidator.validateDataIntegrityCheck(jsonNode); + assertTrue(validationMessages.isEmpty()); + } + + @Test + void validateCheck_CheckMissingDescription() { + JsonNode jsonNode = getJsonNodeFromFile("check_without_description.yaml"); + Set validationMessages = + JsonSchemaValidator.validateDataIntegrityCheck(jsonNode); + assertEquals(1, validationMessages.size()); + assertContainsValidationMessages( + validationMessages, Set.of("$.description: is missing but it is required")); + } + + @Test + void validateCheck_CheckMissingName() { + JsonNode jsonNode = getJsonNodeFromFile("check_without_name.yaml"); + Set validationMessages = + JsonSchemaValidator.validateDataIntegrityCheck(jsonNode); + assertEquals(1, validationMessages.size()); + assertContainsValidationMessages( + validationMessages, Set.of("$.name: is missing but it is required")); + } + + @Test + void validateCheck_CheckMissing10Fields() { + JsonNode jsonNode = getJsonNodeFromFile("check_without_10_required_fields.yaml"); + Set validationMessages = + JsonSchemaValidator.validateDataIntegrityCheck(jsonNode); + assertEquals(8, validationMessages.size()); + assertContainsValidationMessages( + validationMessages, + Set.of( + "$.details_id_type: is missing but it is required", + "$.details_sql: is missing but it is required", + "$.introduction: is missing but it is required", + "$.recommendation: is missing but it is required", + "$.section: is missing but it is required", + "$.section_order: is missing but it is required", + "$.severity: is missing but it is required", + "$.summary_sql: is missing but it is required")); + } + + private void assertContainsValidationMessages( + Set validations, Set expected) { + assertTrue( + validations.stream() + .map(ValidationMessage::getMessage) + .collect(Collectors.toSet()) + .containsAll(expected)); + } + + private JsonNode getJsonNodeFromFile(String file) { + try (InputStream is = + new ClassPathResource("test-data-integrity-checks/" + file).getInputStream()) { + return mapper.readTree(is); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks.yaml b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks.yaml new file mode 100644 index 000000000000..5f6b72c696ae --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks.yaml @@ -0,0 +1,6 @@ +checks: + - check_with_all_required_fields.yaml + - check_without_10_required_fields.yaml + - check_without_description.yaml + - check_without_name.yaml + - invalid_yaml_format.yaml diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_with_all_required_fields.yaml b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_with_all_required_fields.yaml new file mode 100644 index 000000000000..607cc2b28da1 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_with_all_required_fields.yaml @@ -0,0 +1,15 @@ +--- +name: test_description_missing +description: test_description +section: test +section_order: 1 +summary_sql: >- + SQL here +details_sql: >- + SQL here +details_id_type: test +severity: CRITICAL +introduction: > + Intro here +recommendation: > + Recommendation here diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_without_10_required_fields.yaml b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_without_10_required_fields.yaml new file mode 100644 index 000000000000..0ade6e80d781 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_without_10_required_fields.yaml @@ -0,0 +1,3 @@ +--- +name: name +description: test_name_missing diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_without_description.yaml b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_without_description.yaml new file mode 100644 index 000000000000..be450286990f --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_without_description.yaml @@ -0,0 +1,14 @@ +--- +name: test_description_missing +section: test +section_order: 1 +summary_sql: >- + SQL here +details_sql: >- + SQL here +details_id_type: test +severity: CRITICAL +introduction: > + Intro here +recommendation: > + Recommendation here diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_without_name.yaml b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_without_name.yaml new file mode 100644 index 000000000000..6344ff376448 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_without_name.yaml @@ -0,0 +1,14 @@ +--- +description: test_name_missing +section: test +section_order: 1 +summary_sql: >- + SQL here +details_sql: >- + SQL here +details_id_type: test +severity: CRITICAL +introduction: > + Intro here +recommendation: > + Recommendation here diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/invalid_yaml_format.yaml b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/invalid_yaml_format.yaml new file mode 100644 index 000000000000..9d1fded7d807 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/invalid_yaml_format.yaml @@ -0,0 +1,13 @@ +description test_name_missing +section: test +section_order: 1 +summary_sql: >- + SQL here +details_sql: >- + SQL here +details_id_type: test +severity: CRITICAL +introduction: > + Intro here +recommendation: > + Recommendation here diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonCategory.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonCategory.java new file mode 100644 index 000000000000..db907fb75a3d --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonCategory.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.json.domain; + +import org.hisp.dhis.common.DimensionType; +import org.hisp.dhis.jsontree.JsonList; + +/** + * Web API equivalent of a {@link org.hisp.dhis.category.Category}. + * + * @author Jan Bernitt + * @author Jason P. Pickering + */ +public interface JsonCategory extends JsonIdentifiableObject { + default JsonList getCategoryOptions() { + return getList("categoryOptions", JsonCategoryOption.class); + } + + default JsonList getCategoryCombos() { + return getList("categoryCombos", JsonCategoryCombo.class); + } + + default DimensionType getDimensionType() { + return getString("dimensionType").parsed(DimensionType::valueOf); + } +} diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonCategoryCombo.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonCategoryCombo.java new file mode 100644 index 000000000000..5f2cac0f9bf5 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonCategoryCombo.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.json.domain; + +import org.hisp.dhis.common.DimensionType; +import org.hisp.dhis.jsontree.JsonList; + +/** + * Web API equivalent of a {@link org.hisp.dhis.category.Category}. + * + * @author Jan Bernitt + * @author Jason P. Pickering + */ +public interface JsonCategoryCombo extends JsonIdentifiableObject { + default JsonList getCategories() { + return getList("categories", JsonCategory.class); + } + + default DimensionType getDimensionType() { + return getString("dimensionType").parsed(DimensionType::valueOf); + } +} diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonCategoryOption.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonCategoryOption.java new file mode 100644 index 000000000000..928b85059b92 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonCategoryOption.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.json.domain; + +import org.hisp.dhis.jsontree.JsonList; + +/** + * @author Jason Pickering + */ +public interface JsonCategoryOption extends JsonIdentifiableObject { + default JsonList getCategories() { + return getList("categories", JsonCategory.class); + } + + default JsonList getCatOptionCombos() { + return getList("categoryOptionCombos", JsonCategoryOptionCombo.class); + } +} diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonCategoryOptionCombo.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonCategoryOptionCombo.java new file mode 100644 index 000000000000..0b0c29df814f --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonCategoryOptionCombo.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.json.domain; + +import org.hisp.dhis.jsontree.JsonList; + +/** + * @author Jason Pickering + */ +public interface JsonCategoryOptionCombo extends JsonIdentifiableObject { + default JsonCategoryCombo getCategoryCombo() { + return get("categoryCombo", JsonCategoryCombo.class); + } + + default JsonList getCategoryOptions() { + return getList("categoryOptions", JsonCategoryOption.class); + } + + default Boolean getIgnoreApproval() { + return getBoolean("ignoreApproval").bool(); + } +} diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataElement.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataElement.java new file mode 100644 index 000000000000..6e359e320ca0 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataElement.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.json.domain; + +import org.hisp.dhis.analytics.AggregationType; +import org.hisp.dhis.dataelement.DataElementDomain; + +/** + * Web API equivalent of a {@link org.hisp.dhis.dataelement}. + * + * @author Jan Bernitt + * @author Jason P. Pickering + */ +public interface JsonDataElement extends JsonIdentifiableObject { + default DataElementDomain getDomainType() { + return getString("domainType").parsed(DataElementDomain::valueOf); + } + + default AggregationType getAggregationType() { + return getString("aggregationType").parsed(AggregationType::valueOf); + } + + default JsonOptionSet getOptionSet() { + return get("optionSet", JsonOptionSet.class); + } +} diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataElementGroup.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataElementGroup.java new file mode 100644 index 000000000000..e1ff83a719c2 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataElementGroup.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.json.domain; + +import org.hisp.dhis.jsontree.JsonList; + +/** + * Web API equivalent of a {@link org.hisp.dhis.dataelement}. + * + * @author Jan Bernitt + * @author Jason P. Pickering + */ +public interface JsonDataElementGroup extends JsonIdentifiableObject { + default JsonList getDataElements() { + return getList("dataElements", JsonDataElement.class); + } +} diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityCheck.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityCheck.java index 81efd2ff67c1..6a064f8f8c11 100644 --- a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityCheck.java +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityCheck.java @@ -70,4 +70,16 @@ default String getRecommendation() { default String getIssuesIdType() { return getString("issuesIdType").string(); } + + default boolean getIsSlow() { + return getBoolean("isSlow").booleanValue(); + } + + default boolean getIsProgrammatic() { + return getBoolean("isProgrammatic").booleanValue(); + } + + default String getCode() { + return getString("code").string(); + } } diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityDetails.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityDetails.java index acefb76208ff..7f371635bf59 100644 --- a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityDetails.java +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityDetails.java @@ -45,6 +45,11 @@ default LocalDateTime getFinishedTime() { return get("finishedTime", JsonDate.class).date(); } + @Expected + default LocalDateTime getStartTime() { + return get("startTime", JsonDate.class).date(); + } + default String getError() { return getString("error").string(null); } diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegritySummary.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegritySummary.java index 2b9390561b14..f79d8179c038 100644 --- a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegritySummary.java +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegritySummary.java @@ -42,6 +42,11 @@ default LocalDateTime getFinishedTime() { return get("finishedTime", JsonDate.class).date(); } + @Expected + default LocalDateTime getStartTime() { + return get("startTime", JsonDate.class).date(); + } + default String getError() { return getString("error").string(null); } diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonGenerator.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonGenerator.java index 9467f2e113bb..4a65aab176af 100644 --- a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonGenerator.java +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonGenerator.java @@ -104,7 +104,8 @@ private String createObject(JsonSchema schema, boolean addId, Map 0) { diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonIndicator.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonIndicator.java new file mode 100644 index 000000000000..5425efe50e57 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonIndicator.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.json.domain; + +import org.hisp.dhis.jsontree.JsonList; + +public interface JsonIndicator extends JsonIdentifiableObject { + default String getNumerator() { + return getString("numerator").string(); + } + + default String getNumeratorDescription() { + return getString("numeratorDescription").string(); + } + + default String getDenominator() { + return getString("denominator").string(); + } + + default String getDenominatorDescription() { + return getString("denominatorDescription").string(); + } + + default Boolean isAnnualized() { + return Boolean.valueOf(getString("annualized").string()); + } + + default JsonList getIndicatorGroups() { + return getList("indicatorGroups", JsonIndicatorGroup.class); + } +} diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonIndicatorGroup.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonIndicatorGroup.java new file mode 100644 index 000000000000..23d059e59bbb --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonIndicatorGroup.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.json.domain; + +import org.hisp.dhis.jsontree.JsonList; + +public interface JsonIndicatorGroup extends JsonIdentifiableObject { + + default JsonList getIndicators() { + return getList("indicators", JsonIndicator.class); + } +} diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonOption.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonOption.java new file mode 100644 index 000000000000..34c49d959276 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonOption.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.json.domain; + +import org.hisp.dhis.jsontree.JsonList; + +/** + * @author Jason Pickering + */ +public interface JsonOption extends JsonIdentifiableObject { + default JsonList getOptionSet() { + return getList("optionSet", JsonOptionSet.class); + } + + default Number getSortOrder() { + return getNumber("sortOrder").number(); + } +} diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonOptionSet.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonOptionSet.java new file mode 100644 index 000000000000..92cc46cf5fb8 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonOptionSet.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.json.domain; + +import org.hisp.dhis.jsontree.JsonList; + +/** + * @author Jason Pickering + */ +public interface JsonOptionSet extends JsonIdentifiableObject { + default JsonList getOptions() { + return getList("options", JsonOption.class); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/AbstractDataIntegrityIntegrationTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/AbstractDataIntegrityIntegrationTest.java new file mode 100644 index 000000000000..192430d73a3b --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/AbstractDataIntegrityIntegrationTest.java @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; +import static org.junit.jupiter.api.Assertions.*; + +import java.util.Set; +import java.util.stream.Collectors; +import org.hisp.dhis.jsontree.*; +import org.hisp.dhis.web.HttpStatus; +import org.hisp.dhis.webapi.DhisControllerIntegrationTest; +import org.hisp.dhis.webapi.json.domain.JsonDataIntegrityDetails; +import org.hisp.dhis.webapi.json.domain.JsonDataIntegritySummary; +import org.hisp.dhis.webapi.json.domain.JsonWebMessage; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.support.TransactionTemplate; + +class AbstractDataIntegrityIntegrationTest extends DhisControllerIntegrationTest { + final JsonDataIntegrityDetails getDetails(String check) { + + JsonObject content = GET("/dataIntegrity/details?checks={check}&timeout=1000", check).content(); + JsonDataIntegrityDetails details = + content.get(check.replace('-', '_'), JsonDataIntegrityDetails.class); + assertTrue( + details.exists(), "check " + check + " did not complete in time or threw an exception"); + assertTrue(details.isObject()); + return details; + } + + final void postDetails(String check) { + HttpResponse trigger = POST("/dataIntegrity/details?checks=" + check); + assertEquals("http://localhost/dataIntegrity/details?checks=" + check, trigger.location()); + assertTrue(trigger.content().isA(JsonWebMessage.class)); + } + + final JsonDataIntegritySummary getSummary(String check) { + JsonObject content = GET("/dataIntegrity/summary?checks={check}&timeout=1000", check).content(); + JsonDataIntegritySummary summary = + content.get(check.replace('-', '_'), JsonDataIntegritySummary.class); + assertTrue(summary.exists()); + assertTrue(summary.isObject()); + return summary; + } + + public final void postSummary(String check) { + HttpResponse trigger = POST("/dataIntegrity/summary?checks=" + check); + assertEquals("http://localhost/dataIntegrity/summary?checks=" + check, trigger.location()); + assertTrue(trigger.content().isA(JsonWebMessage.class)); + } + + void checkDataIntegritySummary( + String check, Integer expectedCount, Integer expectedPercentage, Boolean hasPercentage) { + + postSummary(check); + + JsonDataIntegritySummary summary = getSummary(check); + assertEquals(expectedCount, summary.getCount()); + + if (hasPercentage) { + assertEquals(expectedPercentage, summary.getPercentage().intValue()); + } else { + assertNull(summary.getPercentage()); + } + } + + private Boolean hasComments(JsonList issues) { + return issues.stream() + .map(issue -> issue.has("comment")) + .reduce(Boolean.FALSE, Boolean::logicalOr); + } + + void checkDataIntegrityDetailsIssues( + String check, + String expectedDetailsUnits, + String expectedDetailsNames, + String expectedDetailsComments, + String issueType) { + postDetails(check); + + JsonDataIntegrityDetails details = getDetails(check); + JsonList issues = details.getIssues(); + assertTrue(issues.exists()); + assertEquals(1, issues.size()); + + if (expectedDetailsUnits != null) { + assertEquals(expectedDetailsUnits, issues.get(0).getId()); + } else { + assertNull(issues.get(0).getId()); + } + + if (expectedDetailsNames != null) { + assertTrue(issues.get(0).getName().startsWith(expectedDetailsNames)); + } + + /* This can be empty if comments do not exist in the JSON response. */ + + if (hasComments(issues) && expectedDetailsComments != null) { + assertTrue(issues.get(0).getComment().toString().contains(expectedDetailsComments)); + } + + assertEquals(issueType, details.getIssuesIdType()); + } + + void checkDataIntegrityDetailsIssues( + String check, + Set expectedDetailsUnits, + Set expectedDetailsNames, + Set expectedDetailsComments, + String issueType) { + + postDetails(check); + + JsonDataIntegrityDetails details = getDetails(check); + JsonList issues = details.getIssues(); + + assertTrue(issues.exists()); + assertEquals(expectedDetailsUnits.size(), issues.size()); + + /* Always check the UIDs */ + Set issueUIDs = issues.stream().map(issue -> issue.getId()).collect(Collectors.toSet()); + assertEquals(issueUIDs, expectedDetailsUnits); + + /* + * Names can be optionally checked, but should always exist in the + * response + */ + if (!expectedDetailsNames.isEmpty()) { + Set detailsNames = + issues.stream().map(issue -> issue.getName()).collect(Collectors.toSet()); + assertEquals(expectedDetailsNames, detailsNames); + } + /* This can be empty if comments do not exist in the JSON response. */ + if (hasComments(issues) && !expectedDetailsComments.isEmpty()) { + Set detailsComments = + issues.stream() + .map(issue -> issue.getComment().string()) + .collect(Collectors.toUnmodifiableSet()); + assertEquals(expectedDetailsComments, detailsComments); + } + + assertEquals(issueType, details.getIssuesIdType()); + } + + final void assertHasDataIntegrityIssues( + String issueType, + String check, + Integer expectedPercentage, + String expectedDetailsUnit, + String expectedDetailsName, + String expectedDetailsComment, + Boolean hasPercentage) { + checkDataIntegritySummary(check, 1, expectedPercentage, hasPercentage); + + checkDataIntegrityDetailsIssues( + check, expectedDetailsUnit, expectedDetailsName, expectedDetailsComment, issueType); + } + + final void assertHasDataIntegrityIssues( + String issueType, + String check, + Integer expectedPercentage, + Set expectedDetailsUnits, + Set expectedDetailsNames, + Set expectedDetailsComments, + Boolean hasPercentage) { + checkDataIntegritySummary( + check, expectedDetailsUnits.size(), expectedPercentage, hasPercentage); + checkDataIntegrityDetailsIssues( + check, expectedDetailsUnits, expectedDetailsNames, expectedDetailsComments, issueType); + } + + final void assertHasNoDataIntegrityIssues(String issueType, String check, Boolean expectPercent) { + checkDataIntegritySummary(check, 0, 0, expectPercent); + Set emptyStringSet = Set.of(); + checkDataIntegrityDetailsIssues( + check, emptyStringSet, emptyStringSet, emptyStringSet, issueType); + } + + final void assertNamedMetadataObjectExists(String endpoint, String name) { + JsonResponse response = GET("/" + endpoint + "/?filter=name:eq:" + name).content(); + JsonArray dimensions = response.getArray(endpoint); + assertEquals(1, dimensions.size()); + } + + final void deleteAllMetadataObjects(String endpoint) { + GET("/" + endpoint + "/gist?fields=id&headless=true") + .content() + .stringValues() + .forEach(id -> DELETE("/" + endpoint + "/" + id)); + JsonResponse response = GET("/" + endpoint).content(); + JsonArray dimensions = response.getArray(endpoint); + assertEquals(0, dimensions.size()); + } + + final void deleteAllOrgUnits() { + deleteAllMetadataObjects("organisationUnits"); + } + + void deleteMetadataObject(String endpoint, String uid) { + + if (uid != null) { + assertStatus(HttpStatus.OK, DELETE("/" + endpoint + "/" + uid)); + assertStatus(HttpStatus.NOT_FOUND, GET("/" + endpoint + "/" + uid)); + } + } + + @Autowired private TransactionTemplate txTemplate; + + protected void doInTransaction(Runnable operation) { + + txTemplate.execute( + status -> { + operation.run(); + return null; + }); + } + + protected final HttpResponse postNewDataValue( + String period, + String value, + String comment, + boolean followup, + String dataElementId, + String orgUnitId) { + String defaultCOC = getDefaultCOC(); + return POST( + "/dataValues?de={de}&pe={pe}&ou={ou}&co={coc}&value={val}&comment={comment}&followUp={followup}", + dataElementId, + period, + orgUnitId, + defaultCOC, + value, + comment, + followup); + } + + protected final HttpResponse deleteDataValue( + String period, + String value, + String comment, + boolean followup, + String dataElementId, + String orgUnitId) { + String defaultCOC = getDefaultCOC(); + return DELETE( + "/dataValues?de={de}&pe={pe}&ou={ou}&co={coc}&value={val}&comment={comment}&followUp={followup}", + dataElementId, + period, + orgUnitId, + defaultCOC, + value, + comment, + followup); + } + + protected final String getDefaultCatCombo() { + JsonObject ccDefault = + GET("/categoryCombos/gist?fields=id,categoryOptionCombos::ids&pageSize=1&headless=true&filter=name:eq:default") + .content() + .getObject(0); + return ccDefault.getString("id").string(); + } + + protected final String getDefaultCOC() { + JsonObject ccDefault = + GET("/categoryCombos/gist?fields=id,categoryOptionCombos::ids&pageSize=1&headless=true&filter=name:eq:default") + .content() + .getObject(0); + return ccDefault.getArray("categoryOptionCombos").getString(0).string(); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoriesDuplicatedControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoriesDuplicatedControllerTest.java new file mode 100644 index 000000000000..23de54b14b05 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoriesDuplicatedControllerTest.java @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import java.util.Set; +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Tests the metadata check for categories with the same category options. + * + *

{@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_same_category_options.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityCategoriesDuplicatedControllerTest extends AbstractDataIntegrityIntegrationTest { + private final String check = "categories_same_category_options"; + + private String categoryWithOptionsA; + + private String categoryWithOptionsB; + + private String categoryOptionRed; + + @Test + void testCategoriesDuplicated() { + + categoryOptionRed = + assertStatus( + HttpStatus.CREATED, POST("/categoryOptions", "{ 'name': 'Red', 'shortName': 'Red' }")); + + categoryWithOptionsA = + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Color', 'shortName': 'Color', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'} ] }")); + + categoryWithOptionsB = + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Colour', 'shortName': 'Colour', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'} ] }")); + + assertNamedMetadataObjectExists("categories", "default"); + assertNamedMetadataObjectExists("categories", "Color"); + assertNamedMetadataObjectExists("categories", "Colour"); + /* + * This percentage may seem strange, but is based on the number of + * duplicated category options + */ + checkDataIntegritySummary(check, 1, 33, true); + + Set expectedCategories = Set.of(categoryWithOptionsA, categoryWithOptionsB); + Set expectedMessages = Set.of("(1) Colour", "(1) Color"); + checkDataIntegrityDetailsIssues( + check, expectedCategories, expectedMessages, Set.of(), "categories"); + } + + @Test + void testCategoriesNotDuplicated() { + + categoryOptionRed = + assertStatus( + HttpStatus.CREATED, POST("/categoryOptions", "{ 'name': 'Red', 'shortName': 'Red' }")); + + String categoryOptionBlue = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Blue', 'shortName': 'Blue' }")); + + categoryWithOptionsA = + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Color', 'shortName': 'Color', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'} ] }")); + + categoryWithOptionsB = + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Colour', 'shortName': 'Colour', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionBlue + + "'} ] }")); + + assertHasNoDataIntegrityIssues("categories", check, true); + } + + @Test + void testInvalidCategoriesDivideByZero() { + + // Expect a percentage here, since there should always be the default category + assertHasNoDataIntegrityIssues("categories", check, true); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryComboUnusedControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryComboUnusedControllerTest.java new file mode 100644 index 000000000000..6ffe813426f5 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryComboUnusedControllerTest.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Tests metadata integrity check for unused category combinations. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_combos_unused.yaml + * } + * + * @author Jason P. Pickering + */ +class DataIntegrityCategoryComboUnusedControllerTest extends AbstractDataIntegrityIntegrationTest { + private final String check = "category_combos_unused"; + + private final String detailsIdType = "categoryCombos"; + + @Test + void testCatCombosNotUsed() { + + setUpTest(); + + String categoryOptionSour = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Sour', 'shortName': 'Sour' }")); + + String categoryOptionRed = + assertStatus( + HttpStatus.CREATED, POST("/categoryOptions", "{ 'name': 'Red', 'shortName': 'Red' }")); + + String categoryColor = + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Color', 'shortName': 'Color', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'} ] }")); + + String categoryTaste = + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Taste', 'shortName': 'Taste', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionSour + + "'} ] }")); + + String testCatCombo = + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryCombos", + "{ 'name' : 'Taste and color', " + + "'dataDimensionType' : 'DISAGGREGATION', 'categories' : [" + + "{'id' : '" + + categoryColor + + "'} , {'id' : '" + + categoryTaste + + "'}]} ")); + + /* + * Note that we already have default here even though it is not created + * explicitly by this test + */ + assertHasDataIntegrityIssues( + detailsIdType, check, 50, testCatCombo, "Taste and color", null, true); + } + + @Test + void testCatCombosUsed() { + + setUpTest(); + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + void setUpTest() { + + String defaultCC = getDefaultCatCombo(); + + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'Widgets', 'shortName': 'Widgets', 'valueType' : 'NUMBER'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' , 'categoryCombo' : { 'id' : '" + + defaultCC + + "'}}")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryNoOptionsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryNoOptionsControllerTest.java new file mode 100644 index 000000000000..789d3c84017e --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryNoOptionsControllerTest.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Tests metadata integrity check for categories with no options. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_no_options.yaml + * } + * + * @author Jason P. Pickering + */ +class DataIntegrityCategoryNoOptionsControllerTest extends AbstractDataIntegrityIntegrationTest { + private final String check = "categories_no_options"; + + private String categoryNoOptions; + + private String categoryOptionRed; + + @Test + void testCategoriesInvalid() { + + categoryOptionRed = + assertStatus( + HttpStatus.CREATED, POST("/categoryOptions", "{ 'name': 'Red', 'shortName': 'Red' }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Color', 'shortName': 'Color', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'} ] }")); + + categoryNoOptions = + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Taste', 'shortName': 'Taste', 'dataDimensionType': 'DISAGGREGATION' }")); + + assertNamedMetadataObjectExists("categories", "default"); + assertNamedMetadataObjectExists("categoryOptions", "default"); + /* + * Note that the default category is implicit here, so the percentage + * need to take that into account + */ + assertHasDataIntegrityIssues("categories", check, 33, categoryNoOptions, "Taste", null, true); + } + + @Test + void testCategoriesAreValid() { + + String categoryOptionSour = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Sour', 'shortName': 'Sour' }")); + + categoryOptionRed = + assertStatus( + HttpStatus.CREATED, POST("/categoryOptions", "{ 'name': 'Red', 'shortName': 'Red' }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Color', 'shortName': 'Color', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'} ] }")); + + categoryNoOptions = + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Taste', 'shortName': 'Taste', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionSour + + "'} ] }")); + + assertHasNoDataIntegrityIssues("categories", check, true); + } + + @Test + void testInvalidCategoriesDivideByZero() { + + assertHasNoDataIntegrityIssues("categories", check, true); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionComboDisjointControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionComboDisjointControllerTest.java new file mode 100644 index 000000000000..05bac9a67080 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionComboDisjointControllerTest.java @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.hisp.dhis.jsontree.JsonList; +import org.hisp.dhis.jsontree.JsonResponse; +import org.hisp.dhis.web.HttpStatus; +import org.hisp.dhis.webapi.json.domain.JsonCategoryOptionCombo; +import org.junit.jupiter.api.Test; + +/** + * Tests metadata integrity check category option combinations with incorrect cardinality. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_disjoint.yaml + * } + * + * @implNote The test for disjoint category option combinations is impossible to set up in current + * versions of DHIS2, as when a category option is deleted from a category, any corresponding + * category option combinations associated with it are also deleted. If there is data associated + * with the category option combination, the DELETE operation will not succeed. Here we test for + * that scenario, which does not result in any integrity violations. + * @author Jason P. Pickering + */ +class DataIntegrityCategoryOptionComboDisjointControllerTest + extends AbstractDataIntegrityIntegrationTest { + private final String check = "category_option_combos_disjoint"; + + private final String detailsIdType = "categoryOptionCombos"; + + private String categoryOptionSweet; + + @Test + void testCanDeleteCategoryOptionCascadeCatOptionCombo() { + + setupTest(); + // We should have three cat option combos now. The two we created and default. + JsonResponse response = GET("/categoryOptionCombos?fields=id,name").content(); + JsonList catOptionCombos = + response.getList("categoryOptionCombos", JsonCategoryOptionCombo.class); + assertEquals(3, catOptionCombos.size()); + + // Delete the category option + assertStatus(HttpStatus.OK, DELETE("/categoryOptions/" + categoryOptionSweet)); + + assertStatus(HttpStatus.NOT_FOUND, GET("/categoryOptions/" + categoryOptionSweet)); + + // The deletion of the category option cascades to the category option combo. + response = GET("/categoryOptionCombos?fields=id,name").content(); + catOptionCombos = response.getList("categoryOptionCombos", JsonCategoryOptionCombo.class); + assertEquals(2, catOptionCombos.size()); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void setTestCatCombosWrongCardinalityDoesNotExist() { + + setupTest(); + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + void setupTest() { + String categoryOptionSour = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Sour', 'shortName': 'Sour' }")); + + categoryOptionSweet = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Sweet', 'shortName': 'Sweet' }")); + + String categoryOptionRed = + assertStatus( + HttpStatus.CREATED, POST("/categoryOptions", "{ 'name': 'Red', 'shortName': 'Red' }")); + + String categoryColor = + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Color', 'shortName': 'Color', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'} ] }")); + + String categoryTaste = + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Taste', 'shortName': 'Taste', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionSour + + "'}, {'id' : '" + + categoryOptionSweet + + "'} ] }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryCombos", + "{ 'name' : 'Taste and color', " + + "'dataDimensionType' : 'DISAGGREGATION', 'categories' : [" + + "{'id' : '" + + categoryColor + + "'} , {'id' : '" + + categoryTaste + + "'}]} ")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionComboNoCatComboControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionComboNoCatComboControllerTest.java new file mode 100644 index 000000000000..f073a33e5e30 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionComboNoCatComboControllerTest.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Tests metadata integrity check for category option combinations without a category combination. + * {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/coc_no_category_combo.yaml + * } + * + * @implNote The test for category combinations without a category combo is impossible to set up in + * current versions of DHIS2 through the API. + * @author Jason P. Pickering + */ +class DataIntegrityCategoryOptionComboNoCatComboControllerTest + extends AbstractDataIntegrityIntegrationTest { + private static final String check = "catoptioncombos_no_catcombo"; + + private static final String detailsIdType = "categoryOptionCombos"; + + @Test + void testCategoryOptionCombosWithCatCombosExist() { + String categoryOptionSour = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Sour', 'shortName': 'Sour' }")); + + String categoryOptionSweet = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Sweet', 'shortName': 'Sweet' }")); + + String categoryOptionRed = + assertStatus( + HttpStatus.CREATED, POST("/categoryOptions", "{ 'name': 'Red', 'shortName': 'Red' }")); + + String categoryColor = + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Color', 'shortName': 'Color', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'} ] }")); + + String categoryTaste = + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Taste', 'shortName': 'Taste', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionSour + + "'}, {'id' : '" + + categoryOptionSweet + + "'} ] }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryCombos", + "{ 'name' : 'Taste and color', " + + "'dataDimensionType' : 'DISAGGREGATION', 'categories' : [" + + "{'id' : '" + + categoryColor + + "'} , {'id' : '" + + categoryTaste + + "'}]} ")); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionComboWrongCardinalityControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionComboWrongCardinalityControllerTest.java new file mode 100644 index 000000000000..d72805afa368 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionComboWrongCardinalityControllerTest.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Tests metadata integrity check category option combinations with incorrect cardinality. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_cardinality.yaml + * } + * + * @author Jason P. Pickering + */ +class DataIntegrityCategoryOptionComboWrongCardinalityControllerTest + extends AbstractDataIntegrityIntegrationTest { + private final String check = "cocs_wrong_cardinality"; + + private final String detailsIdType = "categoryOptionCombos"; + + private String categoryColor; + + private String testCatCombo; + + @Test + void setTestCatCombosWrongCardinalityExists() { + + setupTest(); + assertStatus( + HttpStatus.OK, + PUT( + "/categoryCombos/" + testCatCombo + "?mergeMode=REPLACE", + "{ 'name' : 'Taste and color', " + + "'dataDimensionType' : 'DISAGGREGATION', 'categories' : [" + + "{'id' : '" + + categoryColor + + "'} ]} ")); + + assertHasDataIntegrityIssues( + detailsIdType, check, 50, testCatCombo, "Taste and color", null, true); + } + + @Test + void setTestCatCombosWrongCardinalityDoesNotExist() { + + setupTest(); + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + void setupTest() { + String categoryOptionSour = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Sour', 'shortName': 'Sour' }")); + + String categoryOptionRed = + assertStatus( + HttpStatus.CREATED, POST("/categoryOptions", "{ 'name': 'Red', 'shortName': 'Red' }")); + + categoryColor = + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Color', 'shortName': 'Color', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'} ] }")); + + String categoryTaste = + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Taste', 'shortName': 'Taste', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionSour + + "'} ] }")); + + testCatCombo = + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryCombos", + "{ 'name' : 'Taste and color', " + + "'dataDimensionType' : 'DISAGGREGATION', 'categories' : [" + + "{'id' : '" + + categoryColor + + "'} , {'id' : '" + + categoryTaste + + "'}]} ")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionGroupSetExcessGroupMembershipControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionGroupSetExcessGroupMembershipControllerTest.java new file mode 100644 index 000000000000..665f53e205ce --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionGroupSetExcessGroupMembershipControllerTest.java @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Test for metadata check which category options which are part of two or more category option + * groups within a category option group set. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/category_option_groups_excess_members.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityCategoryOptionGroupSetExcessGroupMembershipControllerTest + extends AbstractDataIntegrityIntegrationTest { + private final String check = "category_options_excess_groupset_membership"; + + private String categoryOptionSour; + + private String categoryOptionGroupColor; + + private String categoryOptionSweet; + + private String categoryOptionRed; + + @Test + void testCategoryOptionInMultipleGroups() { + + setUpTest(); + + String categoryOptionGroupTaste = + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryOptionGroups", + "{ 'name': 'Taste', 'shortName': 'Taste' , 'categoryOptions' : [{'id' : '" + + categoryOptionSweet + + "'}, {'id': '" + + categoryOptionSour + + "'}, {'id' : '" + + categoryOptionRed + + "'}]}")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryOptionGroupSets", + "{ 'name': 'Taste and Color', 'shortName': 'Taste and Color', 'categoryOptionGroups' : [{'id': '" + + categoryOptionGroupColor + + "'}, {'id' : '" + + categoryOptionGroupTaste + + "'}]}")); + + // Total number of category options is 5 with the default + assertHasDataIntegrityIssues("categories", check, 20, categoryOptionRed, "Red", null, true); + } + + @Test + void testCategoryOptionsInOneGroup() { + setUpTest(); + + String categoryOptionGroupTaste = + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryOptionGroups", + "{ 'name': 'Taste', 'shortName': 'Taste' , 'categoryOptions' : [{'id' : '" + + categoryOptionSweet + + "'}, {'id': '" + + categoryOptionSour + + "'}]}")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryOptionGroupSets", + "{ 'name': 'Taste and Color', 'shortName': 'Taste and Color', 'categoryOptionGroups' : [{'id': '" + + categoryOptionGroupColor + + "'}, {'id' : '" + + categoryOptionGroupTaste + + "'}]}")); + + assertHasNoDataIntegrityIssues("categories", check, true); + } + + @Test + void testCategoryOptionsInGroupRuns() { + + assertHasNoDataIntegrityIssues("categories", check, true); + } + + void setUpTest() { + categoryOptionRed = + assertStatus( + HttpStatus.CREATED, POST("/categoryOptions", "{ 'name': 'Red', 'shortName': 'Red' }")); + + String categoryOptionBlue = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Blue', 'shortName': 'Blue' }")); + + categoryOptionSweet = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Sweet', 'shortName': 'Sweet' }")); + + categoryOptionSour = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Sour', 'shortName': 'Sour' }")); + + categoryOptionGroupColor = + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryOptionGroups", + "{ 'name': 'Color', 'shortName': 'Color', 'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'}, {'id': '" + + categoryOptionBlue + + "'}]}")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionGroupSetIncompleteControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionGroupSetIncompleteControllerTest.java new file mode 100644 index 000000000000..0d4181d260ab --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionGroupSetIncompleteControllerTest.java @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Test for metadata check for category option group sets which may not contain all category options + * of related categories. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_sets_incomplete.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityCategoryOptionGroupSetIncompleteControllerTest + extends AbstractDataIntegrityIntegrationTest { + private final String check = "category_option_group_sets_incomplete"; + + private final String detailsIdType = "categoryOptionGroupSets"; + + private String categoryOptionBlue; + + private String categoryOptionRed; + + private String categoryOptionYellow; + + @Test + void testCategoryOptionGroupSetIncomplete() { + + setUpTest(); + + String warmGroup = + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryOptionGroups", + "{ 'name': 'Warm', 'shortName': 'Warm', 'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'}]}")); + + String coldGroup = + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryOptionGroups", + "{ 'name': 'Cold', 'shortName': 'Cold', 'categoryOptions' : [{'id' : '" + + categoryOptionBlue + + "'}]}")); + + String testCatOptionGroupSet = + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryOptionGroupSets", + "{ 'name': 'Color set', 'shortName': 'Color set', 'categoryOptionGroups' : [{'id': '" + + warmGroup + + "'}, {'id' : '" + + coldGroup + + "'}]}")); + + assertHasDataIntegrityIssues( + detailsIdType, check, 50, testCatOptionGroupSet, "Color set", "Yellow", true); + } + + @Test + void testCategoryOptionGroupComplete() { + setUpTest(); + + String warmGroup = + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryOptionGroups", + "{ 'name': 'Warm', 'shortName': 'Warm', 'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'}, {'id' : '" + + categoryOptionYellow + + "'}]}")); + + String coldGroup = + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryOptionGroups", + "{ 'name': 'Cold', 'shortName': 'Cold', 'categoryOptions' : [{'id' : '" + + categoryOptionBlue + + "'}]}")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryOptionGroupSets", + "{ 'name': 'Color set', 'shortName': 'Color set', 'categoryOptionGroups' : [{'id': '" + + warmGroup + + "'}, {'id' : '" + + coldGroup + + "'}]}")); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testCategoryOptionsInGroupRuns() { + + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + void setUpTest() { + categoryOptionRed = + assertStatus( + HttpStatus.CREATED, POST("/categoryOptions", "{ 'name': 'Red', 'shortName': 'Red' }")); + + categoryOptionBlue = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Blue', 'shortName': 'Blue' }")); + + categoryOptionYellow = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Yellow', 'shortName': 'Yellow' }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Colors', 'shortName': 'Colors', 'dataDimensionType': 'DISAGGREGATION', 'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'}, {'id': '" + + categoryOptionBlue + + "'}, {'id' : '" + + categoryOptionYellow + + "'}]}")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionNoCategoryControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionNoCategoryControllerTest.java new file mode 100644 index 000000000000..4928117e1e17 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionNoCategoryControllerTest.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Tests metadata integrity check for category options with no category. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_options_no_categories.yaml + * } + * + * @author Jason P. Pickering + */ +class DataIntegrityCategoryOptionNoCategoryControllerTest + extends AbstractDataIntegrityIntegrationTest { + private final String check = "category_options_no_categories"; + + private final String detailsIdType = "categoryOptions"; + + @Test + void testCategoryOptionNoCategoriesExist() { + + String categoryOptionRed = + assertStatus( + HttpStatus.CREATED, POST("/categoryOptions", "{ 'name': 'Red', 'shortName': 'Red' }")); + + assertNamedMetadataObjectExists("categories", "default"); + assertNamedMetadataObjectExists("categoryOptions", "default"); + /* + * Note that the default category is implicit here, so the percentage + * need to take that into account + */ + assertHasDataIntegrityIssues(detailsIdType, check, 50, categoryOptionRed, "Red", null, true); + } + + @Test + void testCategoryOptionsHaveCategories() { + + String categoryOptionRed = + assertStatus( + HttpStatus.CREATED, POST("/categoryOptions", "{ 'name': 'Red', 'shortName': 'Red' }")); + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Color', 'shortName': 'Color', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'} ] }")); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/DataIntegrityChecksControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityChecksControllerTest.java similarity index 66% rename from dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/DataIntegrityChecksControllerTest.java rename to dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityChecksControllerTest.java index 36a700429ca1..9e0187c82999 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/DataIntegrityChecksControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityChecksControllerTest.java @@ -25,14 +25,16 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.hisp.dhis.webapi.controller; +package org.hisp.dhis.webapi.controller.dataintegrity; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.Set; import org.hisp.dhis.dataintegrity.DataIntegrityCheckType; import org.hisp.dhis.jsontree.JsonList; +import org.hisp.dhis.webapi.controller.DataIntegrityController; import org.hisp.dhis.webapi.json.domain.JsonDataIntegrityCheck; import org.junit.jupiter.api.Test; @@ -42,7 +44,7 @@ * * @author Jan Bernitt */ -class DataIntegrityChecksControllerTest extends AbstractDataIntegrityControllerTest { +class DataIntegrityChecksControllerTest extends AbstractDataIntegrityIntegrationTest { @Test void testGetAvailableChecks() { JsonList checks = @@ -59,22 +61,71 @@ void testGetAvailableChecks() { } } + @Test + void testGetAvailableChecksNamesAreUnique() { + JsonList checks = + GET("/dataIntegrity").content().asList(JsonDataIntegrityCheck.class); + + assertEquals(checks.size(), Set.copyOf(checks.toList(JsonDataIntegrityCheck::getName)).size()); + } + + @Test + void testGetAvailableChecksCodesAreUnique() { + JsonList checks = + GET("/dataIntegrity").content().asList(JsonDataIntegrityCheck.class); + + assertEquals(checks.size(), Set.copyOf(checks.toList(JsonDataIntegrityCheck::getCode)).size()); + } + + @Test + void testGetAvailableChecks_FilterUsingCode() { + JsonList checks = + GET("/dataIntegrity?checks=CNO").content().asList(JsonDataIntegrityCheck.class); + assertEquals(1, checks.size()); + assertEquals("categories_no_options", checks.get(0).getName()); + } + @Test void testGetAvailableChecks_FilterUsingChecksPatterns() { JsonList checks = GET("/dataIntegrity?checks=program*").content().asList(JsonDataIntegrityCheck.class); - assertTrue(checks.size() > 0, "there should be matches"); - checks.forEach(check -> assertTrue(check.getSection().startsWith("Program"))); + assertFalse(checks.isEmpty(), "there should be matches"); + checks.forEach(check -> assertTrue(check.getName().toLowerCase().startsWith("program"))); } @Test void testGetAvailableChecks_FilterUsingSection() { JsonList checks = GET("/dataIntegrity?section=Program Rules").content().asList(JsonDataIntegrityCheck.class); - assertTrue(checks.size() > 0, "there should be matches"); + assertFalse(checks.isEmpty(), "there should be matches"); checks.forEach(check -> assertEquals("Program Rules", check.getSection())); } + @Test + void testGetAvailableChecks_FilterUsingSlow() { + JsonList checks = + GET("/dataIntegrity?slow=true").content().asList(JsonDataIntegrityCheck.class); + assertFalse(checks.isEmpty(), "there should be matches"); + checks.forEach(check -> assertTrue(check.getIsSlow())); + + checks = GET("/dataIntegrity?slow=false").content().asList(JsonDataIntegrityCheck.class); + assertFalse(checks.isEmpty(), "there should be matches"); + checks.forEach(check -> assertFalse(check.getIsSlow())); + } + + @Test + void testGetAvailableChecks_FilterUsingProgrammatic() { + JsonList checks = + GET("/dataIntegrity?programmatic=true").content().asList(JsonDataIntegrityCheck.class); + assertFalse(checks.isEmpty(), "there should be matches"); + checks.forEach(check -> assertTrue(check.getIsProgrammatic())); + + checks = + GET("/dataIntegrity?programmatic=false").content().asList(JsonDataIntegrityCheck.class); + assertFalse(checks.isEmpty(), "there should be matches"); + checks.forEach(check -> assertFalse(check.getIsProgrammatic())); + } + /** * The point of this test is to check if the known i18n texts provided are resolved and assigned * to the correct field diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDashboardsEmptyControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDashboardsEmptyControllerTest.java new file mode 100644 index 000000000000..1d673eaffa53 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDashboardsEmptyControllerTest.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.hisp.dhis.dashboard.Dashboard; +import org.hisp.dhis.dashboard.DashboardItem; +import org.hisp.dhis.dashboard.DashboardService; +import org.hisp.dhis.visualization.Visualization; +import org.hisp.dhis.visualization.VisualizationService; +import org.hisp.dhis.visualization.VisualizationType; +import org.hisp.dhis.webapi.json.domain.JsonDataIntegrityCheck; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Tests for dashboards which do not have any content {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/dashboards_empty.yaml + * } + * + * @author Jason P. Pickering + */ +class DataIntegrityDashboardsEmptyControllerTest extends AbstractDataIntegrityIntegrationTest { + + @Autowired private DashboardService dashboardService; + + @Autowired private VisualizationService visualizationService; + + private static final String check = "dashboards_no_items"; + + private static final String dashboard_uid = BASE_UID + "1"; + + private static final String viz_uid = BASE_UID + "2"; + + private static final String detailsIdType = "dashboards"; + + @Test + void testUnusedDashboardExist() { + + Dashboard dashboardA = new Dashboard(); + dashboardA.setName("Test Dashboard"); + dashboardA.setUid(dashboard_uid); + dashboardService.saveDashboard(dashboardA); + dbmsManager.clearSession(); + + JsonDataIntegrityCheck thisCheck = + GET("/dataIntegrity/?checks=" + check) + .content() + .asList(JsonDataIntegrityCheck.class) + .get(0); + String detailsType = thisCheck.getIssuesIdType(); + assertEquals(detailsIdType, detailsType); + + assertNamedMetadataObjectExists(detailsIdType, "Test Dashboard"); + assertHasDataIntegrityIssues( + detailsIdType, check, 100, dashboard_uid, "Test Dashboard", null, true); + } + + @Test + void testDashboardsWithItemsExist() { + + Visualization viz = new Visualization("myviz"); + viz.setUid(viz_uid); + viz.setType(VisualizationType.SINGLE_VALUE); + visualizationService.save(viz); + + DashboardItem diA = new DashboardItem(); + diA.setAutoFields(); + diA.setVisualization(viz); + + Dashboard dashboardA = new Dashboard(); + dashboardA.setName("Test Dashboard"); + dashboardA.setUid(dashboard_uid); + dashboardA.setAutoFields(); + dashboardA.getItems().add(diA); + dashboardService.saveDashboard(dashboardA); + dbmsManager.clearSession(); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testUnusedDashboardsRuns() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDashboardsNotUsedOneYearControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDashboardsNotUsedOneYearControllerTest.java new file mode 100644 index 000000000000..74947d870ee4 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDashboardsNotUsedOneYearControllerTest.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import java.time.ZonedDateTime; +import java.util.Date; +import org.hisp.dhis.dashboard.Dashboard; +import org.hisp.dhis.dashboard.DashboardService; +import org.hisp.dhis.datastatistics.DataStatisticsEvent; +import org.hisp.dhis.datastatistics.DataStatisticsEventStore; +import org.hisp.dhis.datastatistics.DataStatisticsEventType; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Tests for dashboards which have not been actively viewed in the past year.* {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/dashboards_not_used_1year.yaml + * } + * + * @author Jason P. Pickering + */ +class DataIntegrityDashboardsNotUsedOneYearControllerTest + extends AbstractDataIntegrityIntegrationTest { + + @Autowired private DataStatisticsEventStore dataStatisticsEventStore; + + @Autowired private DashboardService dashboardService; + + private DataStatisticsEvent dse1; + + private static final String check = "dashboards_not_viewed_one_year"; + + private static final String detailsIdType = "dashboards"; + + @Test + void testUnusedDashboardExist() { + + setUpDashboards(); + Date date = Date.from(ZonedDateTime.now().minusYears(1).minusDays(1).toInstant()); + + dse1 = + new DataStatisticsEvent(DataStatisticsEventType.DASHBOARD_VIEW, date, "TestUser", BASE_UID); + dataStatisticsEventStore.save(dse1); + + dbmsManager.clearSession(); + + assertNamedMetadataObjectExists(detailsIdType, "Test Dashboard"); + assertHasDataIntegrityIssues(detailsIdType, check, 100, BASE_UID, "Test Dashboard", null, true); + } + + @Test + void testUsedDashboardsExist() { + + setUpDashboards(); + long millis = System.currentTimeMillis(); + Date date = new Date(millis); + + dse1 = + new DataStatisticsEvent(DataStatisticsEventType.DASHBOARD_VIEW, date, "TestUser", BASE_UID); + dataStatisticsEventStore.save(dse1); + + dbmsManager.clearSession(); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testUnusedDashboardsRuns() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + void setUpDashboards() { + Dashboard dashboardA = new Dashboard(); + dashboardA.setName("Test Dashboard"); + dashboardA.setUid(BASE_UID); + dashboardService.saveDashboard(dashboardA); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsAbandonedControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsAbandonedControllerTest.java new file mode 100644 index 000000000000..e8995c91da03 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsAbandonedControllerTest.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.hisp.dhis.web.WebClient; +import org.junit.jupiter.api.Test; + +/** + * Test for data elements which have been abandoned. This is taken to mean that there is no data + * recorded against them, and they have not been updated in the last hundred days. + * + *

It is not possible to manually set the lastUpdate field for data elements, so it is not + * possible to create a proper unit test for the scenario of identifying abandoned data elements. + * + *

{@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_abandoned.yaml + * } + * + * @author Jason P. Pickering + */ +class DataIntegrityDataElementsAbandonedControllerTest + extends AbstractDataIntegrityIntegrationTest { + private static final String check = "data_elements_aggregate_abandoned"; + + private static final String detailsIdType = "dataElements"; + + private static final String period = "202212"; + + @Test + void testDataElementsNotAbandoned() { + + setUpTest(); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testDataElementsAbandonedDividedByZero() { + + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + void setUpTest() { + + String dataElementA = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC1', 'shortName': 'ANC1', 'valueType' : 'NUMBER'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' }")); + String dataElementB = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC2', 'shortName': 'ANC2', 'valueType' : 'NUMBER'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' }")); + + String orgUnitId = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits/", + "{'name':'My Unit', 'shortName':'OU1', 'openingDate': '2020-01-01'}")); + // add OU to users hierarchy + assertStatus( + HttpStatus.OK, + POST( + "/users/{id}/organisationUnits", + getCurrentUser().getUid(), + WebClient.Body("{'additions':[{'id':'" + orgUnitId + "'}]}"))); + // Add some data to dataElementB + assertStatus( + HttpStatus.CREATED, + postNewDataValue(period, "10", "Test Data", false, dataElementB, orgUnitId)); + /* Both data elements should have data now */ + assertStatus( + HttpStatus.CREATED, + postNewDataValue(period, "20", "Test Data", false, dataElementA, orgUnitId)); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsAggregateExcessGroupMembershipControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsAggregateExcessGroupMembershipControllerTest.java new file mode 100644 index 000000000000..89b6cbe1e5bf --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsAggregateExcessGroupMembershipControllerTest.java @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Test for metadata check which identifies data element group sets which have duplicative data + * elements within one or more of their constituent groups. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_excess_groupset_membership.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityDataElementsAggregateExcessGroupMembershipControllerTest + extends AbstractDataIntegrityIntegrationTest { + private final String check = "data_elements_excess_groupset_membership"; + + private String dataElementA; + + private String dataElementB; + + private String dataElementGroupB; + + private String dataElementGroupA; + + @Test + void testDataElementNotInGroup() { + + setUpDataElements(); + + dataElementGroupA = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElementGroups", + "{ 'name' : 'MCH', 'shortName': 'MCH' , " + + "'dataElements':[{'id':'" + + dataElementA + + "'},{'id': '" + + dataElementB + + "'}]}")); + + dataElementGroupB = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElementGroups", + "{ 'name': 'ANC', 'shortName': 'ANC' , 'dataElements' : [{'id' : '" + + dataElementB + + "'}]}")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElementGroupSets", + "{'name' : 'Maternal Health' , 'shortName': 'Maternal Health', 'dataDimension' : true, " + + "'dataElementGroups': [{'id': '" + + dataElementGroupA + + "'},{'id': '" + + dataElementGroupB + + "'}]}")); + + assertHasDataIntegrityIssues( + "data_elements_aggregate", check, 50, dataElementB, "ANC2", null, true); + } + + @Test + void testDataElementsInGroup() { + setUpDataElements(); + + dataElementGroupA = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElementGroups", + "{ 'name' : 'MCH', 'shortName': 'MCH' , " + + "'dataElements':[{'id':'" + + dataElementA + + "'}]}")); + + dataElementGroupB = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElementGroups", + "{ 'name': 'ANC', 'shortName': 'ANC' , 'dataElements' : [{'id' : '" + + dataElementB + + "'}]}")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElementGroupSets", + "{'name' : 'Maternal Health' , 'shortName': 'Maternal Health', 'dataDimension' : true, " + + "'dataElementGroups': [{'id': '" + + dataElementGroupA + + "'},{'id': '" + + dataElementGroupB + + "'}]}")); + + assertHasNoDataIntegrityIssues("data_elements_aggregate", check, true); + } + + @Test + void testDataElementsInGroupDivideByZero() { + + assertHasNoDataIntegrityIssues("data_elements_aggregate", check, false); + } + + void setUpDataElements() { + dataElementA = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC1', 'shortName': 'ANC1', 'valueType' : 'NUMBER'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' }")); + + dataElementB = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC2', 'shortName': 'ANC2', 'valueType' : 'NUMBER'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' }")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsAggregateNotInDataElementGroupControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsAggregateNotInDataElementGroupControllerTest.java new file mode 100644 index 000000000000..b58b9b4d0bd3 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsAggregateNotInDataElementGroupControllerTest.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.hisp.dhis.jsontree.JsonList; +import org.hisp.dhis.web.HttpStatus; +import org.hisp.dhis.webapi.json.domain.JsonDataElement; +import org.hisp.dhis.webapi.json.domain.JsonDataElementGroup; +import org.junit.jupiter.api.Test; + +/** + * Test for data elements which are not part of any data element group. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_no_groups.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityDataElementsAggregateNotInDataElementGroupControllerTest + extends AbstractDataIntegrityIntegrationTest { + private static final String check = "data_elements_aggregate_no_groups"; + + private static final String detailsIdType = "dataElements"; + + private String dataElementA; + + private String dataElementB; + + private String dataElementGroupA; + + @Test + void testDataElementNotInGroup() { + + setUpDataElements(); + + dataElementGroupA = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElementGroups", + "{ 'name': 'ANC', 'shortName': 'ANC' , 'dataElements' : [{'id' : '" + + dataElementB + + "'}]}")); + + JsonDataElementGroup deg = + GET("/dataElementGroups/" + dataElementGroupA).content().as(JsonDataElementGroup.class); + JsonList des = deg.getDataElements(); + assertEquals(1, des.size()); + + assertHasDataIntegrityIssues(detailsIdType, check, 50, dataElementA, "ANC1", null, true); + } + + @Test + void testDataElementsInGroup() { + setUpDataElements(); + + dataElementGroupA = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElementGroups", + "{ 'name' : 'ANC', 'shortName': 'ANC' , " + + "'dataElements':[{'id':'" + + dataElementA + + "'},{'id': '" + + dataElementB + + "'}]}")); + + JsonDataElementGroup deg = + GET("/dataElementGroups/" + dataElementGroupA).content().as(JsonDataElementGroup.class); + JsonList des = deg.getDataElements(); + assertEquals(2, des.size()); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testDataElementsInGroupDivideByZero() { + + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + void setUpDataElements() { + dataElementA = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC1', 'shortName': 'ANC1', 'valueType' : 'NUMBER'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' }")); + + dataElementB = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC2', 'shortName': 'ANC2', 'valueType' : 'NUMBER'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' }")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsAggregationOperatorControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsAggregationOperatorControllerTest.java new file mode 100644 index 000000000000..9317b088d1f2 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsAggregationOperatorControllerTest.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Generally, non-numeric data elements should have their aggregation type set to NONE, while + * numeric data elements should have their aggregation type set to something other than NONE. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_inconsistent_agg_operator.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityDataElementsAggregationOperatorControllerTest + extends AbstractDataIntegrityIntegrationTest { + private final String check = "data_elements_aggregate_aggregation_operator"; + + private final String detailsIdType = "dataElements"; + + private String dataElementB; + + @Test + void testDataElementInconsistentAggregation() { + setUpDataElements(); + + dataElementB = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC3', 'shortName': 'ANC3', 'valueType' : 'TEXT'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' }")); + + assertHasDataIntegrityIssues(detailsIdType, check, 33, dataElementB, "ANC3", null, true); + } + + @Test + void testDataElementsConsistentAggregation() { + + setUpDataElements(); + + dataElementB = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC3', 'shortName': 'ANC3', 'valueType' : 'TEXT'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'NONE' }")); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testDataElementsAggregationDividedByZero() { + + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + void setUpDataElements() { + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC1', 'shortName': 'ANC1', 'valueType' : 'NUMBER'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' }")); + + dataElementB = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC2', 'shortName': 'ANC2', 'valueType' : 'NUMBER'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' }")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsDifferentPeriodTypesControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsDifferentPeriodTypesControllerTest.java new file mode 100644 index 000000000000..861c36eb2987 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsDifferentPeriodTypesControllerTest.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.common.CodeGenerator.generateUid; +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Test for data elements which belong to datasets of different period types. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_datasets_different_period_types.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityDataElementsDifferentPeriodTypesControllerTest + extends AbstractDataIntegrityIntegrationTest { + private final String check = "data_elements_aggregate_with_different_period_types"; + + private final String detailsIdType = "dataElements"; + + private String dataElementA; + + private String defaultCatCombo; + + @Test + void testDataElementsHaveDifferentPeriodTypes() { + + setUpTest(); + + String datasetUID = generateUid(); + String datasetMetadata = + "{ 'id':'" + + datasetUID + + "', 'name': 'Test Weekly', 'shortName': 'Test Weekly', 'periodType' : 'Weekly'," + + "'categoryCombo' : {'id': '" + + defaultCatCombo + + "'}, " + + "'dataSetElements' : [{'dataSet' : {'id':'" + + datasetUID + + "'}, 'id':'" + + generateUid() + + "', 'dataElement': {'id' : '" + + dataElementA + + "'}}]}"; + assertStatus(HttpStatus.CREATED, POST("/dataSets", datasetMetadata)); + + assertHasDataIntegrityIssues( + detailsIdType, check, 100, dataElementA, "ANC1", "Test Weekly", true); + } + + @Test + void testDataElementHasSamePeriodType() { + + setUpTest(); + String datasetUID = generateUid(); + String datasetMetadata = + "{ 'id':'" + + datasetUID + + "', 'name': 'Test Monthly 2', 'shortName': 'Test Monthly 2', 'periodType' : 'Monthly'," + + "'categoryCombo' : {'id': '" + + defaultCatCombo + + "'}, " + + "'dataSetElements' : [{'dataSet' : {'id':'" + + datasetUID + + "'}, 'id':'" + + generateUid() + + "', 'dataElement': {'id' : '" + + dataElementA + + "'}}]}"; + assertStatus(HttpStatus.CREATED, POST("/dataSets", datasetMetadata)); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testDataElementPeriodTypeCheckRuns() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + void setUpTest() { + + defaultCatCombo = getDefaultCatCombo(); + + dataElementA = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC1', 'shortName': 'ANC1', 'valueType' : 'NUMBER'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' }")); + + String datasetUID = generateUid(); + String datasetMetadata = + "{ 'id':'" + + datasetUID + + "', 'name': 'Test Monthly', 'shortName': 'Test Monthly', 'periodType' : 'Monthly'," + + "'categoryCombo' : {'id': '" + + defaultCatCombo + + "'}, " + + "'dataSetElements' : [{'dataSet' : {'id':'" + + datasetUID + + "'}, 'id':'" + + generateUid() + + "', 'dataElement': {'id' : '" + + dataElementA + + "'}}]}"; + + assertStatus(HttpStatus.CREATED, POST("/dataSets", datasetMetadata)); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsNoAnalysisControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsNoAnalysisControllerTest.java new file mode 100644 index 000000000000..57c1e78dc03f --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsNoAnalysisControllerTest.java @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Test for aggregate data elements which do not have any analysis. The condition we test for here + * is whether the data element is part of an indicator, which is also used in a favorite. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_no_analysis.yaml + * } + * + * @author Jason P. Pickering + */ +class DataIntegrityDataElementsNoAnalysisControllerTest + extends AbstractDataIntegrityIntegrationTest { + private static final String check = "data_elements_aggregate_no_analysis"; + + private static final String detailsIdType = "dataElements"; + + @Test + void testDataElementNoAnalysis() { + + String dataElementA = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC1', 'shortName': 'ANC1', 'valueType' : 'NUMBER'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' }")); + + assertHasDataIntegrityIssues(detailsIdType, check, 100, dataElementA, "ANC1", null, true); + } + + @Test + void testDataElementHasAnalysis() { + + String dataElementA = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC1', 'shortName': 'ANC1', 'valueType' : 'NUMBER'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' }")); + + String indicatorTypeA = + assertStatus( + HttpStatus.CREATED, + POST("/indicatorTypes", "{ 'name': 'Per cent', 'factor' : 100, 'number' : false }")); + + String indicatorA = + assertStatus( + HttpStatus.CREATED, + POST( + "/indicators", + "{ 'name': 'Indicator A', 'shortName': 'Indicator A', 'indicatorType' : {'id' : '" + + indicatorTypeA + + "'}," + + " 'numerator' : ' " + + dataElementA + + "', 'numeratorDescription' : 'ANC1', 'denominator' : '1', " + + "'denominatorDescription' : '1'} }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/visualizations?skipTranslations=true&skipSharing=true", + "{'type':'SINGLE_VALUE','columns':[{'dimension':'dx','items':[{'id':'" + + indicatorA + + "'}]}],'rows':[],'filters':[{'dimension':'ou','items':[{'id':'USER_ORGUNIT'}]}, " + + "{'dimension':'pe','items':[{'id':'LAST_12_MONTHS'}]}],'axes':[],'colorSet':'DEFAULT', " + + "'cumulativeValues':false,'hideEmptyRowItems':'NONE','seriesKey':{},'legend':{}," + + "'noSpaceBetweenColumns':false,'percentStackedValues':false,'regressionType':'NONE', " + + "'showData':true,'aggregationType':'DEFAULT','completedOnly':false,'hideSubtitle':false," + + "'hideTitle':false,'sortOrder':0,'series':[],'fontStyle':{},'outlierAnalysis':null,'colTotals':false," + + "'colSubTotals':false,'rowTotals':false,'rowSubTotals':false,'showDimensionLabels':false," + + "'hideEmptyColumns':false,'hideEmptyRows':false,'skipRounding':false,'numberType':'VALUE', " + + "'showHierarchy':false,'displayDensity':'NORMAL','fontSize':'NORMAL','digitGroupSeparator':'SPACE'," + + "'fixColumnHeaders':false,'fixRowHeaders':false,'regression':false,'cumulative':false,'topLimit':0,'" + + "" + + "reportingParams':{'organisationUnit':false,'reportingPeriod':false,'parentOrganisationUnit':false," + + "'grandParentOrganisationUnit':false},'name':'Test viz'}")); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testDataElementHasAnalysisRuns() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsNoDataControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsNoDataControllerTest.java new file mode 100644 index 000000000000..2ab63aa23375 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDataElementsNoDataControllerTest.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.hisp.dhis.web.WebClient; +import org.junit.jupiter.api.Test; + +/** + * Test for data elements which have no data values associated with them {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_nodata.yaml} + * + * @implNote Note that we clear the database session prior to the actual test. This seems to be + * required for the data to actually be flushed to the datavalue table. + * @author Jason P. Pickering + */ +class DataIntegrityDataElementsNoDataControllerTest extends AbstractDataIntegrityIntegrationTest { + private static final String check = "data_elements_aggregate_no_data"; + + private static final String detailsIdType = "dataElements"; + + private String dataElementA; + + private String dataElementB; + + private static final String period = "202212"; + + private String orgUnitId; + + @Test + void testDataElementsHaveData() { + + setUpTest(); + // Add some data to dataElementB + assertStatus( + HttpStatus.CREATED, + postNewDataValue(period, "10", "Test Data", false, dataElementB, orgUnitId)); + // Add some data to dataElementB + assertStatus( + HttpStatus.CREATED, + postNewDataValue(period, "10", "Test Data", false, dataElementA, orgUnitId)); + + dbmsManager.clearSession(); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testDataElementsDoNotHaveData() { + + setUpTest(); + + // Add some data to dataElementB + assertStatus( + HttpStatus.CREATED, + postNewDataValue(period, "10", "Test Data", false, dataElementB, orgUnitId)); + dbmsManager.clearSession(); + // One of the data elements should not have data + assertHasDataIntegrityIssues(detailsIdType, check, 50, dataElementA, "ANC1", null, true); + } + + @Test + void testDataElementsNoDataRuns() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + void setUpTest() { + + dataElementA = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC1', 'shortName': 'ANC1', 'valueType' : 'NUMBER'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' }")); + + dataElementB = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC2', 'shortName': 'ANC2', 'valueType' : 'NUMBER'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' }")); + + orgUnitId = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits/", + "{'name':'My Unit', 'shortName':'OU1', 'openingDate': '2020-01-01'}")); + + // add OU to users hierarchy + assertStatus( + HttpStatus.OK, + POST( + "/users/{id}/organisationUnits", + getCurrentUser().getUid(), + WebClient.Body("{'additions':[{'id':'" + orgUnitId + "'}]}"))); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDatasetsEmptyControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDatasetsEmptyControllerTest.java new file mode 100644 index 000000000000..ff22c77ae07c --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDatasetsEmptyControllerTest.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Tests for aggregate datasets with no data elements. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/dashboards_empty.yaml + * } + * + * @author Jason P. Pickering + */ +class DataIntegrityDatasetsEmptyControllerTest extends AbstractDataIntegrityIntegrationTest { + + private static final String check = "datasets_empty"; + + private static final String dataSetUID = "CowXAwmulDG"; + + @Test + void testEmptyDataSetExists() { + + String defaultCatCombo = getDefaultCatCombo(); + String datasetA = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataSets", + "{ 'name': 'Test', 'shortName': 'Test', 'periodType' : 'Monthly', 'categoryCombo' : {'id': '" + + defaultCatCombo + + "'}}")); + + assertHasDataIntegrityIssues("dataSets", check, 100, datasetA, "Test", null, true); + } + + @Test + void testDataSetHasDataElements() { + + String defaultCatCombo = getDefaultCatCombo(); + String dataElementA = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC1', 'shortName': 'ANC1', 'valueType' : 'NUMBER'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/dataSets", + "{ 'name': 'Test', 'shortName': 'Test', 'periodType' : 'Monthly', 'categoryCombo' : {'id': '" + + defaultCatCombo + + "'}, " + + " 'dataSetElements': [{ 'dataSet': { 'id': '" + + dataSetUID + + "'}, 'dataElement': { 'id': '" + + dataElementA + + "'}}]}")); + + assertHasNoDataIntegrityIssues("dataSets", check, true); + } + + @Test + void testEmptyDataSetsRuns() { + assertHasNoDataIntegrityIssues("dataSets", check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDefaultChecksTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDefaultChecksTest.java new file mode 100644 index 000000000000..9b9c956265ce --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityDefaultChecksTest.java @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.hisp.dhis.jsontree.JsonList; +import org.hisp.dhis.web.HttpStatus; +import org.hisp.dhis.webapi.json.domain.JsonDataIntegrityCheck; +import org.hisp.dhis.webapi.json.domain.JsonDataIntegritySummary; +import org.junit.jupiter.api.Test; + +/** + * Be default, metadata checks which are marked as "slow" should be excluded from a default run of + * all checks. These "slow" checks may require significant computational resources. Users should be + * able to trigger these checks individually though as needed. + * + * @author Jason P. Pickering + */ +class DataIntegrityDefaultChecksTest extends AbstractDataIntegrityIntegrationTest { + + @Test + void testNonSlowChecksNotRunByDefault() { + final String check = "data_elements_aggregate_abandoned"; + JsonList checks = + GET("/dataIntegrity?checks=" + check).content().asList(JsonDataIntegrityCheck.class); + assertEquals(1, checks.size()); + JsonDataIntegrityCheck slowCheck = checks.get(0); + assertTrue(slowCheck.getIsSlow()); + + // Be sure we start with a clean slate + assertStatus(HttpStatus.NO_CONTENT, POST("/maintenance?cacheClear=true")); + + // Trigger the default checks + assertStatus(HttpStatus.OK, POST("/dataIntegrity/summary")); + + // The slow check should not exist + JsonDataIntegritySummary summary = + GET("/dataIntegrity/" + check + "/summary").content().as(JsonDataIntegritySummary.class); + assertTrue(summary.exists()); + assertFalse(summary.has("count")); + assertFalse(summary.has("percentage")); + assertFalse(summary.has("finishedTime")); + + summary = + GET("/dataIntegrity/categories-no-options/summary") + .content() + .as(JsonDataIntegritySummary.class); + assertTrue(summary.exists()); + assertTrue(summary.isObject()); + assertEquals(0, summary.getCount()); + assertFalse(summary.getIsSlow()); + assertNotNull(summary.getFinishedTime()); + assertEquals(0, summary.getPercentage()); + + // Trigger the slow check + assertStatus(HttpStatus.OK, POST("/dataIntegrity/summary?checks=" + check)); + + summary = + GET("/dataIntegrity/" + check + "/summary").content().as(JsonDataIntegritySummary.class); + assertTrue(summary.exists()); + assertTrue(summary.isObject()); + assertEquals(0, summary.getCount()); + assertTrue(summary.getIsSlow()); + assertNotNull(summary.getFinishedTime()); + assertNull(summary.getPercentage()); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeCategoryOptionGroupSetControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeCategoryOptionGroupSetControllerTest.java new file mode 100644 index 000000000000..ad5830d2c52b --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeCategoryOptionGroupSetControllerTest.java @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import java.util.Set; +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Tests the metadata check for category option group sets which have fewer than two members. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_category_option_group_sets.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityGroupSizeCategoryOptionGroupSetControllerTest + extends AbstractDataIntegrityIntegrationTest { + private static final String check = "category_option_group_sets_scarce"; + + private static final String detailsIdType = "categoryOptionGroupSets"; + + private String categoryOptionGroupTaste; + + @Test + void testCategoryOptionGroupSetSizeTooLow() { + + setUpTest(); + + String categoryOptionGroupSetOne = + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryOptionGroupSets", + "{'name': 'One', 'shortName' : 'One', 'categoryOptionGroups' : [{'id' : '" + + categoryOptionGroupTaste + + "'}]}")); + + String categoryOptionGroupSetNil = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptionGroupSets", "{ 'name': 'Nil', 'shortName': 'Nil' }")); + + assertHasDataIntegrityIssues( + detailsIdType, + check, + 66, + Set.of(categoryOptionGroupSetOne, categoryOptionGroupSetNil), + Set.of("One", "Nil"), + Set.of("0", "1"), + true); + } + + @Test + void testCategoryOptionGroupSetSizeOK() { + setUpTest(); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testCategoryOptionGroupSetSizeRuns() { + + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + void setUpTest() { + + String categoryOptionBlue = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Blue', 'shortName': 'Blue' }")); + + String categoryOptionRed = + assertStatus( + HttpStatus.CREATED, POST("/categoryOptions", "{ 'name': 'Red', 'shortName': 'Red' }")); + + String categoryOptionGroupColors = + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryOptionGroups", + "{ 'name': 'Color', 'shortName': 'Color', 'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'}, {'id': '" + + categoryOptionBlue + + "'}]}")); + + String categoryOptionSweet = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Sweet', 'shortName': 'Sweet' }")); + + categoryOptionGroupTaste = + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryOptionGroups", + "{ 'name': 'Taste', 'shortName': 'Taste', 'categoryOptions' : [{'id' : '" + + categoryOptionSweet + + "'}]}")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryOptionGroupSets", + "{'name': 'Two', 'shortName' : 'Two', 'categoryOptionGroups' : [{'id' : '" + + categoryOptionGroupTaste + + "'},{'id' : '" + + categoryOptionGroupColors + + "'}]}")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeCategoryOptionGroupsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeCategoryOptionGroupsControllerTest.java new file mode 100644 index 000000000000..9d66cbd15a69 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeCategoryOptionGroupsControllerTest.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import java.util.Set; +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Tests the metadata check for category option groups which have fewer than two members. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_category_option_groups.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityGroupSizeCategoryOptionGroupsControllerTest + extends AbstractDataIntegrityIntegrationTest { + private final String check = "category_option_groups_scarce"; + + private static final String detailsIdType = "categoryOptionGroups"; + + @Test + void testCategoryOptionGroupSizeTooSmall() { + + setUpTest(); + + String categoryOptionSweet = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Sweet', 'shortName': 'Sweet' }")); + + String categoryOptionGroupColors = + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryOptionGroups", + "{ 'name': 'Taste', 'shortName': 'Taste', 'categoryOptions' : [{'id' : '" + + categoryOptionSweet + + "'}]}")); + String categoryOptionGroupNil = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptionGroups", "{ 'name': 'Nil', 'shortName': 'Nil' }")); + + assertHasDataIntegrityIssues( + detailsIdType, + check, + 66, + Set.of(categoryOptionGroupColors, categoryOptionGroupNil), + Set.of("Taste", "Nil"), + Set.of("0", "1"), + true); + } + + @Test + void testCategoryOptionGroupSizeOK() { + setUpTest(); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testCategoryOptionGroupSizeRuns() { + + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + void setUpTest() { + + String categoryOptionBlue = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Blue', 'shortName': 'Blue' }")); + + String categoryOptionRed = + assertStatus( + HttpStatus.CREATED, POST("/categoryOptions", "{ 'name': 'Red', 'shortName': 'Red' }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryOptionGroups", + "{ 'name': 'Color', 'shortName': 'Color', 'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'}, {'id': '" + + categoryOptionBlue + + "'}]}")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeDataElementGroupsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeDataElementGroupsControllerTest.java new file mode 100644 index 000000000000..40dfcb88bbaa --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeDataElementGroupsControllerTest.java @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import java.util.Set; +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Test for metadata integrity check for data element groups which have less than two members. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_data_element_groups.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityGroupSizeDataElementGroupsControllerTest + extends AbstractDataIntegrityIntegrationTest { + private static final String check = "data_element_groups_scarce"; + + private static final String detailsIdType = "dataElementGroups"; + + private String dataElementA; + + private String dataElementB; + + @Test + void testDataElementGroupSizeTooLow() { + + setUpDataElements(); + + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElementGroups", + "{ 'name' : 'MCH', 'shortName': 'MCH' , " + + "'dataElements':[{'id':'" + + dataElementA + + "'},{'id': '" + + dataElementB + + "'}]}")); + + String dataElementGroupB = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElementGroups", + "{ 'name': 'ANC', 'shortName': 'ANC' , 'dataElements' : [{'id' : '" + + dataElementB + + "'}]}")); + + String dataElementGroupC = + assertStatus( + HttpStatus.CREATED, + POST("/dataElementGroups", "{ 'name': 'Morbidity', 'shortName': 'Morbidity' }")); + + assertHasDataIntegrityIssues( + detailsIdType, + check, + 66, + Set.of(dataElementGroupB, dataElementGroupC), + Set.of("ANC", "Morbidity"), + Set.of("0", "1"), + true); + } + + @Test + void testDataElementGroupSizeOK() { + setUpDataElements(); + + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElementGroups", + "{ 'name' : 'MCH', 'shortName': 'MCH' , " + + "'dataElements':[{'id':'" + + dataElementA + + "'},{'id': '" + + dataElementB + + "'}]}")); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testDataElementGroupSizeRuns() { + + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + void setUpDataElements() { + dataElementA = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC1', 'shortName': 'ANC1', 'valueType' : 'NUMBER'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' }")); + + dataElementB = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC2', 'shortName': 'ANC2', 'valueType' : 'NUMBER'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' }")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeIndicatorGroupSetsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeIndicatorGroupSetsControllerTest.java new file mode 100644 index 000000000000..cb169c5c407b --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeIndicatorGroupSetsControllerTest.java @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import java.util.Set; +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Test metadata check for indicator group sets which are composed of less than two indicator + * groups. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/group_size_indicator_group_sets.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityGroupSizeIndicatorGroupSetsControllerTest + extends AbstractDataIntegrityIntegrationTest { + + private static final String check = "indicator_group_sets_scarce"; + + private static final String detailsIdType = "indicatorGroupSets"; + + private String indicatorGroupA; + + @Test + void testIndicatorGroupSetSizeTooSmall() { + + setUpTest(); + String indicatorGroupSetA = + assertStatus( + HttpStatus.CREATED, + POST( + "/indicatorGroupSets", + "{ 'name' : 'IGS1', 'shortName' : 'IGS1', 'indicatorGroups' : [{'id' : '" + + indicatorGroupA + + "'}]}")); + + String indicatorGroupSetB = + assertStatus( + HttpStatus.CREATED, + POST("/indicatorGroupSets", "{ 'name' : 'IGS2', 'shortName' : 'IGS2' }")); + + assertHasDataIntegrityIssues( + detailsIdType, + check, + 66, + Set.of(indicatorGroupSetA, indicatorGroupSetB), + Set.of("IGS1", "IGS2"), + Set.of("0", "1"), + true); + } + + @Test + void testIndicatorGroupSetSizeOK() { + + setUpTest(); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testIndicatorsInGroupsRuns() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + void setUpTest() { + + String indicatorTypeA = + assertStatus( + HttpStatus.CREATED, + POST("/indicatorTypes", "{ 'name': 'Per cent', 'factor' : 100, 'number' : false }")); + + String indicatorA = + assertStatus( + HttpStatus.CREATED, + POST( + "/indicators", + "{ 'name': 'Indicator A', 'shortName': 'Indicator A', 'indicatorType' : {'id' : '" + + indicatorTypeA + + "'}," + + " 'numerator' : 'abc123', 'numeratorDescription' : 'One', 'denominator' : 'abc123', " + + "'denominatorDescription' : 'Zero'} }")); + + String indicatorB = + assertStatus( + HttpStatus.CREATED, + POST( + "/indicators", + "{ 'name': 'Indicator B', 'shortName': 'Indicator B', 'indicatorType' : {'id' : '" + + indicatorTypeA + + "'}," + + " 'numerator' : 'abc123', 'numeratorDescription' : 'One', 'denominator' : 'abc123', " + + "'denominatorDescription' : 'Zero'}")); + + indicatorGroupA = + assertStatus( + HttpStatus.CREATED, + POST( + "/indicatorGroups", + "{ 'name' : 'An indicator group', 'shortName' : 'An indicator group', 'indicators' : [{'id' : '" + + indicatorA + + "'}]}")); + + String indicatorGroupB = + assertStatus( + HttpStatus.CREATED, + POST( + "/indicatorGroups", + "{ 'name' : 'Another indicator group', 'shortName' : 'Another indicator group', 'indicators' : [{'id' : '" + + indicatorB + + "'}]}")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/indicatorGroupSets", + "{ 'name' : 'An indicator group set', 'shortName' : 'An indicator group set', 'indicatorGroups' : [{'id' : '" + + indicatorGroupA + + "'}, " + + "{'id' : '" + + indicatorGroupB + + "'}]}")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeIndicatorGroupsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeIndicatorGroupsControllerTest.java new file mode 100644 index 000000000000..58ce19e1cb05 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeIndicatorGroupsControllerTest.java @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import java.util.Set; +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Test for metadata check for indicator groups which have less than two members. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_indicator_groups.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityGroupSizeIndicatorGroupsControllerTest + extends AbstractDataIntegrityIntegrationTest { + private static final String check = "indicator_groups_scarce"; + + private static final String detailsIdType = "indicatorGroups"; + + private String indicatorA; + + @Test + void testIndicatorGroupSizeTooLow() { + + setUpTest(); + + String indicatorGroupA = + assertStatus( + HttpStatus.CREATED, + POST( + "/indicatorGroups", + "{ 'name' : 'ANC', 'indicators' : [{'id' : '" + indicatorA + "'}]}")); + + String indicatorGroupB = + assertStatus(HttpStatus.CREATED, POST("/indicatorGroups", "{ 'name' : 'HIV/AIDS' }")); + + assertHasDataIntegrityIssues( + detailsIdType, + check, + 66, + Set.of(indicatorGroupA, indicatorGroupB), + Set.of("ANC", "HIV/AIDS"), + Set.of("0", "1"), + true); + } + + @Test + void testIndicatorGroupSizeOK() { + setUpTest(); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testDataElementGroupSizeRuns() { + + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + void setUpTest() { + + String indicatorTypeA = + assertStatus( + HttpStatus.CREATED, + POST("/indicatorTypes", "{ 'name': 'Per cent', 'factor' : 100, 'number' : false }")); + + indicatorA = + assertStatus( + HttpStatus.CREATED, + POST( + "/indicators", + "{ 'name': 'Indicator A', 'shortName': 'Indicator A', 'indicatorType' : {'id' : '" + + indicatorTypeA + + "'}," + + " 'numerator' : 'abc123', 'numeratorDescription' : 'One', 'denominator' : 'abc123', " + + "'denominatorDescription' : 'Zero'} }")); + + String indicatorB = + assertStatus( + HttpStatus.CREATED, + POST( + "/indicators", + "{ 'name': 'Indicator B', 'shortName': 'Indicator B', 'indicatorType' : {'id' : '" + + indicatorTypeA + + "'}," + + " 'numerator' : 'abc123', 'numeratorDescription' : 'One', 'denominator' : 'abc123', " + + "'denominatorDescription' : 'Zero'}")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/indicatorGroups", + "{ 'name' : 'MCH', 'shortName': 'MCH' , " + + "'indicators':[{'id':'" + + indicatorA + + "'},{'id': '" + + indicatorB + + "'}]}")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeOrganisationUnitGroupsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeOrganisationUnitGroupsControllerTest.java new file mode 100644 index 000000000000..1787ef0c4993 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeOrganisationUnitGroupsControllerTest.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import java.util.Set; +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Test for metadata check for orgunit groups with fewer than two members. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_organisation_unit_groups.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityGroupSizeOrganisationUnitGroupsControllerTest + extends AbstractDataIntegrityIntegrationTest { + private static final String check = "orgunit_groups_scarce"; + + private static final String detailsIdType = "organisationUnitGroups"; + + private String orgunitB; + + @Test + void testOrgunitGroupSizeTooLow() { + + setUpTest(); + + String testOrgUnitGroupB = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnitGroups", + "{'name': 'Type B', 'shortName': 'Type B', 'organisationUnits' : [{'id' : '" + + orgunitB + + "'}]}")); + + String testOrgUnitGroupC = + assertStatus( + HttpStatus.CREATED, + POST("/organisationUnitGroups", "{'name': 'Type C', 'shortName': 'Type C' }")); + + assertHasDataIntegrityIssues( + detailsIdType, + check, + 66, + Set.of(testOrgUnitGroupB, testOrgUnitGroupC), + Set.of("Type B", "Type C"), + Set.of("0", "1"), + true); + } + + @Test + void testOrgunitGroupSizeOK() { + + setUpTest(); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testOrgunitGroupSizeRuns() { + + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + void setUpTest() { + String orgunitA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Fish District', 'shortName': 'Fish District', 'openingDate' : '2022-01-01'}")); + + orgunitB = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Pizza District', 'shortName': 'Pizza District', 'openingDate' : '2022-01-01'}")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnitGroups", + "{'name': 'Type A', 'shortName': 'Type A', 'organisationUnits' : [{'id' : '" + + orgunitA + + "'}, {'id' : '" + + orgunitB + + "'}]}")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeProgramIndicatorGroupControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeProgramIndicatorGroupControllerTest.java new file mode 100644 index 000000000000..589828ba93d7 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeProgramIndicatorGroupControllerTest.java @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import java.util.Set; +import org.hisp.dhis.program.*; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Minimal test for program indicator groups which contain less than two members. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_program_indicator_groups.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityGroupSizeProgramIndicatorGroupControllerTest + extends AbstractDataIntegrityIntegrationTest { + private static final String check = "program_indicator_groups_scarce"; + + private static final String detailsIdType = "programIndicatorGroups"; + + @Autowired private ProgramIndicatorService programIndicatorService; + + @Autowired private ProgramService programService; + + private ProgramIndicator testPIb; + + @Test + void testProgramIndicatorGroupsTooSmall() { + + setUpTest(); + + // Add a group with one indicator + ProgramIndicatorGroup programIndicatorGroupB = new ProgramIndicatorGroup("Test PI Group B"); + programIndicatorGroupB.setAutoFields(); + programIndicatorGroupB.addProgramIndicator(testPIb); + programIndicatorService.addProgramIndicatorGroup(programIndicatorGroupB); + + // Add a group with zero program indicators + ProgramIndicatorGroup programIndicatorGroupC = new ProgramIndicatorGroup("Test PI Group C"); + programIndicatorGroupC.setAutoFields(); + programIndicatorService.addProgramIndicatorGroup(programIndicatorGroupC); + dbmsManager.clearSession(); + + Set expected_uids = + Set.of(programIndicatorGroupC.getUid(), programIndicatorGroupB.getUid()); + Set expected_names = + Set.of(programIndicatorGroupC.getName(), programIndicatorGroupB.getName()); + + assertHasDataIntegrityIssues( + detailsIdType, check, 66, expected_uids, expected_names, Set.of("0", "1"), true); + } + + @Test + void testProgramIndicatorGroupSizeOK() { + + setUpTest(); + + dbmsManager.clearSession(); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testProgramIndicatorGroupSizeRuns() { + + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + public void setUpTest() { + + Program programA = new Program(); + programA.setName("Program A"); + programA.setShortName("Program A"); + programA.setProgramType(ProgramType.WITHOUT_REGISTRATION); + categoryService.getCategoryCombo(getDefaultCatCombo()); + programA.setCategoryCombo(categoryService.getCategoryCombo(getDefaultCatCombo())); + programService.addProgram(programA); + + ProgramIndicatorGroup programIndicatorGroupA = new ProgramIndicatorGroup("Test PI Group A"); + programIndicatorGroupA.setAutoFields(); + programIndicatorService.addProgramIndicatorGroup(programIndicatorGroupA); + + ProgramIndicator testPIa = new ProgramIndicator(); + testPIa.setAutoFields(); + testPIa.setName("Test PI A"); + testPIa.setShortName("Test PI A"); + testPIa.setProgram(programA); + programIndicatorService.addProgramIndicator(testPIa); + + testPIb = new ProgramIndicator(); + testPIb.setAutoFields(); + testPIb.setName("Test PI B"); + testPIb.setShortName("Test PI B"); + testPIb.setProgram(programA); + programIndicatorService.addProgramIndicator(testPIb); + + // Add two indicators to this group + programIndicatorGroupA.addProgramIndicator(testPIa); + programIndicatorGroupA.addProgramIndicator(testPIb); + programIndicatorService.addProgramIndicatorGroup(programIndicatorGroupA); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeUserGroupsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeUserGroupsControllerTest.java new file mode 100644 index 000000000000..ac384b18de81 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeUserGroupsControllerTest.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import java.util.Set; +import org.hisp.dhis.user.User; +import org.hisp.dhis.user.UserService; +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Tests the metadata check for category option groups which have fewer than two members. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_category_option_groups.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityGroupSizeUserGroupsControllerTest extends AbstractDataIntegrityIntegrationTest { + + @Autowired UserService userService; + + private static final String check = "user_groups_scarce"; + + private static final String detailsIdType = "userGroups"; + + private User bill; + + @Test + void testUserGroupsScarce() { + + setUpTest(); + + String userGroupB = + assertStatus( + HttpStatus.CREATED, + POST( + "/userGroups", + "{'name' : 'TestB', 'code' : 'TestB', 'users' : [{'id' : '" + + bill.getUid() + + "'}], 'managedGroups': [], 'attributeValues' : []}")); + + String userGroupC = + assertStatus( + HttpStatus.CREATED, + POST( + "/userGroups", + "{'name' : 'TestC', 'code' : 'TestC', 'users' : [], 'managedGroups': [], 'attributeValues' : []}")); + + assertHasDataIntegrityIssues( + detailsIdType, + check, + 66, + Set.of(userGroupB, userGroupC), + Set.of("TestB", "TestC"), + Set.of("0", "1"), + true); + } + + @Test + void testUserGroupsOK() { + setUpTest(); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testUserGroupScarceRuns() { + + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + private static User createUser(String uniquePart) { + User user = new User(); + user.setCode("Code" + uniquePart); + user.setFirstName("FirstName" + uniquePart); + user.setSurname("Surname" + uniquePart); + user.setUsername("username" + uniquePart); + return user; + } + + void setUpTest() { + + bill = createUser("Bill"); + User mary = createUser("Mary"); + + userService.addUser(bill); + userService.addUser(mary); + dbmsManager.clearSession(); + + assertStatus( + HttpStatus.CREATED, + POST( + "/userGroups", + "{'name' : 'TestA', 'code' : 'TestA', 'users' : [{'id' : '" + + bill.getUid() + + "'}, {'id' : '" + + mary.getUid() + + "'}], 'managedGroups': [], 'attributeValues' : []}")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeValidationRuleGroupsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeValidationRuleGroupsControllerTest.java new file mode 100644 index 000000000000..56271b65169e --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeValidationRuleGroupsControllerTest.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import java.util.Set; +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Test metadata check for minimum validation rule group size. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_validation_rule_groups.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityGroupSizeValidationRuleGroupsControllerTest + extends AbstractDataIntegrityIntegrationTest { + + private static final String check = "validation_rule_groups_scarce"; + + private static final String detailsIdType = "validationRuleGroups"; + + private String validationRuleA; + + @Test + void testValidationRuleGroupsTooSmall() { + + setUpTest(); + String indicatorGroupA = + assertStatus( + HttpStatus.CREATED, + POST( + "/validationRuleGroups", + "{ 'name' : 'One', 'shortName' : 'One', 'validationRules' : [{'id' : '" + + validationRuleA + + "'}]}")); + + String indicatorGroupB = + assertStatus( + HttpStatus.CREATED, + POST("/validationRuleGroups", "{ 'name' : 'None', 'shortName' : 'None'} ")); + + assertHasDataIntegrityIssues( + detailsIdType, + check, + 66, + Set.of(indicatorGroupA, indicatorGroupB), + Set.of("One", "None"), + Set.of("0", "1"), + true); + } + + @Test + void testValidationRuleGroupsSizeOK() { + + setUpTest(); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testIndicatorsInGroupsRuns() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + void setUpTest() { + + validationRuleA = + assertStatus( + HttpStatus.CREATED, + POST( + "/validationRules", + "{'importance':'MEDIUM','operator':'not_equal_to','leftSide':{'missingValueStrategy':'NEVER_SKIP', " + + "'description':'Test','expression':'abc123'}," + + "'rightSide':{'missingValueStrategy': 'NEVER_SKIP', 'description':'Test2'," + + "'expression':'xyz456'},'periodType':'Monthly','name':'Test rule A'}")); + + String validationRuleB = + assertStatus( + HttpStatus.CREATED, + POST( + "/validationRules", + "{'importance':'MEDIUM','operator':'not_equal_to','leftSide':{'missingValueStrategy':'NEVER_SKIP', " + + "'description':'Test 3','expression':'abc123'}," + + "'rightSide':{'missingValueStrategy': 'NEVER_SKIP', 'description':'Test 4'," + + "'expression':'xyz456'},'periodType':'Monthly','name':'Test rule B'}")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/validationRuleGroups", + "{ 'name' : 'A validation rule group', 'shortName' : 'A validation rule group', 'validationRules' : [{'id' : '" + + validationRuleA + + "'}, " + + "{'id' : '" + + validationRuleB + + "'}]}")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityIndicatorsDuplicatedFactorsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityIndicatorsDuplicatedFactorsControllerTest.java new file mode 100644 index 000000000000..3a77164be01e --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityIndicatorsDuplicatedFactorsControllerTest.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import java.util.Set; +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Tests for duplicated indicator types, namely those which have the same factor. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_duplicate_types.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityIndicatorsDuplicatedFactorsControllerTest + extends AbstractDataIntegrityIntegrationTest { + + private static final String check = "indicator_types_duplicated"; + + private static final String detailsIdType = "indicatorTypes"; + + @Test + void testDuplicatedIndicatorFactorsExist() { + String IndicatorA = + assertStatus( + HttpStatus.CREATED, + POST("/indicatorTypes", "{ 'name': 'Per cent', 'factor' : 100, 'number' : false }")); + + String IndicatorB = + assertStatus( + HttpStatus.CREATED, + POST( + "/indicatorTypes", + "{ 'name': 'Per one hundred', 'factor' : 100, 'number' : false }")); + + assertStatus( + HttpStatus.CREATED, + POST("/indicatorTypes", "{ 'name': 'Per thousand', 'factor' : 1000, 'number' : false }")); + + assertNamedMetadataObjectExists("indicatorTypes", "Per cent"); + assertNamedMetadataObjectExists("indicatorTypes", "Per one hundred"); + + assertHasDataIntegrityIssues( + detailsIdType, check, 66, Set.of(IndicatorA, IndicatorB), Set.of(), Set.of(), true); + } + + @Test + void testIndicatorFactorsUnique() { + + assertStatus( + HttpStatus.CREATED, + POST("/indicatorTypes", "{ 'name': 'Percent', 'factor' : 100, 'number' : false }")); + + assertStatus( + HttpStatus.CREATED, + POST("/indicatorTypes", "{ 'name': 'Per thousand', 'factor' : 1000, 'number' : false }")); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testDuplicatedIndicatorFactorsRuns() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityIndicatorsDuplicatedTermsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityIndicatorsDuplicatedTermsControllerTest.java new file mode 100644 index 000000000000..49956f8ba091 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityIndicatorsDuplicatedTermsControllerTest.java @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Set; +import org.hisp.dhis.jsontree.JsonList; +import org.hisp.dhis.web.HttpStatus; +import org.hisp.dhis.webapi.json.domain.JsonDataIntegrityDetails; +import org.junit.jupiter.api.Test; + +/** + * Tests metadata check for potentially duplicated indicator formulas. The check should identify + * formulas with the same term, regardless of order and spaces. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_duplicated_terms.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityIndicatorsDuplicatedTermsControllerTest + extends AbstractDataIntegrityIntegrationTest { + + private static final String check = "indicators_duplicated_terms"; + + @Test + void testIndicatorsDuplicatedTermsExist() { + String indicatorTypeA = + assertStatus( + HttpStatus.CREATED, + POST("/indicatorTypes", "{ 'name': 'Per cent', 'factor' : 100, 'number' : false }")); + + String indicatorA = + assertStatus( + HttpStatus.CREATED, + POST( + "/indicators", + "{ 'name': 'Indicator A', 'shortName': 'Indicator A', 'indicatorType' : {'id' : '" + + indicatorTypeA + + "'}," + + " 'numerator' : 'abc123 + def456', 'numeratorDescription' : 'One', 'denominator' : '1', " + + "'denominatorDescription' : 'One'} }")); + + String indicatorB = + assertStatus( + HttpStatus.CREATED, + POST( + "/indicators", + "{ 'name': 'Indicator B', 'shortName': 'Indicator B', 'indicatorType' : {'id' : '" + + indicatorTypeA + + "'}," + + " 'numerator' : ' def456 + abc123 ', 'numeratorDescription' : 'One', 'denominator' : '1', " + + "'denominatorDescription' : 'One'} }")); + + postDetails(check); + JsonDataIntegrityDetails details = getDetails(check); + JsonList issues = details.getIssues(); + assertTrue(issues.exists()); + + assertHasDataIntegrityIssues( + "indicators", + check, + 100, + Set.of(indicatorA, indicatorB), + Set.of("Indicator A", "Indicator B"), + Set.of(), + true); + } + + @Test + void testIndicatorFactorsUnique() { + String indicatorTypeA = + assertStatus( + HttpStatus.CREATED, + POST("/indicatorTypes", "{ 'name': 'Per cent', 'factor' : 100, 'number' : false }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/indicators", + "{ 'name': 'Indicator A', 'shortName': 'Indicator A', 'indicatorType' : {'id' : '" + + indicatorTypeA + + "'}," + + " 'numerator' : 'abc123 + def456', 'numeratorDescription' : 'One', 'denominator' : '1', " + + "'denominatorDescription' : 'One'} }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/indicators", + "{ 'name': 'Indicator B', 'shortName': 'Indicator B', 'indicatorType' : {'id' : '" + + indicatorTypeA + + "'}," + + " 'numerator' : 'abc123 + def456', 'numeratorDescription' : 'One', 'denominator' : 'xyz675', " + + "'denominatorDescription' : 'One'} }")); + + assertHasNoDataIntegrityIssues("indicators", check, true); + } + + @Test + void testDuplicatedIndicatorFactorsRuns() { + assertHasNoDataIntegrityIssues("indicators", check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityIndicatorsExactDuplicatesControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityIndicatorsExactDuplicatesControllerTest.java new file mode 100644 index 000000000000..e73278de6f52 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityIndicatorsExactDuplicatesControllerTest.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import java.util.Set; +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Tests metadata check for indicators with identical formulas.* {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_duplicated_terms.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityIndicatorsExactDuplicatesControllerTest + extends AbstractDataIntegrityIntegrationTest { + + private static final String check = "indicators_exact_duplicates"; + + @Test + void testIndicatorsExactDuplicatesExist() { + String indicatorTypeA = + assertStatus( + HttpStatus.CREATED, + POST("/indicatorTypes", "{ 'name': 'Per cent', 'factor' : 100, 'number' : false }")); + + String indicatorA = + assertStatus( + HttpStatus.CREATED, + POST( + "/indicators", + "{ 'name': 'Indicator A', 'shortName': 'Indicator A', 'indicatorType' : {'id' : '" + + indicatorTypeA + + "'}," + + " 'numerator' : 'abc123 + def456', 'numeratorDescription' : 'One', 'denominator' : '1', " + + "'denominatorDescription' : 'One'} }")); + + String indicatorB = + assertStatus( + HttpStatus.CREATED, + POST( + "/indicators", + "{ 'name': 'Indicator B', 'shortName': 'Indicator B', 'indicatorType' : {'id' : '" + + indicatorTypeA + + "'}," + + " 'numerator' : ' abc123 + def456 ', 'numeratorDescription' : 'One', 'denominator' : '1', " + + "'denominatorDescription' : 'One'} }")); + + assertHasDataIntegrityIssues( + "indicators", + check, + 100, + Set.of(indicatorA, indicatorB), + Set.of("Indicator A", "Indicator B"), + Set.of(), + true); + } + + @Test + void testIndicators() { + String indicatorTypeA = + assertStatus( + HttpStatus.CREATED, + POST("/indicatorTypes", "{ 'name': 'Per cent', 'factor' : 100, 'number' : false }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/indicators", + "{ 'name': 'Indicator A', 'shortName': 'Indicator A', 'indicatorType' : {'id' : '" + + indicatorTypeA + + "'}," + + " 'numerator' : 'abc123 + def456', 'numeratorDescription' : 'One', 'denominator' : '1', " + + "'denominatorDescription' : 'One'} }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/indicators", + "{ 'name': 'Indicator B', 'shortName': 'Indicator B', 'indicatorType' : {'id' : '" + + indicatorTypeA + + "'}," + + " 'numerator' : 'abc123 + def456', 'numeratorDescription' : 'One', 'denominator' : 'xyz675', " + + "'denominatorDescription' : 'One'} }")); + + assertHasNoDataIntegrityIssues("indicators", check, true); + } + + @Test + void testDuplicatedIndicatorFactorsRuns() { + assertHasNoDataIntegrityIssues("indicators", check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityIndicatorsNoAnalysisControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityIndicatorsNoAnalysisControllerTest.java new file mode 100644 index 000000000000..78823320fd62 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityIndicatorsNoAnalysisControllerTest.java @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Test metadata check for indicators not used in any analysis {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_noanalysis.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityIndicatorsNoAnalysisControllerTest extends AbstractDataIntegrityIntegrationTest { + + private static final String check = "indicator_no_analysis"; + + private String indicatorA; + + @Test + void testIndicatorsWithoutAnalysisExist() { + + setUpTest(); + assertHasDataIntegrityIssues("indicators", check, 100, indicatorA, null, null, true); + } + + @Test + void testIndicatorUsedInAnalysis() { + + setUpTest(); + // Create a visualization with the provided indicator + assertStatus( + HttpStatus.CREATED, + POST( + "/visualizations?skipTranslations=true&skipSharing=true", + "{'type':'SINGLE_VALUE','columns':[{'dimension':'dx','items':[{'id':'" + + indicatorA + + "'}]}],'rows':[],'filters':[{'dimension':'ou','items':[{'id':'USER_ORGUNIT'}]}, " + + "{'dimension':'pe','items':[{'id':'LAST_12_MONTHS'}]}],'axes':[],'colorSet':'DEFAULT', " + + "'cumulativeValues':false,'hideEmptyRowItems':'NONE','seriesKey':{},'legend':{}," + + "'noSpaceBetweenColumns':false,'percentStackedValues':false,'regressionType':'NONE', " + + "'showData':true,'aggregationType':'DEFAULT','completedOnly':false,'hideSubtitle':false," + + "'hideTitle':false,'sortOrder':0,'series':[],'fontStyle':{},'outlierAnalysis':null,'colTotals':false," + + "'colSubTotals':false,'rowTotals':false,'rowSubTotals':false,'showDimensionLabels':false," + + "'hideEmptyColumns':false,'hideEmptyRows':false,'skipRounding':false,'numberType':'VALUE', " + + "'showHierarchy':false,'displayDensity':'NORMAL','fontSize':'NORMAL','digitGroupSeparator':'SPACE'," + + "'fixColumnHeaders':false,'fixRowHeaders':false,'regression':false,'cumulative':false,'topLimit':0,'" + + "" + + "reportingParams':{'organisationUnit':false,'reportingPeriod':false,'parentOrganisationUnit':false," + + "'grandParentOrganisationUnit':false},'name':'Test viz'}")); + + assertHasNoDataIntegrityIssues("indicators", check, true); + } + + @Test + void testIndicatorsInAnalysisRuns() { + assertHasNoDataIntegrityIssues("indicators", check, false); + } + + void setUpTest() { + + String indicatorTypeA = + assertStatus( + HttpStatus.CREATED, + POST("/indicatorTypes", "{ 'name': 'Per cent', 'factor' : 100, 'number' : false }")); + + indicatorA = + assertStatus( + HttpStatus.CREATED, + POST( + "/indicators", + "{ 'name': 'Indicator A', 'shortName': 'Indicator A', 'indicatorType' : {'id' : '" + + indicatorTypeA + + "'}," + + " 'numerator' : 'abc123', 'numeratorDescription' : 'One', 'denominator' : 'abc123', " + + "'denominatorDescription' : 'Zero'} }")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityIndicatorsNotGroupedControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityIndicatorsNotGroupedControllerTest.java new file mode 100644 index 000000000000..6f84a9ad61f6 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityIndicatorsNotGroupedControllerTest.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.hisp.dhis.jsontree.JsonList; +import org.hisp.dhis.jsontree.JsonObject; +import org.hisp.dhis.web.HttpStatus; +import org.hisp.dhis.webapi.json.domain.JsonIndicator; +import org.hisp.dhis.webapi.json.domain.JsonIndicatorGroup; +import org.junit.jupiter.api.Test; + +/** + * Test metadata check for indicators which are not part of an indicator group {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_nongrouped.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityIndicatorsNotGroupedControllerTest extends AbstractDataIntegrityIntegrationTest { + + private static final String check = "indicators_not_grouped"; + + private String indicatorA; + + private String indicatorB; + + @Test + void testIndicatorsWithoutGroupsExist() { + + setUpTest(); + assertStatus( + HttpStatus.CREATED, + POST( + "/indicatorGroups", + "{ 'name' : 'An indicator group', 'indicators' : [{'id' : '" + indicatorA + "'}]}")); + + JsonObject content = GET("/indicatorGroups?fields=id,name,indicators[id,name]").content(); + JsonList myIndicatorGroup = + content.getList("indicatorGroups", JsonIndicatorGroup.class); + assertEquals(1, myIndicatorGroup.size()); + JsonList myIndicators = myIndicatorGroup.get(0).getIndicators(); + assertEquals(1, myIndicators.size()); + assertEquals(indicatorA, myIndicators.get(0).getId()); + + assertHasDataIntegrityIssues("indicators", check, 50, indicatorB, null, null, true); + } + + @Test + void testIndicatorsInGroups() { + + setUpTest(); + assertStatus( + HttpStatus.CREATED, + POST( + "/indicatorGroups", + "{ 'name' : 'An indicator group', 'indicators' : [{'id' : '" + + indicatorA + + "'}, " + + " {'id' : '" + + indicatorB + + "'}]}")); + + assertHasNoDataIntegrityIssues("indicators", check, true); + } + + @Test + void testIndicatorsInGroupsRuns() { + assertHasNoDataIntegrityIssues("indicators", check, false); + } + + void setUpTest() { + + String indicatorTypeA = + assertStatus( + HttpStatus.CREATED, + POST("/indicatorTypes", "{ 'name': 'Per cent', 'factor' : 100, 'number' : false }")); + + indicatorA = + assertStatus( + HttpStatus.CREATED, + POST( + "/indicators", + "{ 'name': 'Indicator A', 'shortName': 'Indicator A', 'indicatorType' : {'id' : '" + + indicatorTypeA + + "'}," + + " 'numerator' : 'abc123', 'numeratorDescription' : 'One', 'denominator' : 'abc123', " + + "'denominatorDescription' : 'Zero'} }")); + + indicatorB = + assertStatus( + HttpStatus.CREATED, + POST( + "/indicators", + "{ 'name': 'Indicator B', 'shortName': 'Indicator B', 'indicatorType' : {'id' : '" + + indicatorTypeA + + "'}," + + " 'numerator' : 'abc123', 'numeratorDescription' : 'One', 'denominator' : 'abc123', " + + "'denominatorDescription' : 'Zero'}")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityMapsNotUsedOneYearControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityMapsNotUsedOneYearControllerTest.java new file mode 100644 index 000000000000..c9afc8dd3130 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityMapsNotUsedOneYearControllerTest.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import java.time.ZonedDateTime; +import java.util.Date; +import org.hisp.dhis.datastatistics.DataStatisticsEvent; +import org.hisp.dhis.datastatistics.DataStatisticsEventStore; +import org.hisp.dhis.datastatistics.DataStatisticsEventType; +import org.hisp.dhis.mapping.Map; +import org.hisp.dhis.mapping.MapView; +import org.hisp.dhis.mapping.MappingService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Test for maps which have not been viewed in at least a year. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/maps_not_used_1year.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityMapsNotUsedOneYearControllerTest extends AbstractDataIntegrityIntegrationTest { + + @Autowired private DataStatisticsEventStore dataStatisticsEventStore; + + @Autowired private MappingService mappingService; + + private DataStatisticsEvent dse1; + + private static final String check = "maps_not_viewed_one_year"; + + @Test + void testUnusedVisualizationsExist() { + + Date oneYearAgo = Date.from(ZonedDateTime.now().minusYears(1).minusDays(1).toInstant()); + + dse1 = + new DataStatisticsEvent(DataStatisticsEventType.MAP_VIEW, oneYearAgo, "TestUser", BASE_UID); + dataStatisticsEventStore.save(dse1); + + dbmsManager.clearSession(); + + assertNamedMetadataObjectExists("maps", "MapA"); + assertHasDataIntegrityIssues("maps", check, 100, BASE_UID, "MapA", null, true); + } + + @Test + void testUsedVisualizationsExist() { + + long millis = System.currentTimeMillis(); + Date rightNow = new Date(millis); + + dse1 = + new DataStatisticsEvent(DataStatisticsEventType.MAP_VIEW, rightNow, "TestUser", BASE_UID); + dataStatisticsEventStore.save(dse1); + + dbmsManager.clearSession(); + + assertHasNoDataIntegrityIssues("maps", check, true); + } + + @Test + void testUnusedVisualizationsRuns() { + assertHasNoDataIntegrityIssues("maps", check, false); + } + + @BeforeEach + void setUp() { + /* Simplified setup from MappingServiceTest */ + MapView mvA = new MapView("thematic"); + Map mpA = new Map("MapA", null, 0d, 0d, 0); + mpA.setUid(BASE_UID); + mpA.getMapViews().add(mvA); + mappingService.addMap(mpA); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOptionSetNoOptionsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOptionSetNoOptionsControllerTest.java new file mode 100644 index 000000000000..cbe0fa797cce --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOptionSetNoOptionsControllerTest.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.hisp.dhis.jsontree.JsonList; +import org.hisp.dhis.jsontree.JsonObject; +import org.hisp.dhis.web.HttpStatus; +import org.hisp.dhis.webapi.json.domain.JsonOption; +import org.junit.jupiter.api.Test; + +/** + * Test for option sets with no options. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/option_sets_empty.yaml + * } + * + * @author Jason P. Pickering + */ +class DataIntegrityOptionSetNoOptionsControllerTest extends AbstractDataIntegrityIntegrationTest { + private static final String check = "options_sets_empty"; + + private static final String detailsIdType = "optionSets"; + + @Test + void testOptionSetInvalid() { + + String goodOptionSet = + assertStatus( + HttpStatus.CREATED, + POST("/optionSets", "{ 'name': 'Taste', 'shortName': 'Taste', 'valueType' : 'TEXT' }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/options", + "{ 'code': 'SWEET'," + + " 'sortOrder': 1," + + " 'name': 'Sweet'," + + " 'optionSet': { " + + " 'id': '" + + goodOptionSet + + "'" + + " }}")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/options", + "{ 'code': 'SOUR'," + + " 'sortOrder': 2," + + " 'name': 'Sour'," + + " 'optionSet': { " + + " 'id': '" + + goodOptionSet + + "'" + + " }}")); + + String badOptionSet = + assertStatus( + HttpStatus.CREATED, + POST("/optionSets", "{ 'name': 'Color', 'shortName': 'Color', 'valueType' : 'TEXT' }")); + + JsonObject content = GET("/" + detailsIdType + "/" + goodOptionSet).content(); + JsonList optionSetOptions = content.getList("options", JsonOption.class); + assertEquals(2, optionSetOptions.size()); + + content = GET("/" + detailsIdType + "/" + badOptionSet).content(); + optionSetOptions = content.getList("options", JsonOption.class); + assertEquals(0, optionSetOptions.size()); + + assertHasDataIntegrityIssues(detailsIdType, check, 50, badOptionSet, "Color", null, true); + } + + @Test + void testOptionSetsValid() { + + String goodOptionSet = + assertStatus( + HttpStatus.CREATED, + POST("/optionSets", "{ 'name': 'Taste', 'shortName': 'Taste', 'valueType' : 'TEXT' }")); + + String badOptionSet = + assertStatus( + HttpStatus.CREATED, + POST("/optionSets", "{ 'name': 'Color', 'shortName': 'Color', 'valueType' : 'TEXT' }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/options", + "{ 'code': 'SWEET'," + + " 'sortOrder': 1," + + " 'name': 'Sweet'," + + " 'optionSet': { " + + " 'id': '" + + goodOptionSet + + "'" + + " }}")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/options", + "{ 'code': 'SOUR'," + + " 'sortOrder': 1," + + " 'name': 'Sour'," + + " 'optionSet': { " + + " 'id': '" + + badOptionSet + + "'" + + " }}")); + + JsonObject content = GET("/" + detailsIdType + "/" + goodOptionSet).content(); + JsonList optionSetOptions = content.getList("options", JsonOption.class); + assertEquals(1, optionSetOptions.size()); + + content = GET("/" + detailsIdType + "/" + badOptionSet).content(); + optionSetOptions = content.getList("options", JsonOption.class); + assertEquals(1, optionSetOptions.size()); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testInvalidCategoriesDivideByZero() { + + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOptionSetsNotUsedControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOptionSetsNotUsedControllerTest.java new file mode 100644 index 000000000000..cadbd5c36dfa --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOptionSetsNotUsedControllerTest.java @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.hisp.dhis.jsontree.JsonList; +import org.hisp.dhis.jsontree.JsonObject; +import org.hisp.dhis.web.HttpStatus; +import org.hisp.dhis.webapi.json.domain.JsonDataElement; +import org.hisp.dhis.webapi.json.domain.JsonOption; +import org.hisp.dhis.webapi.json.domain.JsonOptionSet; +import org.junit.jupiter.api.Test; + +/** + * Test for option sets which are not used. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/unused_option_sets.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityOptionSetsNotUsedControllerTest extends AbstractDataIntegrityIntegrationTest { + private static final String check = "options_sets_unused"; + + private static final String detailsIdType = "optionSets"; + + private String goodOptionSet; + + @Test + void testOptionSetNotUsed() { + + goodOptionSet = + assertStatus( + HttpStatus.CREATED, + POST("/optionSets", "{ 'name': 'Taste', 'shortName': 'Taste', 'valueType' : 'TEXT' }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/options", + "{ 'code': 'SWEET'," + + " 'sortOrder': 1," + + " 'name': 'Sweet'," + + " 'optionSet': { " + + " 'id': '" + + goodOptionSet + + "'" + + " }}")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/options", + "{ 'code': 'SOUR'," + + " 'sortOrder': 2," + + " 'name': 'Sour'," + + " 'optionSet': { " + + " 'id': '" + + goodOptionSet + + "'" + + " }}")); + + JsonObject content = GET("/optionSets?fields=id,name,options[id]").content(); + JsonList myOptionSets = content.getList(detailsIdType, JsonOptionSet.class); + assertEquals(1, myOptionSets.size()); + JsonOptionSet myOptionSet = myOptionSets.get(0); + assertEquals(goodOptionSet, myOptionSet.getId()); + JsonList optionSetOptions = myOptionSet.getOptions(); + assertEquals(2, optionSetOptions.size()); + + assertHasDataIntegrityIssues(detailsIdType, check, 100, goodOptionSet, "Taste", null, true); + } + + @Test + void testOptionSetsUsed() { + + goodOptionSet = + assertStatus( + HttpStatus.CREATED, + POST("/optionSets", "{ 'name': 'Taste', 'shortName': 'Taste', 'valueType' : 'TEXT' }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/options", + "{ 'code': 'SWEET'," + + " 'sortOrder': 1," + + " 'name': 'Sweet'," + + " 'optionSet': { " + + " 'id': '" + + goodOptionSet + + "'" + + " }}")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/options", + "{ 'code': 'SOUR'," + + " 'sortOrder': 2," + + " 'name': 'Sour'," + + " 'optionSet': { " + + " 'id': '" + + goodOptionSet + + "'" + + " }}")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'Candy', 'shortName': 'Candy', 'valueType' : 'TEXT', " + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'NONE'," + + "'optionSet' : { 'id' : '" + + goodOptionSet + + "'} }")); + + JsonObject content = GET("/dataElements/?fields=id,name,optionSet").content(); + JsonList testDataElementJSON = + content.getList("dataElements", JsonDataElement.class); + assertEquals(1, testDataElementJSON.size()); + assertEquals("Candy", testDataElementJSON.get(0).getName()); + assertEquals(goodOptionSet, testDataElementJSON.get(0).getOptionSet().getId()); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testInvalidCategoriesDivideByZero() { + + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOptionSetsWrongSortOrderControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOptionSetsWrongSortOrderControllerTest.java new file mode 100644 index 000000000000..fb25a589c0b0 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOptionSetsWrongSortOrderControllerTest.java @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Set; +import java.util.stream.Collectors; +import org.hisp.dhis.common.ValueType; +import org.hisp.dhis.jsontree.JsonObject; +import org.hisp.dhis.option.Option; +import org.hisp.dhis.option.OptionService; +import org.hisp.dhis.option.OptionSet; +import org.hisp.dhis.web.HttpStatus; +import org.hisp.dhis.webapi.json.domain.JsonOption; +import org.hisp.dhis.webapi.json.domain.JsonOptionSet; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Test for option sets which are not used. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/option_sets_wrong_sort_order.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityOptionSetsWrongSortOrderControllerTest + extends AbstractDataIntegrityIntegrationTest { + + @Autowired private OptionService myOptionService; + + private String goodOptionSet; + + private static final String check = "option_sets_wrong_sort_order"; + + private static final String detailsIdType = "optionSets"; + + @Test + @Disabled("Cannot directly set the sort order of option sets") + void testOptionSetWrongSortOrder() { + + Option optionA = new Option("Sweet", "SWEET", 1); + Option optionB = new Option("Sour", "SOUR", 2); + Option optionC = new Option("Salty", "SALTY", 3); + OptionSet optionSetA = new OptionSet("Taste", ValueType.TEXT); + optionSetA.addOption(optionA); + optionSetA.addOption(optionB); + optionSetA.addOption(optionC); + myOptionService.saveOptionSet(optionSetA); + optionSetA.removeOption(optionB); + myOptionService.saveOptionSet(optionSetA); + dbmsManager.clearSession(); + + goodOptionSet = optionSetA.getUid(); + + JsonObject content = + GET("/optionSets/" + goodOptionSet + "?fields=id,name,options[id,name,sortOrder").content(); + JsonOptionSet myOptionSet = content.asObject(JsonOptionSet.class); + assertEquals(myOptionSet.getId(), goodOptionSet); + + Set sortOrders = + myOptionSet.getOptions().stream().map(JsonOption::getSortOrder).collect(Collectors.toSet()); + Set expectedSortOrders = Set.of(1, 4); + assertEquals(expectedSortOrders, sortOrders); + + assertHasDataIntegrityIssues(detailsIdType, check, 100, goodOptionSet, "Taste", "4 != 2", true); + } + + @Test + void testOptionSetRightSortOrder() { + + goodOptionSet = + assertStatus( + HttpStatus.CREATED, + POST("/optionSets", "{ 'name': 'Taste', 'shortName': 'Taste', 'valueType' : 'TEXT' }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/options", + "{ 'code': 'SWEET'," + + " 'sortOrder': 1," + + " 'name': 'Sweet'," + + " 'optionSet': { " + + " 'id': '" + + goodOptionSet + + "'" + + " }}")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/options", + "{ 'code': 'SOUR'," + + " 'sortOrder': 2," + + " 'name': 'Sour'," + + " 'optionSet': { " + + " 'id': '" + + goodOptionSet + + "'" + + " }}")); + + JsonObject content = + GET("/optionSets/" + goodOptionSet + "?fields=id,name,options[id,name,sortOrder").content(); + JsonOptionSet myOptionSet = content.asObject(JsonOptionSet.class); + assertEquals(myOptionSet.getId(), goodOptionSet); + + Set sortOrders = + myOptionSet.getOptions().stream().map(JsonOption::getSortOrder).collect(Collectors.toSet()); + Set expectedSortOrders = Set.of(1, 2); + assertEquals(expectedSortOrders, sortOrders); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testInvalidCategoriesDivideByZero() { + + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitCompulsoryGroupControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitCompulsoryGroupControllerTest.java new file mode 100644 index 000000000000..722d80c84511 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitCompulsoryGroupControllerTest.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Test for organisation units which are not part of a compulsory organisation unit group set. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/compulsory_orgunit_groups.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityOrganisationUnitCompulsoryGroupControllerTest + extends AbstractDataIntegrityIntegrationTest { + + private String inGroup; + + private String testOrgUnitGroup; + + private static final String check = "orgunits_compulsory_group_count"; + + private static final String detailsIdType = "organisationUnits"; + + @Test + void testOrgUnitNotInCompulsoryGroup() { + + String outOfGroup = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Fish District', 'shortName': 'Fish District', 'openingDate' : '2022-01-01'}")); + + inGroup = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Pizza District', 'shortName': 'Pizza District', 'openingDate' : '2022-01-01'}")); + + // Create an orgunit group + testOrgUnitGroup = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnitGroups", + "{'name': 'Type A', 'shortName': 'Type A', 'organisationUnits' : [{'id' : '" + + inGroup + + "'}]}")); + + // Add it to a group set + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnitGroupSets", + "{'name': 'Type', 'shortName': 'Type', 'compulsory' : 'true' , 'organisationUnitGroups' :[{'id' : '" + + testOrgUnitGroup + + "'}]}")); + + assertHasDataIntegrityIssues(detailsIdType, check, 50, outOfGroup, "Fish District", "", true); + } + + @Test + void testOrgunitInCompulsoryGroupSet() { + + inGroup = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Pizza District', 'shortName': 'Pizza District', 'openingDate' : '2022-01-01'}")); + + // Create an orgunit group + testOrgUnitGroup = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnitGroups", + "{'name': 'Type A', 'shortName': 'Type A', 'organisationUnits' : [{'id' : '" + + inGroup + + "'}]}")); + + // Add it to a group set + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnitGroupSets", + "{'name': 'Type', 'shortName': 'Type', 'compulsory' : 'true' , 'organisationUnitGroups' :[{'id' : '" + + testOrgUnitGroup + + "'}]}")); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testOrgUnitsCompulsoryGroupsRuns() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitExcessGroupsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitExcessGroupsControllerTest.java new file mode 100644 index 000000000000..d5339fad751b --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitExcessGroupsControllerTest.java @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Checks for organisation units which are part of multiple organisation unit groups within the same + * group set. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_excess_group_memberships.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityOrganisationUnitExcessGroupsControllerTest + extends AbstractDataIntegrityIntegrationTest { + + private String orgunitA; + + private String orgunitB; + + private String testOrgUnitGroupA; + + private String testOrgUnitGroupB; + + private static final String check = "orgunit_group_sets_excess_groups"; + + private static final String detailsIdType = "organisationUnits"; + + @Test + void testOrganisationUnitInMultipleGroupSetGroups() { + + orgunitA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Fish District', 'shortName': 'Fish District', 'openingDate' : '2022-01-01'}")); + + orgunitB = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Pizza District', 'shortName': 'Pizza District', 'openingDate' : '2022-01-01'}")); + + // Create an orgunit group + testOrgUnitGroupA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnitGroups", + "{'name': 'Type A', 'shortName': 'Type A', 'organisationUnits' : [{'id' : '" + + orgunitA + + "'}, {'id' : '" + + orgunitB + + "'}]}")); + + testOrgUnitGroupB = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnitGroups", + "{'name': 'Type B', 'shortName': 'Type B', 'organisationUnits' : [{'id' : '" + + orgunitB + + "'}]}")); + + // Add it to a group set + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnitGroupSets", + "{'name': 'Type', 'shortName': 'Type', 'compulsory' : 'true' , " + + "'organisationUnitGroups' :[{'id' : '" + + testOrgUnitGroupA + + "'}, {'id' : '" + + testOrgUnitGroupB + + "'}]}")); + + assertHasDataIntegrityIssues( + detailsIdType, check, 50, orgunitB, "Pizza District", "Type", true); + } + + @Test + void testOrganisationUnitNotInMultipleGroupSetGroups() { + + orgunitA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Fish District', 'shortName': 'Fish District', 'openingDate' : '2022-01-01'}")); + + orgunitB = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Pizza District', 'shortName': 'Pizza District', 'openingDate' : '2022-01-01'}")); + + // Create an orgunit group + testOrgUnitGroupA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnitGroups", + "{'name': 'Type A', 'shortName': 'Type A', 'organisationUnits' : [{'id' : '" + + orgunitA + + "'}]}")); + + testOrgUnitGroupB = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnitGroups", + "{'name': 'Type B', 'shortName': 'Type B', 'organisationUnits' : [{'id' : '" + + orgunitB + + "'}]}")); + + // Add it to a group set + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnitGroupSets", + "{'name': 'Type', 'shortName': 'Type', 'compulsory' : 'true' , " + + "'organisationUnitGroups' :[{'id' : '" + + testOrgUnitGroupA + + "'}, {'id' : '" + + testOrgUnitGroupB + + "'}]}")); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testOrganisationMultipleGroupsInGroupSetDivideByZero() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitNamesMultipleSpacesControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitNamesMultipleSpacesControllerTest.java new file mode 100644 index 000000000000..44084a5dcdd4 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitNamesMultipleSpacesControllerTest.java @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import java.util.Set; +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Tests for orgunits which have multiple spaces in their names or shortnames {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_multiple_spaces.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityOrganisationUnitNamesMultipleSpacesControllerTest + extends AbstractDataIntegrityIntegrationTest { + + private String orgunitA; + + private String orgunitB; + + private String orgunitC; + + private static final String check = "orgunits_multiple_spaces"; + + private static final String detailsIdType = "organisationUnits"; + + @Test + void testOrgUnitMultipleSpaces() { + + orgunitA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Space District', 'shortName': 'Space District', 'openingDate' : '2022-01-01'}")); + + orgunitB = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'TwoSpace District', 'shortName': 'Two Space District', 'openingDate' : '2022-01-01', " + + "'parent': {'id' : '" + + orgunitA + + "'}}")); + + orgunitC = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'NospaceDistrict', 'shortName': 'NospaceDistrict', 'openingDate' : '2022-01-01'}")); + + assertHasDataIntegrityIssues( + detailsIdType, check, 66, Set.of(orgunitA, orgunitB), Set.of(), Set.of(), true); + } + + @Test + void orgunitsNoMultipleSpaces() { + orgunitC = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'NospaceDistrict', 'shortName': 'NospaceDistrict', 'openingDate' : '2022-01-01'}")); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testOrgunitsMultipleSpacesZeroCase() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + @BeforeEach + void setUp() { + deleteAllOrgUnits(); + } + + @AfterEach + void tearDown() { + deleteMetadataObject("organisationUnits", orgunitC); + deleteMetadataObject("organisationUnits", orgunitB); + deleteMetadataObject("organisationUnits", orgunitA); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitNullIslandControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitNullIslandControllerTest.java new file mode 100644 index 000000000000..e030224eb087 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitNullIslandControllerTest.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Checks for organisation units with coordinates close to Null Island (0 N, 0 E). {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunit_null_island.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityOrganisationUnitNullIslandControllerTest + extends AbstractDataIntegrityIntegrationTest { + + private String nullIsland; + + private static final String check = "orgunits_null_island"; + + private static final String detailsIdType = "organisationUnits"; + + @Test + void testOrgUnitNullIsland() { + + nullIsland = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Null Island', 'shortName': 'Null Island', " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [ 0.001, 0.004]} }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Not Null Island', 'shortName': 'Null Island', " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [ 10.2, 13.2]} }")); + + assertHasDataIntegrityIssues(detailsIdType, check, 50, nullIsland, "Null Island", null, true); + } + + @Test + void testOrgUnitNotNullIsland() { + + nullIsland = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Null Island', 'shortName': 'Null Island', " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [ 5,6]} }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Not Null Island', 'shortName': 'Null Island', " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [ 10.2, 13.2]} }")); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testOrgUnitNotNullIslandZeroCase() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitOpenClosedDateControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitOpenClosedDateControllerTest.java new file mode 100644 index 000000000000..2f39723001ce --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitOpenClosedDateControllerTest.java @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.organisationunit.OrganisationUnitService; +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Tests for orgunits whose closed dates are after their opening dates {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunit_open_date_gt_closed_date.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityOrganisationUnitOpenClosedDateControllerTest + extends AbstractDataIntegrityIntegrationTest { + @Autowired private OrganisationUnitService orgUnitService; + + private static final String check = "orgunits_openingdate_gt_closeddate"; + + private static final String detailsIdType = "organisationUnits"; + + @Test + void testOrgUnitOpeningDateAfterClosedDate() { + + OrganisationUnit unitA = createOrganisationUnit('A'); + unitA.setOpeningDate(getDate("2022-01-01")); + unitA.setClosedDate(getDate("2020-01-01")); + orgUnitService.addOrganisationUnit(unitA); + + OrganisationUnit unitB = createOrganisationUnit('B'); + unitB.setOpeningDate(getDate("2022-01-01")); + unitB.setClosedDate(getDate("2023-01-01")); + orgUnitService.addOrganisationUnit(unitB); + + OrganisationUnit unitC = createOrganisationUnit('C'); + unitC.setOpeningDate(getDate("2022-01-01")); + unitC.setClosedDate(null); + orgUnitService.addOrganisationUnit(unitC); + + dbmsManager.clearSession(); + + assertHasDataIntegrityIssues( + detailsIdType, check, 33, unitA.getUid(), unitA.getName(), null, true); + } + + @Test + void testOrgunitsWithOpenClosedDates() { + + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Null Island', 'shortName': 'Null Island', " + + "'openingDate' : '2022-01-01', 'closedDate' : '2023-02-22', 'geometry' : {'type' : 'Point', 'coordinates' : [5,6]} }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Not Null Island', 'shortName': 'Null Island', " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [ 10.2, 13.2]} }")); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testOrgunitsOpenClosedDateRuns() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsInvalidGeometryControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsInvalidGeometryControllerTest.java new file mode 100644 index 000000000000..4b24e92bfa9e --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsInvalidGeometryControllerTest.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Checks for organisation units which have an invalid geometry. The reasons for this may vary, but + * in this test case, we look for a polygon with self-intersection. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_invalid_geometry.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityOrganisationUnitsInvalidGeometryControllerTest + extends AbstractDataIntegrityIntegrationTest { + + private static final String check = "orgunits_invalid_geometry"; + + private static final String detailsIdType = "organisationUnits"; + + @Test + void testOrgunitsInvalidGeometry() { + + String districtA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Bowtie District', 'shortName': 'District A', " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Polygon', 'coordinates' : [[[10,20],[10,10],[20,20],[20,10],[10,20]]]} }")); + + createFacilities(districtA); + + assertHasDataIntegrityIssues( + detailsIdType, check, 33, districtA, "Bowtie District", "Self-intersection", true); + } + + @Test + void testOrgunitsValidGeometry() { + + String districtA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'District A', 'shortName': 'District A', " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Polygon', 'coordinates' : [[[0,0],[3,0],[3,3],[0,3],[0,0]]]} }")); + + createFacilities(districtA); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testOrgunitsInvalidGeometryDivideByZero() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + private void createFacilities(String districtA) { + + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Clinic A', 'shortName': 'Clinic A', " + + "'parent': {'id' : '" + + districtA + + "'}, " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [1, 1]} }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Clinic B', 'shortName': 'Clinic B', " + + "'parent': {'id' : '" + + districtA + + "'}, " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [2, 2]} }")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsMultipleRootsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsMultipleRootsControllerTest.java new file mode 100644 index 000000000000..b1fd263d8ce3 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsMultipleRootsControllerTest.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import java.util.Set; +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Test for multiple roots in the organisation unit hierarchy. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_multiple_roots.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityOrganisationUnitsMultipleRootsControllerTest + extends AbstractDataIntegrityIntegrationTest { + private static final String check = "orgunits_multiple_roots"; + + private static final String detailsIdType = "organisationUnits"; + + private String nullIsland; + + private String notNullIsland; + + @Test + void testOrgunitMultipleRoots() { + + nullIsland = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Null Island', 'shortName': 'Null Island', " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [ 0.001, 0.004]} }")); + + notNullIsland = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Not Null Island', 'shortName': 'Null Island', " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [ 10.2, 13.2]} }")); + + Set orgUnitUIDs = Set.of(); + if (nullIsland != null && notNullIsland != null) { + orgUnitUIDs = Set.of(nullIsland, notNullIsland); + } + assertHasDataIntegrityIssues(detailsIdType, check, 100, orgUnitUIDs, Set.of(), Set.of(), true); + } + + @Test + void testOrgunitNoMultipleRoots() { + + nullIsland = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Null Island', 'shortName': 'Null Island', " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [ 0.001, 0.004]} }")); + + notNullIsland = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Not Null Island', 'shortName': 'Null Island', " + + "'parent' : {'id': '" + + nullIsland + + "'}, " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [ 10.2, 13.2]} }")); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testOrgUnitsMultipleRootsRuns() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGeometryControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGeometryControllerTest.java new file mode 100644 index 000000000000..c549ceaa7493 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGeometryControllerTest.java @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Checks for organisation units with no geometry. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_no_geometry.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityOrganisationUnitsNoGeometryControllerTest + extends AbstractDataIntegrityIntegrationTest { + + private String clinicA; + + private String clinicB; + + private String districtA; + + private static final String check = "orgunits_no_coordinates"; + + private static final String detailsIdType = "organisationUnits"; + + @Test + void testOrgunitsNoGeometry() { + + districtA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Offgrid District', 'shortName': 'Offgrid District', 'openingDate' : '2022-01-01' }")); + + clinicA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Clinic A', 'shortName': 'Clinic A', " + + "'parent': {'id' : '" + + districtA + + "'}, " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [1, 1]} }")); + + clinicB = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Clinic B', 'shortName': 'Clinic B', " + + "'parent': {'id' : '" + + districtA + + "'}, " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [2, 2]} }")); + + assertNamedMetadataObjectExists(detailsIdType, "Clinic B"); + assertHasDataIntegrityIssues( + detailsIdType, check, 33, districtA, "Offgrid District", "1", true); + } + + @Test + void testOrgunitsHasGeometry() { + + districtA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'District A', 'shortName': 'District A', " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Polygon', 'coordinates' : [[[0,0],[3,0],[3,3],[0,3],[0,0]]]} }")); + + clinicA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Clinic A', 'shortName': 'Clinic A', " + + "'parent': {'id' : '" + + districtA + + "'}, " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [1, 1]} }")); + + clinicB = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Clinic B', 'shortName': 'Clinic B', " + + "'parent': {'id' : '" + + districtA + + "'}, " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [2, 2]} }")); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testOrgunitsNoGeometryDivideByZero() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + @BeforeEach + void setUp() { + deleteAllOrgUnits(); + } + + @AfterEach + void tearDown() { + deleteMetadataObject("organisationUnits", clinicB); + deleteMetadataObject("organisationUnits", clinicA); + deleteMetadataObject("organisationUnits", districtA); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNotContainedByParentControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNotContainedByParentControllerTest.java new file mode 100644 index 000000000000..40f88f1ba8a4 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNotContainedByParentControllerTest.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Checks for organisation units which have point coordinates which are not contained by their + * parent organisation unit. This only applies to situations where the parent has geometry of type + * Polygon or Multipolygon. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_trailing_spaces.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityOrganisationUnitsNotContainedByParentControllerTest + extends AbstractDataIntegrityIntegrationTest { + + private String clinicB; + + private String districtA; + + private static final String check = "orgunits_not_contained_by_parent"; + + private static final String detailsIdType = "organisationUnits"; + + @Test + void testOrgunitsNotContainedByParent() { + + districtA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'District A', 'shortName': 'District A', " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Polygon', 'coordinates' : [[[0,0],[3,0],[3,3],[0,3],[0,0]]]} }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Clinic A', 'shortName': 'Clinic A', " + + "'parent': {'id' : '" + + districtA + + "'}, " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [1, 1]} }")); + + clinicB = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Clinic B', 'shortName': 'Clinic B', " + + "'parent': {'id' : '" + + districtA + + "'}, " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [5, 5]} }")); + + assertHasDataIntegrityIssues(detailsIdType, check, 50, clinicB, "Clinic B", null, true); + } + + @Test + void testOrgunitsContainedByParent() { + + districtA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'District A', 'shortName': 'District A', " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Polygon', 'coordinates' : [[[0,0],[3,0],[3,3],[0,3],[0,0]]]} }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Clinic A', 'shortName': 'Clinic A', " + + "'parent': {'id' : '" + + districtA + + "'}, " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [1, 1]} }")); + + clinicB = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Clinic B', 'shortName': 'Clinic B', " + + "'parent': {'id' : '" + + districtA + + "'}, " + + "'openingDate' : '2022-01-01', 'geometry' : {'type' : 'Point', 'coordinates' : [2, 2]} }")); + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testOrgunitsContainedByParentDivideRuns() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsSameNameAndParentControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsSameNameAndParentControllerTest.java new file mode 100644 index 000000000000..10f375bcd44e --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsSameNameAndParentControllerTest.java @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import java.util.Set; +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Tests for organisation units which have the same name and parent. + * + *

{@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_same_name_and_parent.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityOrganisationUnitsSameNameAndParentControllerTest + extends AbstractDataIntegrityIntegrationTest { + private static final String check = "orgunits_same_name_and_parent"; + + private static final String detailsIdType = "organisationUnits"; + + private String orgunitA; + + private String orgunitB; + + @Test + void DataIntegrityOrganisationUnitsSameParentAndName() { + + setupTest(); + String orgunitC = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Pizza District', 'shortName': 'Pizza District', 'openingDate' : '2023-01-01', " + + "'parent': {'id' : '" + + orgunitA + + "'}}")); + + assertHasDataIntegrityIssues( + detailsIdType, + check, + 66, + Set.of(orgunitB, orgunitC), + Set.of("Pizza District"), + Set.of(), + true); + } + + @Test + void orgUnitsDifferentNames() { + setupTest(); + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Taco District', 'shortName': 'Taco District', 'openingDate' : '2023-01-01', " + + "'parent': {'id' : '" + + orgunitA + + "'}}")); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testOrgunitsSameNameParentZeroCase() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + void setupTest() { + orgunitA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Fish District', 'shortName': 'Fish District', 'openingDate' : '2022-01-01'}")); + + orgunitB = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Pizza District', 'shortName': 'Pizza District', 'openingDate' : '2022-01-01', " + + "'parent': {'id' : '" + + orgunitA + + "'}}")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsTrailingSpacesControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsTrailingSpacesControllerTest.java new file mode 100644 index 000000000000..04048d97d4c7 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsTrailingSpacesControllerTest.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Set; +import org.hisp.dhis.jsontree.JsonResponse; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.organisationunit.OrganisationUnitService; +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Tests for organisation units with trailing spaces. Currently, the API should trim trailing spaces + * from organisation units but this may still be an issue with legacy databases.* {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_trailing_spaces.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityOrganisationUnitsTrailingSpacesControllerTest + extends AbstractDataIntegrityIntegrationTest { + @Autowired private OrganisationUnitService orgUnitService; + + private static final String unitAName = "Space District "; + + private static final String unitBName = "Spaced Out District"; + + private static final String check = "orgunits_trailing_spaces"; + + private static final String detailsIdType = "organisationUnits"; + + @Test + void DataIntegrityOrganisationUnitsTrailingSpacesTest() { + + OrganisationUnit unitA = createOrganisationUnit('A'); + unitA.setName(unitAName); + unitA.setShortName(unitAName); + unitA.setOpeningDate(getDate("2022-01-01")); + orgUnitService.addOrganisationUnit(unitA); + + OrganisationUnit unitB = createOrganisationUnit('B'); + unitB.setName(unitBName); + unitB.setShortName(unitBName + " "); + unitB.setOpeningDate(getDate("2022-01-01")); + orgUnitService.addOrganisationUnit(unitB); + + OrganisationUnit unitC = createOrganisationUnit('C'); + unitC.setName("NoSpaceDistrict"); + unitC.setShortName("NoSpaceDistrict"); + unitC.setOpeningDate(getDate("2022-01-01")); + orgUnitService.addOrganisationUnit(unitC); + dbmsManager.clearSession(); + + JsonResponse json_unitA = + GET("/organisationUnits/" + unitA.getUid()).content().as(JsonResponse.class); + assertEquals(unitAName, json_unitA.getString("name").string()); + + Set orgUnitUIDs = Set.of(unitA.getUid(), unitB.getUid()); + Set orgunitNames = Set.of(unitA.getName(), unitB.getName()); + + assertHasDataIntegrityIssues( + detailsIdType, check, 66, orgUnitUIDs, orgunitNames, Set.of(), true); + } + + @Test + void orgunitsNoTrailingSpaces() { + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'NospaceDistrict', 'shortName': 'NospaceDistrict', 'openingDate' : '2022-01-01'}")); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testOrgunitsTrailingSpacesZeroCase() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrphanedOrganisationUnitControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrphanedOrganisationUnitControllerTest.java new file mode 100644 index 000000000000..1fbf0fc1b916 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrphanedOrganisationUnitControllerTest.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Checks for orphaned organisation units, namely those which are not part of any hierarchy. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_orphaned.yaml + * } + * + * @author Jason P. Pickering + */ +class DataIntegrityOrphanedOrganisationUnitControllerTest + extends AbstractDataIntegrityIntegrationTest { + + private String orgunitA; + + private static final String check = "orgunits_orphaned"; + + private static final String detailsIdType = "organisationUnits"; + + @Test + void testOrphanedOrganisationUnits() { + + orgunitA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Fish District', 'shortName': 'Fish District', 'openingDate' : '2022-01-01'}")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Pizza District', 'shortName': 'Pizza District', 'openingDate' : '2022-01-01', " + + "'parent': {'id' : '" + + orgunitA + + "'}}")); + + /* Create the orphaned organisation unit */ + String orgunitC = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Cupcake District', 'shortName': 'Cupcake District', 'openingDate' : '2022-01-01'}")); + + assertHasDataIntegrityIssues( + detailsIdType, check, 33, orgunitC, "Cupcake District", null, true); + } + + @Test + void testNotOrphanedOrganisationUnits() { + orgunitA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Fish District', 'shortName': 'Fish District', 'openingDate' : '2022-01-01'}")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Pizza District', 'shortName': 'Pizza District', 'openingDate' : '2022-01-01', " + + "'parent': {'id' : '" + + orgunitA + + "'}}")); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testOrphansZeroCase() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityPeriodsDistantPastControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityPeriodsDistantPastControllerTest.java new file mode 100644 index 000000000000..68aff0da60be --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityPeriodsDistantPastControllerTest.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import java.time.ZonedDateTime; +import java.util.Date; +import org.hisp.dhis.period.MonthlyPeriodType; +import org.hisp.dhis.period.Period; +import org.hisp.dhis.period.PeriodService; +import org.hisp.dhis.period.PeriodType; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Test the metadata check for periods which have the same period type and which have the same start + * and end date. The test scenario is not possible to recreate in current versions of DHIS2 because + * of a unique constraint placed on the period type, start date and end date. Here, we will only + * test that the check actually runs.* {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_distant_past.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityPeriodsDistantPastControllerTest extends AbstractDataIntegrityIntegrationTest { + @Autowired private PeriodService periodService; + + private static final String check = "periods_distant_past"; + + @Test + void testPeriodsInFarFutureExist() { + + PeriodType periodType = new MonthlyPeriodType(); + Date date_past = Date.from(ZonedDateTime.now().minusYears(30).minusMonths(3).toInstant()); + Period periodA = periodType.createPeriod(date_past); + + Date date_now = Date.from(ZonedDateTime.now().toInstant()); + Period periodB = periodType.createPeriod(date_now); + + periodService.addPeriod(periodA); + periodService.addPeriod(periodB); + dbmsManager.clearSession(); + + assertHasDataIntegrityIssues("periods", check, 50, (String) null, null, null, true); + } + + @Test + void testPeriodsInFarFutureDoNotExist() { + + PeriodType periodType = new MonthlyPeriodType(); + Date date_future = Date.from(ZonedDateTime.now().minusYears(10).minusDays(1).toInstant()); + Period periodA = periodType.createPeriod(date_future); + + Date date_now = Date.from(ZonedDateTime.now().toInstant()); + Period periodB = periodType.createPeriod(date_now); + + periodService.addPeriod(periodA); + periodService.addPeriod(periodB); + dbmsManager.clearSession(); + assertHasNoDataIntegrityIssues("periods", check, true); + } + + @Test + void testPeriodsInFutureRuns() { + assertHasNoDataIntegrityIssues("periods", check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityPeriodsSameStartEndDateControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityPeriodsSameStartEndDateControllerTest.java new file mode 100644 index 000000000000..0a343727701e --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityPeriodsSameStartEndDateControllerTest.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import java.time.ZonedDateTime; +import java.util.Date; +import org.hisp.dhis.period.*; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Test the metadata check for periods which have the same period type and which have the same start + * and end date. The test scenario is not possible to recreate in current versions of DHIS2 because + * of a unique constraint placed on the period type, start date and end date. Here, we will only + * test that the check actually runs. + * + *

{@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_same_start_end_date.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityPeriodsSameStartEndDateControllerTest + extends AbstractDataIntegrityIntegrationTest { + @Autowired private PeriodService periodService; + + private static final String check = "periods_same_start_end_date"; + + @Test + void testNoBadPeriodsExist() { + + PeriodType periodType = new MonthlyPeriodType(); + Date date_future = Date.from(ZonedDateTime.now().plusYears(1).plusDays(1).toInstant()); + Period periodA = periodType.createPeriod(date_future); + periodService.addPeriod(periodA); + + Date date_past = Date.from(ZonedDateTime.now().minusMonths(5).toInstant()); + Period periodC = periodType.createPeriod(date_past); + periodService.addPeriod(periodC); + dbmsManager.clearSession(); + + assertHasNoDataIntegrityIssues("periods", check, true); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityPeriodsThreeYearFutureControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityPeriodsThreeYearFutureControllerTest.java new file mode 100644 index 000000000000..a892e46758cc --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityPeriodsThreeYearFutureControllerTest.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import java.time.ZonedDateTime; +import java.util.Date; +import org.hisp.dhis.period.MonthlyPeriodType; +import org.hisp.dhis.period.Period; +import org.hisp.dhis.period.PeriodService; +import org.hisp.dhis.period.PeriodType; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Test the metadata check for periods which have the same period type and which have the same start + * and end date. The test scenario is not possible to recreate in current versions of DHIS2 because + * of a unique constraint placed on the period type, start date and end date. Here, we will only + * test that the check actually runs.* {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_same_start_end_date.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityPeriodsThreeYearFutureControllerTest + extends AbstractDataIntegrityIntegrationTest { + @Autowired private PeriodService periodService; + + private static final String check = "periods_3y_future"; + + @Test + void testPeriodsInFarFutureExist() { + + PeriodType periodType = new MonthlyPeriodType(); + Date threeYearsFromNow = Date.from(ZonedDateTime.now().plusYears(3).plusDays(1).toInstant()); + Period periodA = periodType.createPeriod(threeYearsFromNow); + + Date date_now = Date.from(ZonedDateTime.now().toInstant()); + Period periodB = periodType.createPeriod(date_now); + + periodService.addPeriod(periodA); + periodService.addPeriod(periodB); + dbmsManager.clearSession(); + + assertHasDataIntegrityIssues("periods", check, 50, (String) null, null, null, true); + } + + @Test + void testPeriodsInFarFutureDoNotExist() { + + PeriodType periodType = new MonthlyPeriodType(); + Date oneYearFromNow = Date.from(ZonedDateTime.now().plusYears(1).plusDays(1).toInstant()); + Period periodA = periodType.createPeriod(oneYearFromNow); + + Date date_now = Date.from(ZonedDateTime.now().toInstant()); + Period periodB = periodType.createPeriod(date_now); + + periodService.addPeriod(periodA); + periodService.addPeriod(periodB); + dbmsManager.clearSession(); + + assertHasNoDataIntegrityIssues("periods", check, true); + } + + @Test + void testPeriodsInFutureRuns() { + assertHasNoDataIntegrityIssues("periods", check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityProgramRulesControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityProgramRulesControllerTest.java new file mode 100644 index 000000000000..1500486b1b09 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityProgramRulesControllerTest.java @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import org.hisp.dhis.program.Program; +import org.hisp.dhis.program.ProgramService; +import org.hisp.dhis.program.ProgramStage; +import org.hisp.dhis.program.ProgramStageService; +import org.hisp.dhis.program.ProgramType; +import org.hisp.dhis.programrule.ProgramRule; +import org.hisp.dhis.programrule.ProgramRuleAction; +import org.hisp.dhis.programrule.ProgramRuleActionType; +import org.hisp.dhis.programrule.ProgramRuleService; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Combined integrity test for program rules. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/} + * + * @author Jason P. Pickering + */ +class DataIntegrityProgramRulesControllerTest extends AbstractDataIntegrityIntegrationTest { + + @Autowired private ProgramService programService; + + @Autowired private ProgramRuleService programRuleService; + + @Autowired private ProgramStageService programStageService; + + private ProgramRule programRuleA; + + private ProgramStage programStageA; + + private static final String detailsIdType = "programRules"; + + @Test + void testProgramRuleNoAction() { + + setUpTest(); + + programStageService.saveProgramStage(programStageA); + programRuleService.addProgramRule(programRuleA); + + dbmsManager.clearSession(); + + assertHasDataIntegrityIssues( + detailsIdType, + "program_rules_no_action", + 100, + programRuleA.getUid(), + programRuleA.getName(), + null, + true); + assertHasDataIntegrityIssues( + detailsIdType, + "program_rules_no_expression", + 100, + programRuleA.getUid(), + programRuleA.getName(), + null, + true); + } + + @Test + void testProgramRuleChecksRun() { + assertHasNoDataIntegrityIssues(detailsIdType, "program_rules_no_action", false); + assertHasNoDataIntegrityIssues(detailsIdType, "program_rules_no_expression", false); + assertHasNoDataIntegrityIssues(detailsIdType, "program_rules_message_no_template", false); + } + + @Test + void testProgramRuleMessageTemplate() { + + setUpTest(); + + ProgramRuleAction programRuleAction = new ProgramRuleAction(); + programRuleAction.setAutoFields(); + programRuleAction.setName("Rule Action A"); + programRuleAction.setProgramRuleActionType(ProgramRuleActionType.SENDMESSAGE); + programRuleA.getProgramRuleActions().add(programRuleAction); + programStageService.saveProgramStage(programStageA); + programRuleService.addProgramRule(programRuleA); + + dbmsManager.clearSession(); + + assertHasDataIntegrityIssues( + detailsIdType, + "program_rules_message_no_template", + 100, + programRuleA.getUid(), + programRuleA.getName(), + null, + true); + } + + public void setUpTest() { + + Program programA = new Program(); + programA.setAutoFields(); + programA.setName("Program A"); + programA.setShortName("Program A"); + programA.setProgramType(ProgramType.WITHOUT_REGISTRATION); + programA.setCategoryCombo(categoryService.getCategoryCombo(getDefaultCatCombo())); + programService.addProgram(programA); + + programStageA = new ProgramStage(); + programStageA.setAutoFields(); + programStageA.setName("programStageA"); + programStageA.setProgram(programA); + + programRuleA = new ProgramRule(); + programRuleA.setAutoFields(); + programRuleA.setName("ProgramRuleA"); + programRuleA.setProgram(programA); + programRuleA.setProgramStage(programStageA); + programA.getProgramStages().add(programStageA); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegritySharedCategoryOptionsInCatComboControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegritySharedCategoryOptionsInCatComboControllerTest.java new file mode 100644 index 000000000000..06ce37022a98 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegritySharedCategoryOptionsInCatComboControllerTest.java @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Tests metadata integrity check for category options which are shared between two or more + * categories within a category combination. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_shared_category_options_in_combo.yaml + * } + * + * @author Jason P. Pickering + */ +class DataIntegritySharedCategoryOptionsInCatComboControllerTest + extends AbstractDataIntegrityIntegrationTest { + private static final String check = "category_options_shared_within_category_combo"; + + private static final String detailsIdType = "categoryCombos"; + + private String categoryColor; + + private String categoryTaste; + + private String categoryOptionSour; + + private String categoryOptionRed; + + private String testCatCombo; + + private String categoryOptionUnknown; + + @Test + void testSharedCategoryOptionsInCatCombo() { + + setUpTest(); + + categoryColor = + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Color', 'shortName': 'Color', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'}, " + + "{'id' : '" + + categoryOptionUnknown + + "'}] }")); + + categoryTaste = + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Taste', 'shortName': 'Taste', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionSour + + "'}," + + "{'id' : '" + + categoryOptionUnknown + + "'}]}")); + + testCatCombo = + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryCombos", + "{ 'name' : 'Taste and color', " + + "'dataDimensionType' : 'DISAGGREGATION', 'categories' : [" + + "{'id' : '" + + categoryColor + + "'} , {'id' : '" + + categoryTaste + + "'}]} ")); + + assertHasDataIntegrityIssues( + detailsIdType, check, 100, testCatCombo, "Taste and color", null, true); + } + + @Test + void testCategoryOptionsNotDuplicatedInCatCombo() { + + setUpTest(); + + categoryColor = + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Color', 'shortName': 'Color', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'}] }")); + + categoryTaste = + assertStatus( + HttpStatus.CREATED, + POST( + "/categories", + "{ 'name': 'Taste', 'shortName': 'Taste', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionSour + + "'}," + + "{'id' : '" + + categoryOptionUnknown + + "'}] }")); + + testCatCombo = + assertStatus( + HttpStatus.CREATED, + POST( + "/categoryCombos", + "{ 'name' : 'Taste and color', " + + "'dataDimensionType' : 'DISAGGREGATION', 'categories' : [" + + "{'id' : '" + + categoryColor + + "'} , {'id' : '" + + categoryTaste + + "'}]} ")); + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + void setUpTest() { + + categoryOptionSour = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Sour', 'shortName': 'Sour' }")); + + categoryOptionRed = + assertStatus( + HttpStatus.CREATED, POST("/categoryOptions", "{ 'name': 'Red', 'shortName': 'Red' }")); + + categoryOptionUnknown = + assertStatus( + HttpStatus.CREATED, + POST("/categoryOptions", "{ 'name': 'Unknown', 'shortName': 'Unknown' }")); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityValidationRulesMissingStrategyControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityValidationRulesMissingStrategyControllerTest.java new file mode 100644 index 000000000000..47ace4023903 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityValidationRulesMissingStrategyControllerTest.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Test for visualizations which have not been viewed in the past year. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/visualizations_not_used_1year.yaml + * } + * + * @implNote The API and service layer will coerce a missing value strategy to a non-null type, thus + * it is not possible to create a test for this situation. + * @author Jason P. Pickering + */ +class DataIntegrityValidationRulesMissingStrategyControllerTest + extends AbstractDataIntegrityIntegrationTest { + + private static final String check = "validation_rules_missing_value_strategy_null"; + + private static final String detailsIdType = "validationRules"; + + @Test + void testValidationRulesWithNoStrategyExist() { + + assertStatus( + HttpStatus.CREATED, + POST( + "/validationRules", + "{'importance':'MEDIUM','operator':'not_equal_to','leftSide':{'missingValueStrategy':'NEVER_SKIP', " + + "" + + "'description':'Test','expression':'#{FTRrcoaog83.qk6n4eMAdtK}'}," + + "'rightSide':{'missingValueStrategy': 'NEVER_SKIP', 'description':'Test2'," + + "'expression':'#{FTRrcoaog83.sqGRzCziswD}'},'periodType':'Monthly','name':'Test rule'}")); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testValidationRulesMissingStrategyRuns() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } +} diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityVisualizationNotUsedOneYearControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityVisualizationNotUsedOneYearControllerTest.java new file mode 100644 index 000000000000..1bd1e19c73b4 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityVisualizationNotUsedOneYearControllerTest.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import java.time.ZonedDateTime; +import java.util.Date; +import org.hisp.dhis.datastatistics.DataStatisticsEvent; +import org.hisp.dhis.datastatistics.DataStatisticsEventStore; +import org.hisp.dhis.datastatistics.DataStatisticsEventType; +import org.hisp.dhis.visualization.Visualization; +import org.hisp.dhis.visualization.VisualizationService; +import org.hisp.dhis.visualization.VisualizationType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Test for visualizations which have not been viewed in the past year. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/visualizations_not_used_1year.yaml + * } + * + * @author Jason P. Pickering + */ +class DataIntegrityVisualizationNotUsedOneYearControllerTest + extends AbstractDataIntegrityIntegrationTest { + + @Autowired private DataStatisticsEventStore dataStatisticsEventStore; + + @Autowired private VisualizationService visualizationService; + + private DataStatisticsEvent dse1; + + private Visualization viz; + + private static final String check = "visualizations_not_viewed_one_year"; + + private static final String detailsIdType = "visualizations"; + + private static final String viz_uid = "YngaQVeOC44"; + + @Test + void testUnusedVisualizationsExist() { + + Date oneYearAgo = Date.from(ZonedDateTime.now().minusYears(1).minusDays(1).toInstant()); + dse1 = + new DataStatisticsEvent( + DataStatisticsEventType.VISUALIZATION_VIEW, oneYearAgo, "TestUser", viz.getUid()); + dataStatisticsEventStore.save(dse1); + dbmsManager.clearSession(); + + assertHasDataIntegrityIssues(detailsIdType, check, 100, viz.getUid(), "myviz", null, true); + } + + @Test + void testUsedVisualizationsExist() { + + long millis = System.currentTimeMillis(); + Date date = new Date(millis); + + dse1 = + new DataStatisticsEvent( + DataStatisticsEventType.VISUALIZATION_VIEW, date, "TestUser", viz.getUid()); + dataStatisticsEventStore.save(dse1); + + dbmsManager.clearSession(); + + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testUnusedVisualizationsRuns() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } + + @BeforeEach + void setUp() { + viz = new Visualization("myviz"); + viz.setUid(viz_uid); + viz.setType(VisualizationType.SINGLE_VALUE); + visualizationService.save(viz); + } +} diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataIntegrityController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataIntegrityController.java index c881860056d3..fa1942f7b701 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataIntegrityController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataIntegrityController.java @@ -27,15 +27,18 @@ */ package org.hisp.dhis.webapi.controller; +import static java.lang.System.currentTimeMillis; import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toUnmodifiableSet; +import static org.hisp.dhis.commons.collection.CollectionUtils.isEmpty; import static org.hisp.dhis.dxf2.webmessage.WebMessageUtils.conflict; import static org.hisp.dhis.dxf2.webmessage.WebMessageUtils.jobConfigurationReport; import java.util.Collection; -import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; import lombok.AllArgsConstructor; import org.hisp.dhis.common.DhisApiVersion; import org.hisp.dhis.dataintegrity.DataIntegrityCheck; @@ -44,8 +47,10 @@ import org.hisp.dhis.dataintegrity.DataIntegritySummary; import org.hisp.dhis.dxf2.webmessage.WebMessage; import org.hisp.dhis.scheduling.JobConfiguration; +import org.hisp.dhis.scheduling.JobParameters; import org.hisp.dhis.scheduling.JobType; import org.hisp.dhis.scheduling.SchedulingManager; +import org.hisp.dhis.scheduling.parameters.DataIntegrityDetailsJobParameters; import org.hisp.dhis.scheduling.parameters.DataIntegrityJobParameters; import org.hisp.dhis.scheduling.parameters.DataIntegrityJobParameters.DataIntegrityReportType; import org.hisp.dhis.user.CurrentUser; @@ -56,6 +61,7 @@ import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; @@ -68,87 +74,124 @@ @ApiVersion({DhisApiVersion.DEFAULT, DhisApiVersion.ALL}) @AllArgsConstructor public class DataIntegrityController { - private final SchedulingManager schedulingManager; + private final SchedulingManager schedulingManager; private final DataIntegrityService dataIntegrityService; @PreAuthorize("hasRole('ALL') or hasRole('F_PERFORM_MAINTENANCE')") @PostMapping @ResponseBody public WebMessage runDataIntegrity( - @RequestParam(required = false) List checks, @CurrentUser User currentUser) { - return runDataIntegrityAsync( - checks, currentUser, "runDataIntegrity", DataIntegrityReportType.REPORT) - .setLocation("/dataIntegrity/details?checks=" + toChecksList(checks)); + @CheckForNull @RequestParam(required = false) Set checks, + @CheckForNull @RequestBody(required = false) Set checksBody, + @CurrentUser User currentUser) { + Set names = getCheckNames(checksBody, checks); + return runDataIntegrityAsync(names, currentUser, DataIntegrityReportType.REPORT) + .setLocation("/dataIntegrity/details?checks=" + toChecksList(names)); } private WebMessage runDataIntegrityAsync( - Collection checks, - User currentUser, - String description, - DataIntegrityReportType type) { - DataIntegrityJobParameters params = new DataIntegrityJobParameters(); - params.setChecks(toUniformCheckNames(checks)); - params.setType(type); - JobConfiguration config = - new JobConfiguration(description, JobType.DATA_INTEGRITY, null, params, true, true); + @Nonnull Set checks, User currentUser, DataIntegrityReportType type) { + JobType jobType = + type == DataIntegrityReportType.DETAILS + ? JobType.DATA_INTEGRITY_DETAILS + : JobType.DATA_INTEGRITY; + JobParameters params = + type == DataIntegrityReportType.DETAILS + ? new DataIntegrityDetailsJobParameters(checks) + : new DataIntegrityJobParameters(type, checks); + String name = "DATA_INTEGRITY_" + currentTimeMillis(); + JobConfiguration config = new JobConfiguration(name, jobType, null, params, true, true); config.setUserUid(currentUser.getUid()); config.setAutoFields(); if (!schedulingManager.executeNow(config)) { return conflict("Data integrity check is already running"); } + return jobConfigurationReport(config); } @GetMapping @ResponseBody public Collection getAvailableChecks( - @RequestParam(required = false) Set checks, - @RequestParam(required = false) String section) { + @CheckForNull @RequestParam(required = false) Set checks, + @CheckForNull @RequestParam(required = false) String section, + @CheckForNull @RequestParam(required = false) Boolean slow, + @CheckForNull @RequestParam(required = false) Boolean programmatic) { Collection matches = - dataIntegrityService.getDataIntegrityChecks(toUniformCheckNames(checks)); - return section == null || section.isBlank() - ? matches - : matches.stream().filter(check -> section.equals(check.getSection())).collect(toList()); + dataIntegrityService.getDataIntegrityChecks(getCheckNames(checks)); + Predicate filter = check -> true; + if (section != null && !section.isBlank()) filter = check -> section.equals(check.getSection()); + if (slow != null) filter = filter.and(check -> check.isSlow() == slow); + if (programmatic != null) filter = filter.and(check -> check.isProgrammatic() == programmatic); + return matches.stream().filter(filter).collect(toList()); + } + + @GetMapping("/summary/running") + @ResponseBody + public Set getRunningSummaryChecks() { + return dataIntegrityService.getRunningSummaryChecks(); + } + + @GetMapping("/summary/completed") + @ResponseBody + public Set getCompletedSummaryChecks() { + return dataIntegrityService.getCompletedSummaryChecks(); } @PreAuthorize("hasRole('ALL') or hasRole('F_PERFORM_MAINTENANCE')") @GetMapping("/summary") @ResponseBody public Map getSummaries( - @RequestParam(required = false) Set checks, + @CheckForNull @RequestParam(required = false) Set checks, @RequestParam(required = false, defaultValue = "0") long timeout) { - return dataIntegrityService.getSummaries(toUniformCheckNames(checks), timeout); + return dataIntegrityService.getSummaries(getCheckNames(checks), timeout); } @PreAuthorize("hasRole('ALL') or hasRole('F_PERFORM_MAINTENANCE')") @PostMapping("/summary") @ResponseBody public WebMessage runSummariesCheck( - @RequestParam(required = false) Set checks, @CurrentUser User currentUser) { - return runDataIntegrityAsync( - checks, currentUser, "runSummariesCheck", DataIntegrityReportType.SUMMARY) - .setLocation("/dataIntegrity/summary?checks=" + toChecksList(checks)); + @CheckForNull @RequestParam(required = false) Set checks, + @CheckForNull @RequestBody(required = false) Set checksBody, + @CurrentUser User currentUser) { + Set names = getCheckNames(checksBody, checks); + return runDataIntegrityAsync(names, currentUser, DataIntegrityReportType.SUMMARY) + .setLocation("/dataIntegrity/summary?checks=" + toChecksList(names)); + } + + @GetMapping("/details/running") + @ResponseBody + public Set getRunningDetailsChecks() { + return dataIntegrityService.getRunningDetailsChecks(); + } + + @GetMapping("/details/completed") + @ResponseBody + public Set getCompletedDetailsChecks() { + return dataIntegrityService.getCompletedDetailsChecks(); } @PreAuthorize("hasRole('ALL') or hasRole('F_PERFORM_MAINTENANCE')") @GetMapping("/details") @ResponseBody public Map getDetails( - @RequestParam(required = false) Set checks, + @CheckForNull @RequestParam(required = false) Set checks, @RequestParam(required = false, defaultValue = "0") long timeout) { - return dataIntegrityService.getDetails(toUniformCheckNames(checks), timeout); + return dataIntegrityService.getDetails(getCheckNames(checks), timeout); } @PreAuthorize("hasRole('ALL') or hasRole('F_PERFORM_MAINTENANCE')") @PostMapping("/details") @ResponseBody public WebMessage runDetailsCheck( - @RequestParam(required = false) Set checks, @CurrentUser User currentUser) { - return runDataIntegrityAsync( - checks, currentUser, "runDetailsCheck", DataIntegrityReportType.DETAILS) - .setLocation("/dataIntegrity/details?checks=" + toChecksList(checks)); + @CheckForNull @RequestParam(required = false) Set checks, + @RequestBody(required = false) Set checksBody, + @CurrentUser User currentUser) { + Set names = getCheckNames(checksBody, checks); + return runDataIntegrityAsync(names, currentUser, DataIntegrityReportType.DETAILS) + .setLocation("/dataIntegrity/details?checks=" + toChecksList(names)); } @PreAuthorize("hasRole('ALL') or hasRole('F_PERFORM_MAINTENANCE')") @@ -157,8 +200,9 @@ public WebMessage runDetailsCheck( public DataIntegritySummary getSummary( @PathVariable String check, @RequestParam(required = false, defaultValue = "0") long timeout) { - Set checks = toUniformCheckNames(Set.of(check)); - return dataIntegrityService.getSummaries(checks, timeout).get(checks.iterator().next()); + Collection summaries = + dataIntegrityService.getSummaries(Set.of(check), timeout).values(); + return summaries.isEmpty() ? null : summaries.iterator().next(); } @PreAuthorize("hasRole('ALL') or hasRole('F_PERFORM_MAINTENANCE')") @@ -167,15 +211,20 @@ public DataIntegritySummary getSummary( public DataIntegrityDetails getDetails( @PathVariable String check, @RequestParam(required = false, defaultValue = "0") long timeout) { - Set checks = toUniformCheckNames(Set.of(check)); - return dataIntegrityService.getDetails(checks, timeout).get(checks.iterator().next()); + Collection details = + dataIntegrityService.getDetails(Set.of(check), timeout).values(); + return details.isEmpty() ? null : details.iterator().next(); } - /** Allow both dash or underscore in the API */ - private static Set toUniformCheckNames(Collection checks) { - return checks == null - ? Set.of() - : checks.stream().map(check -> check.replace('-', '_')).collect(toUnmodifiableSet()); + @SafeVarargs + @Nonnull + private static Set getCheckNames(Set... checks) { + for (Set names : checks) { + if (!isEmpty(names)) { + return names; + } + } + return Set.of(); } private String toChecksList(Collection checks) {