diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheck.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheck.java index 85bddcfab9eb..71feb0af1b6e 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheck.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheck.java @@ -27,12 +27,16 @@ */ package org.hisp.dhis.dataintegrity; +import static java.util.Comparator.comparingLong; import static java.util.stream.Collectors.joining; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import java.io.Serializable; +import java.util.Comparator; import java.util.function.Function; import java.util.stream.Stream; +import javax.annotation.Nonnull; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; @@ -53,36 +57,58 @@ @Builder @AllArgsConstructor(access = AccessLevel.PRIVATE) public final class DataIntegrityCheck implements Serializable { - @JsonProperty private final String name; - @JsonProperty private final String displayName; + public static final Comparator FAST_TO_SLOW = + comparingLong(DataIntegrityCheck::getExecutionTimeIndicator); + @JsonProperty private final String name; + @JsonProperty private final String displayName; @JsonProperty private final String section; - + @JsonProperty private final int sectionOrder; @JsonProperty private final DataIntegritySeverity severity; - @JsonProperty private final String description; - @JsonProperty private final String introduction; - @JsonProperty private final String recommendation; - @JsonProperty private final String issuesIdType; - @JsonProperty private final boolean isSlow; + @JsonProperty private final boolean isProgrammatic; - @JsonProperty - public String getCode() { - return Stream.of(name.split("_")) - .map(f -> String.valueOf(f.charAt(0)).toUpperCase()) - .collect(joining()); + private long executionTime; + private int executionCount; + + public @JsonProperty Long getAverageExecutionTime() { + return executionTime <= 0L ? null : executionTime / executionCount; } - private final String detailsID; + @JsonIgnore + long getExecutionTimeIndicator() { + Long time = getAverageExecutionTime(); + if (time != null) return time; + return isSlow ? Long.MAX_VALUE : 1000; + } - private final String summaryID; + @JsonProperty + public String getCode() { + return getCodeFromName(name); + } private final transient Function runSummaryCheck; private final transient Function runDetailsCheck; + + public DataIntegrityCheck addExecution(long time) { + executionCount++; + executionTime += time; + return this; + } + + /** + * Method that takes in a name of a {@link DataIntegrityCheck} and converts it to an acronym of + * its name using the first letter from each word e.g. my_data_integrity_check -> MDIC + */ + public static String getCodeFromName(@Nonnull String fullName) { + return Stream.of(fullName.split("_")) + .map(word -> String.valueOf(word.charAt(0)).toUpperCase()) + .collect(joining()); + } } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfiguration.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfiguration.java index 7de3ca849c43..f6031003f2a6 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfiguration.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfiguration.java @@ -45,6 +45,7 @@ import org.hisp.dhis.scheduling.parameters.AggregateDataExchangeJobParameters; import org.hisp.dhis.scheduling.parameters.AnalyticsJobParameters; import org.hisp.dhis.scheduling.parameters.ContinuousAnalyticsJobParameters; +import org.hisp.dhis.scheduling.parameters.DataIntegrityDetailsJobParameters; import org.hisp.dhis.scheduling.parameters.DataIntegrityJobParameters; import org.hisp.dhis.scheduling.parameters.DataSynchronizationJobParameters; import org.hisp.dhis.scheduling.parameters.DisableInactiveUsersJobParameters; @@ -346,6 +347,9 @@ public void setDelay(Integer delay) { value = TrackerTrigramIndexJobParameters.class, name = "TRACKER_SEARCH_OPTIMIZATION"), @JsonSubTypes.Type(value = DataIntegrityJobParameters.class, name = "DATA_INTEGRITY"), + @JsonSubTypes.Type( + value = DataIntegrityDetailsJobParameters.class, + name = "DATA_INTEGRITY_DETAILS"), @JsonSubTypes.Type( value = AggregateDataExchangeJobParameters.class, name = "AGGREGATE_DATA_EXCHANGE"), diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java index 7287807fe418..a2ee29a0b986 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java @@ -31,6 +31,7 @@ import org.hisp.dhis.scheduling.parameters.AggregateDataExchangeJobParameters; import org.hisp.dhis.scheduling.parameters.AnalyticsJobParameters; import org.hisp.dhis.scheduling.parameters.ContinuousAnalyticsJobParameters; +import org.hisp.dhis.scheduling.parameters.DataIntegrityDetailsJobParameters; import org.hisp.dhis.scheduling.parameters.DataIntegrityJobParameters; import org.hisp.dhis.scheduling.parameters.DataSynchronizationJobParameters; import org.hisp.dhis.scheduling.parameters.DisableInactiveUsersJobParameters; @@ -62,6 +63,11 @@ public enum JobType { SchedulingType.CRON, DataIntegrityJobParameters.class, Map.of("checks", "/api/dataIntegrity")), + DATA_INTEGRITY_DETAILS( + true, + SchedulingType.CRON, + DataIntegrityDetailsJobParameters.class, + Map.of("checks", "/api/dataIntegrity")), RESOURCE_TABLE(true), ANALYTICS_TABLE( true, diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityDetailsJobParameters.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityDetailsJobParameters.java new file mode 100644 index 000000000000..fbd9b5010f4d --- /dev/null +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityDetailsJobParameters.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.scheduling.parameters; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import java.util.Set; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.hisp.dhis.common.DxfNamespaces; +import org.hisp.dhis.scheduling.JobParameters; + +/** + * @author Jan Bernitt + */ +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class DataIntegrityDetailsJobParameters implements JobParameters { + + @JsonProperty(required = false) + @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0) + private Set checks; +} diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityJobParameters.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityJobParameters.java index a9261e991f28..3241a55639f6 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityJobParameters.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityJobParameters.java @@ -30,7 +30,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import java.util.Set; +import lombok.AllArgsConstructor; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; import org.hisp.dhis.common.DxfNamespaces; import org.hisp.dhis.scheduling.JobParameters; @@ -40,6 +42,8 @@ */ @Getter @Setter +@NoArgsConstructor +@AllArgsConstructor public class DataIntegrityJobParameters implements JobParameters { public enum DataIntegrityReportType { REPORT, @@ -49,9 +53,9 @@ public enum DataIntegrityReportType { @JsonProperty(required = false) @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0) - private Set checks; + private DataIntegrityReportType type; @JsonProperty(required = false) @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0) - private DataIntegrityReportType type; + private Set checks; } diff --git a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityCheckTest.java b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityCheckTest.java new file mode 100644 index 000000000000..b37394df49a4 --- /dev/null +++ b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityCheckTest.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.dataintegrity; + +import static java.util.stream.Collectors.toList; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import org.junit.jupiter.api.Test; + +/** + * @author Jan Bernitt + */ +class DataIntegrityCheckTest { + + @Test + void testSortingFastToSlow_NoExecutionTime() { + List checks = + List.of( + DataIntegrityCheck.builder().name("a").isSlow(true).build(), + DataIntegrityCheck.builder().name("b").isSlow(false).build(), + DataIntegrityCheck.builder().name("c").isSlow(true).build()); + List actual = + checks.stream().sorted(DataIntegrityCheck.FAST_TO_SLOW).collect(toList()); + assertEquals( + List.of("b", "a", "c"), actual.stream().map(DataIntegrityCheck::getName).collect(toList())); + } + + @Test + void testSortingFastToSlow() { + List checks = + List.of( + DataIntegrityCheck.builder().name("a").isSlow(true).build().addExecution(12345), + DataIntegrityCheck.builder().name("b").isSlow(false).build().addExecution(20), + DataIntegrityCheck.builder().name("c").isSlow(true).build().addExecution(500)); + List actual = + checks.stream().sorted(DataIntegrityCheck.FAST_TO_SLOW).collect(toList()); + assertEquals( + List.of("b", "c", "a"), actual.stream().map(DataIntegrityCheck::getName).collect(toList())); + } + + @Test + void testSortingFastToSlow_MixedExecutionTime() { + List checks = + List.of( + DataIntegrityCheck.builder().name("a").isSlow(true).build().addExecution(12345), + DataIntegrityCheck.builder().name("b").isSlow(false).build(), + DataIntegrityCheck.builder().name("c").isSlow(true).build().addExecution(500)); + List actual = + checks.stream().sorted(DataIntegrityCheck.FAST_TO_SLOW).collect(toList()); + assertEquals( + List.of("c", "b", "a"), actual.stream().map(DataIntegrityCheck::getName).collect(toList())); + } + + @Test + void testAddExecution() { + DataIntegrityCheck check = DataIntegrityCheck.builder().name("a").build(); + + check.addExecution(1000).addExecution(1000); // sum is 2000, 2 times + + assertEquals(1000L, check.getAverageExecutionTime()); + + check.addExecution(4000); // sum is now 6000, 3 times + + assertEquals(2000L, check.getAverageExecutionTime()); + } +} diff --git a/dhis-2/dhis-services/dhis-service-administration/pom.xml b/dhis-2/dhis-services/dhis-service-administration/pom.xml index 4012b7b8bb3e..4ed705f7b315 100644 --- a/dhis-2/dhis-services/dhis-service-administration/pom.xml +++ b/dhis-2/dhis-services/dhis-service-administration/pom.xml @@ -135,6 +135,21 @@ com.google.code.findbugs jsr305 + + com.networknt + json-schema-validator + 1.0.87 + + + org.apache.commons + commons-lang3 + + + + + org.apache.commons + commons-collections4 + diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReader.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReader.java index 8418f42a6e02..c1a6ced00264 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReader.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReader.java @@ -29,15 +29,20 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.ValidationMessage; +import java.io.InputStream; import java.nio.file.Path; import java.util.List; -import java.util.function.BinaryOperator; -import java.util.function.Consumer; -import java.util.function.Function; +import java.util.Set; import lombok.extern.slf4j.Slf4j; +import org.hisp.dhis.jsonschema.JsonSchemaValidator; +import org.springframework.core.io.AbstractFileResolvingResource; import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.UrlResource; /** * Reads {@link DataIntegrityCheck}s from YAML files. @@ -62,6 +67,9 @@ static class CheckYamlFile { @JsonProperty String section; + @JsonProperty("section_order") + int sectionOrder; + @JsonProperty("summary_sql") String summarySql; @@ -72,68 +80,161 @@ static class CheckYamlFile { String detailsIdType; @JsonProperty("is_slow") - Boolean isSlow; + boolean isSlow; @JsonProperty String introduction; @JsonProperty String recommendation; - @JsonProperty("details_uid") - String detailsID; - - @JsonProperty("summary_uid") - String summaryID; - @JsonProperty DataIntegritySeverity severity; } public static void readDataIntegrityYaml( - String listFile, - Consumer adder, - BinaryOperator info, - Function> sqlToSummary, - Function> sqlToDetails) { - + DefaultDataIntegrityService.DataIntegrityRecord dataIntegrityRecord) { ObjectMapper yaml = new ObjectMapper(new YAMLFactory()); ListYamlFile file; - try { - file = yaml.readValue(new ClassPathResource(listFile).getInputStream(), ListYamlFile.class); + + AbstractFileResolvingResource resource = + getResourceFromType( + dataIntegrityRecord.resourceLocation(), dataIntegrityRecord.yamlFileChecks()); + + if (resource == null) { + log.warn( + "Failed to get resource from location `{}` and with file`{}`", + dataIntegrityRecord.resourceLocation(), + dataIntegrityRecord.yamlFileChecks()); + return; + } + + try (InputStream is = resource.getInputStream()) { + file = yaml.readValue(is, ListYamlFile.class); } catch (Exception ex) { - log.warn("Failed to load data integrity checks from YAML", ex); + log.warn( + "Failed to load data integrity check from YAML. Error message `{}`", ex.getMessage()); return; } - for (String checkFile : file.checks) { - try { - String path = - Path.of(listFile) - .resolve("..") - .resolve("data-integrity-checks") - .resolve(checkFile) - .toString(); - CheckYamlFile e = - yaml.readValue(new ClassPathResource(path).getInputStream(), CheckYamlFile.class); - - String name = e.name.trim(); - adder.accept( + + file.checks.forEach( + dataIntegrityCheckFile -> processFile(dataIntegrityCheckFile, yaml, dataIntegrityRecord)); + } + + /** + * This method processes a {@link DataIntegrityCheck} file by: + * + *
    + *
  1. resolving the file path + *
  2. converting file to {@link JsonNode} + *
  3. validate JsonNode against {@link JsonSchema} + *
  4. Then either + *
      + *
    1. add {@link DataIntegrityCheck} to map if no validation errors or + *
    2. log a warning if any validation errors and do not add {@link DataIntegrityCheck} to + * map + *
    + *
+ * + * @param dataIntegrityCheckFile the yaml file version of a {@link DataIntegrityCheck} + * @param yaml object mapper + * @param dataIntegrityRecord record used for storing {@link DataIntegrityCheck} details used for + * processing + */ + private static void processFile( + String dataIntegrityCheckFile, + ObjectMapper yaml, + DefaultDataIntegrityService.DataIntegrityRecord dataIntegrityRecord) { + + try { + String path = + Path.of(dataIntegrityRecord.yamlFileChecks()) + .resolve("..") + .resolve(dataIntegrityRecord.checksDir()) + .resolve(dataIntegrityCheckFile) + .toString(); + + AbstractFileResolvingResource resource = + getResourceFromType(dataIntegrityRecord.resourceLocation(), path); + + if (resource == null) { + log.warn( + "Failed to get resource from location `{}` and with file`{}`", + dataIntegrityRecord.resourceLocation(), + path); + return; + } + + try (InputStream is = resource.getInputStream()) { + + JsonNode jsonNode = yaml.readValue(is, JsonNode.class); + Set validationMessages = + JsonSchemaValidator.validateDataIntegrityCheck(jsonNode); + + if (validationMessages.isEmpty()) { + CheckYamlFile yamlFile = yaml.convertValue(jsonNode, CheckYamlFile.class); + acceptDataIntegrityCheck(dataIntegrityRecord, yamlFile); + } else { + log.warn( + "JsonSchema validation errors found for Data Integrity Check `{}`. Errors: {}", + dataIntegrityCheckFile, + validationMessages); + } + } + } catch (Exception ex) { + log.error( + "Failed to load data integrity check `{}` with error message {}", + dataIntegrityCheckFile, + ex.getMessage()); + } + } + + private static void acceptDataIntegrityCheck( + DefaultDataIntegrityService.DataIntegrityRecord dataIntegrityRecord, CheckYamlFile yamlFile) { + String name = yamlFile.name.trim(); + dataIntegrityRecord + .adder() + .accept( DataIntegrityCheck.builder() .name(name) - .displayName(info.apply(name + ".name", name.replace('_', ' '))) - .description(info.apply(name + ".description", trim(e.description))) - .introduction(info.apply(name + ".introduction", trim(e.introduction))) - .recommendation(info.apply(name + ".recommendation", trim(e.recommendation))) - .issuesIdType(trim(e.detailsIdType)) - .section(trim(e.section)) - .severity(e.severity) - .isSlow(e.isSlow != null && e.isSlow) - .detailsID(e.detailsID) - .summaryID(e.summaryID) - .runSummaryCheck(sqlToSummary.apply(sanitiseSQL(e.summarySql))) - .runDetailsCheck(sqlToDetails.apply(sanitiseSQL(e.detailsSql))) + .displayName( + dataIntegrityRecord.info().apply(name + ".name", name.replace('_', ' '))) + .description( + dataIntegrityRecord + .info() + .apply(name + ".description", trim(yamlFile.description))) + .introduction( + dataIntegrityRecord + .info() + .apply(name + ".introduction", trim(yamlFile.introduction))) + .recommendation( + dataIntegrityRecord + .info() + .apply(name + ".recommendation", trim(yamlFile.recommendation))) + .issuesIdType(trim(yamlFile.detailsIdType)) + .section(trim(yamlFile.section)) + .sectionOrder(yamlFile.sectionOrder) + .severity(yamlFile.severity) + .isSlow(yamlFile.isSlow) + .runSummaryCheck( + dataIntegrityRecord.sqlToSummary().apply(sanitiseSQL(yamlFile.summarySql))) + .runDetailsCheck( + dataIntegrityRecord.sqlToDetails().apply(sanitiseSQL(yamlFile.detailsSql))) .build()); - } catch (Exception ex) { - log.error("Failed to load data integrity check " + checkFile, ex); - } + } + + private static AbstractFileResolvingResource getResourceFromType( + ResourceLocation resourceLocation, String filePath) { + AbstractFileResolvingResource resource = null; + try { + resource = + resourceLocation == ResourceLocation.CLASS_PATH + ? new ClassPathResource(filePath) + : new UrlResource("file://" + filePath); + } catch (Exception ex) { + log.warn( + "Failed to load data integrity checks from YAML file path `{}`. Error message: {}", + filePath, + ex.getMessage()); } + return resource; } private static String trim(String str) { @@ -152,4 +253,9 @@ private static String sanitiseSQL(String sql) { .replace("::varchar", "") .replace("|| '%'", "")); } + + enum ResourceLocation { + CLASS_PATH, + FILE_SYSTEM + } } diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java index 732288c7ee1b..d52e878348ef 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2022, University of Oslo + * Copyright (c) 2004-2023, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -33,12 +33,13 @@ import static java.util.function.Predicate.not; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toUnmodifiableList; import static java.util.stream.Collectors.toUnmodifiableSet; import static java.util.stream.StreamSupport.stream; import static org.hisp.dhis.commons.collection.ListUtils.getDuplicates; import static org.hisp.dhis.dataintegrity.DataIntegrityDetails.DataIntegrityIssue.toIssue; import static org.hisp.dhis.dataintegrity.DataIntegrityDetails.DataIntegrityIssue.toRefsList; +import static org.hisp.dhis.dataintegrity.DataIntegrityYamlReader.ResourceLocation.CLASS_PATH; +import static org.hisp.dhis.dataintegrity.DataIntegrityYamlReader.ResourceLocation.FILE_SYSTEM; import static org.hisp.dhis.dataintegrity.DataIntegrityYamlReader.readDataIntegrityYaml; import static org.hisp.dhis.expression.ParseType.INDICATOR_EXPRESSION; import static org.hisp.dhis.expression.ParseType.VALIDATION_RULE_EXPRESSION; @@ -59,6 +60,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BinaryOperator; +import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -67,7 +69,10 @@ import javax.annotation.Nonnull; import javax.annotation.PostConstruct; import lombok.RequiredArgsConstructor; +import lombok.Value; +import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.hisp.dhis.antlr.ParserException; import org.hisp.dhis.cache.Cache; import org.hisp.dhis.cache.CacheProvider; @@ -84,6 +89,8 @@ import org.hisp.dhis.expression.Expression; import org.hisp.dhis.expression.ExpressionService; import org.hisp.dhis.expression.ExpressionValidationOutcome; +import org.hisp.dhis.external.location.LocationManager; +import org.hisp.dhis.external.location.LocationManagerException; import org.hisp.dhis.i18n.I18n; import org.hisp.dhis.i18n.I18nManager; import org.hisp.dhis.indicator.Indicator; @@ -125,6 +132,8 @@ public class DefaultDataIntegrityService implements DataIntegrityService { private final I18nManager i18nManager; + private final LocationManager locationManager; + private final ProgramRuleService programRuleService; private final ProgramRuleActionService programRuleActionService; @@ -182,7 +191,7 @@ private static List toSimpleIssueList( return items .map(DataIntegrityIssue::toIssue) .sorted(DefaultDataIntegrityService::alphabeticalOrder) - .collect(toUnmodifiableList()); + .collect(toList()); } private static List toIssueList( @@ -190,7 +199,7 @@ private static List toIssueLi return items .map(e -> DataIntegrityIssue.toIssue(e, toRefs.apply(e))) .sorted(DefaultDataIntegrityService::alphabeticalOrder) - .collect(toUnmodifiableList()); + .collect(toList()); } @Nonnull @@ -421,7 +430,7 @@ List getDuplicatePeriods() { null, group.getValue().stream() .map(p -> p.toString() + ":" + p.getUid()) - .collect(toUnmodifiableList()))); + .collect(toList()))); } } return issues; @@ -487,7 +496,6 @@ private List getInvalidValidationRuleExpressions( issues.add(toIssue(rule, i18n.getString(result.getKey()))); } } - return issues; } @@ -504,11 +512,13 @@ private void registerNonDatabaseIntegrityCheck( try { Schema issueSchema = issueIdType == null ? null : schemaService.getDynamicSchema(issueIdType); String issueIdTypeName = issueSchema == null ? null : issueSchema.getPlural(); - checksByName.put( + checksByName.putIfAbsent( name, DataIntegrityCheck.builder() .name(name) .displayName(info.apply("name", name.replace('_', ' '))) + .isSlow(true) + .isProgrammatic(true) .severity( DataIntegritySeverity.valueOf( info.apply("severity", DataIntegritySeverity.WARNING.name()).toUpperCase())) @@ -826,7 +836,7 @@ private static List groupActionsByProgramRule( List groupBy(Function property, Collection values) { return values.stream().collect(groupingBy(property)).entrySet().stream() .map(e -> DataIntegrityIssue.toIssue(e.getKey(), e.getValue())) - .collect(toUnmodifiableList()); + .collect(toList()); } /* @@ -938,7 +948,10 @@ private void runDataIntegrityChecks( progress.startingProcess("Data Integrity check"); progress.startingStage(stageDesc, checks.size(), SKIP_ITEM); progress.runStage( - checks.stream().map(checksByName::get).filter(Objects::nonNull), + checks.stream() + .map(checksByName::get) + .filter(Objects::nonNull) + .sorted(DataIntegrityCheck.FAST_TO_SLOW), DataIntegrityCheck::getDescription, check -> { Date startTime = new Date(); @@ -952,6 +965,7 @@ private void runDataIntegrityChecks( running.remove(check.getName()); } if (res != null) { + check.addExecution(currentTimeMillis() - startTime.getTime()); cache.put(check.getName(), res); } }); @@ -964,7 +978,7 @@ private void runDataIntegrityChecks( private Set expandChecks(Set names) { ensureConfigurationsAreLoaded(); - if (names == null || names.isEmpty()) { + if (CollectionUtils.isEmpty(names)) { return getDefaultChecks(); } Set expanded = new LinkedHashSet<>(); @@ -977,9 +991,13 @@ private Set expandChecks(Set names) { .map(DataIntegrityCheck::getName) .forEach(expanded::add); } else if (name.contains("*")) { - String pattern = name.toLowerCase().replace('-', '_').replace("*", ".*"); + String pattern = + name.toLowerCase() + .replace('-', '_') // make uniform + .replaceAll("[^*_a-z0-9]+", "") // sanitise against regex attacks + .replace("*", ".*"); // expand regex wildcard match for (DataIntegrityCheck check : checksByName.values()) { - if (!check.isSlow() && check.getName().matches(pattern)) { + if (check.getName().matches(pattern)) { expanded.add(check.getName()); } } @@ -1001,18 +1019,82 @@ private Set getDefaultChecks() { private void ensureConfigurationsAreLoaded() { if (configurationsAreLoaded.compareAndSet(false, true)) { + // load system-packaged data integrity checks + loadChecks(CLASS_PATH, "data-integrity-checks.yaml", "data-integrity-checks"); + // programmatic checks initIntegrityChecks(); - // YAML based checks - I18n i18n = i18nManager.getI18n(DataIntegrityService.class); - readDataIntegrityYaml( - "data-integrity-checks.yaml", - check -> checksByName.put(check.getName(), check), - (property, defaultValue) -> - i18n.getString(format("data_integrity.%s", property), defaultValue), - sql -> check -> dataIntegrityStore.querySummary(check, sql), - sql -> check -> dataIntegrityStore.queryDetails(check, sql)); + // load user-packaged custom data integrity checks + try { + String dhis2Home = locationManager.getExternalDirectoryPath(); + loadChecks( + FILE_SYSTEM, + dhis2Home + "/custom-data-integrity-checks.yaml", + dhis2Home + "/custom-data-integrity-checks"); + } catch (LocationManagerException ex) { + log.warn( + "Could not get DHIS2_HOME external directory. No custom data integrity checks loaded."); + } } } + + private void loadChecks( + DataIntegrityYamlReader.ResourceLocation resourceLocation, + String yamlFileChecks, + String checksDir) { + I18n i18n = i18nManager.getI18n(DataIntegrityService.class); + readDataIntegrityYaml( + new DataIntegrityRecord( + resourceLocation, + yamlFileChecks, + checksDir, + addToChecks, + (property, defaultValue) -> + i18n.getString(format("data_integrity.%s", property), defaultValue), + sql -> check -> dataIntegrityStore.querySummary(check, sql), + sql -> check -> dataIntegrityStore.queryDetails(check, sql))); + } + + /** + * Consumer that adds a {@link DataIntegrityCheck} to a map. It only adds the {@link + * DataIntegrityCheck} to the map if: + * + *
    + *
  1. the {@link DataIntegrityCheck} code is unique + *
  2. the map does not already have a key with the same {@link DataIntegrityCheck} name + *
+ */ + private final Consumer addToChecks = + check -> { + String checkCode = DataIntegrityCheck.getCodeFromName(check.getName()); + Set checkCodes = + checksByName.keySet().stream() + .map(DataIntegrityCheck::getCodeFromName) + .collect(Collectors.toSet()); + if (!checkCodes.contains(checkCode)) { + DataIntegrityCheck dataIntegrityCheck = checksByName.putIfAbsent(check.getName(), check); + if (dataIntegrityCheck != null) { + log.warn( + "Data Integrity Check `{}` not added as a check with that name already exists", + check.getName()); + } + } else + log.warn( + "Data Integrity Check `{}` not added as a check with the code `{}` already exists", + check.getName(), + check.getCode()); + }; + + @Value + @Accessors(fluent = true) + static class DataIntegrityRecord { + DataIntegrityYamlReader.ResourceLocation resourceLocation; + String yamlFileChecks; + String checksDir; + Consumer adder; + BinaryOperator info; + Function> sqlToSummary; + Function> sqlToDetails; + } } diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityDetailsJob.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityDetailsJob.java new file mode 100644 index 000000000000..d6ed5cfbd70b --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityDetailsJob.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.dataintegrity.jobs; + +import java.util.Set; +import lombok.AllArgsConstructor; +import org.hisp.dhis.dataintegrity.DataIntegrityService; +import org.hisp.dhis.scheduling.Job; +import org.hisp.dhis.scheduling.JobConfiguration; +import org.hisp.dhis.scheduling.JobProgress; +import org.hisp.dhis.scheduling.JobType; +import org.hisp.dhis.scheduling.parameters.DataIntegrityDetailsJobParameters; +import org.springframework.stereotype.Component; + +/** + * @author Jan Bernitt + */ +@Component +@AllArgsConstructor +public class DataIntegrityDetailsJob implements Job { + + private final DataIntegrityService dataIntegrityService; + + @Override + public JobType getJobType() { + return JobType.DATA_INTEGRITY_DETAILS; + } + + @Override + public void execute(JobConfiguration config, JobProgress progress) { + DataIntegrityDetailsJobParameters parameters = + (DataIntegrityDetailsJobParameters) config.getJobParameters(); + Set checks = parameters == null ? Set.of() : parameters.getChecks(); + + dataIntegrityService.runDetailsChecks(checks, progress); + } +} diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityJob.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityJob.java index 70376595dfa3..9e418a69b1cb 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityJob.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityJob.java @@ -43,7 +43,8 @@ import org.springframework.stereotype.Component; /** - * @author Halvdan Hoem Grelland + * @author Halvdan Hoem Grelland (original) + * @author Jan Bernitt (refactored) */ @Component @AllArgsConstructor diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/jsonschema/JsonSchemaValidator.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/jsonschema/JsonSchemaValidator.java new file mode 100644 index 000000000000..27f6c13ad656 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/jsonschema/JsonSchemaValidator.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.jsonschema; + +import com.fasterxml.jackson.databind.JsonNode; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.JsonSchemaFactory; +import com.networknt.schema.SpecVersion; +import com.networknt.schema.ValidationMessage; +import java.io.IOException; +import java.io.InputStream; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.io.ClassPathResource; + +@Slf4j +public class JsonSchemaValidator { + + private static final String DATA_INTEGRITY_CHECK_DIR = "data-integrity-checks/"; + private static final String DATA_INTEGRITY_CHECK_SCHEMA_FILE = "integrity_check_schema.json"; + private static final String DATA_INTEGRITY_CHECK_SCHEMA = + DATA_INTEGRITY_CHECK_DIR + DATA_INTEGRITY_CHECK_SCHEMA_FILE; + private static JsonSchema dataIntegritySchema; + + static { + JsonSchemaFactory schemaFactory = + JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012); + try (InputStream is = new ClassPathResource(DATA_INTEGRITY_CHECK_SCHEMA).getInputStream()) { + dataIntegritySchema = schemaFactory.getSchema(is); + } catch (IOException e) { + log.error( + "Error loading data integrity check schema at class path location {}. Error message: {}", + DATA_INTEGRITY_CHECK_SCHEMA, + e.getMessage()); + } + } + + private JsonSchemaValidator() { + throw new UnsupportedOperationException("util"); + } + + public static Set validateDataIntegrityCheck(JsonNode json) { + return dataIntegritySchema.validate(json); + } +} diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/dashboards_empty.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/dashboards_empty.yaml index f4ad44db760a..512ad4485bad 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/dashboards_empty.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/dashboards_empty.yaml @@ -29,13 +29,11 @@ description: Dashboards with no items. section: Dashboards section_order: 4 - summary_uid: mQ8XfW10fOQ summary_sql: >- select count(*) as value, 100*count(*) / NULLIF( (select count(*) from dashboard),0) as percent from dashboard where dashboardid not in (select dashboardid from dashboard_items); - details_uid: NMZhvSZpTtW details_sql: >- SELECT uid,name from dashboard WHERE dashboardid not in (select dashboardid from dashboard_items); diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/dashboards_not_used_1year.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/dashboards_not_used_1year.yaml index 231ee41bb149..3d3753207b82 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/dashboards_not_used_1year.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/dashboards_not_used_1year.yaml @@ -26,7 +26,6 @@ # --- name: dashboards_not_viewed_one_year - summary_uid: UmKSWH5Id3E description: Dashboards which have not been actively viewed in the past 12 months section: Visualizations section_order: 3 @@ -45,7 +44,6 @@ SELECT COUNT(*) as value, 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM dashboard), 0) as percent from unused_dashboards; - details_uid: eDb3aWioyRQ details_sql: >- SELECT c.uid,c.name, CAST(c.last_viewed as text) from ( SELECT a.uid,a.name,b.last_viewed diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/maps_not_used_1year.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/maps_not_used_1year.yaml index a952c3d44b8b..365184e8716d 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/maps_not_used_1year.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/maps_not_used_1year.yaml @@ -26,7 +26,6 @@ # --- name: maps_not_viewed_one_year - summary_uid: qPDGbYKCZn3 description: Maps which have not been viewed in the past 12 months section: Visualizations section_order: 1 @@ -45,7 +44,6 @@ SELECT COUNT(*) as value, 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM map), 0) as percent from unused_maps; - details_uid: hPSszzUDmSE details_sql: >- SELECT c.uid,c.name, CAST(c.last_viewed as text) from ( SELECT a.uid,a.name,b.last_viewed diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/visualizations_not_used_1year.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/visualizations_not_used_1year.yaml index d0b8a270c488..9dcd774b2319 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/visualizations_not_used_1year.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/analytical_objects/visualizations_not_used_1year.yaml @@ -26,7 +26,6 @@ # --- name: visualizations_not_viewed_one_year - summary_uid: c3ZBCRuYpEO description: Visualizations which have not been viewed in the past 12 months section: Visualizations section_order: 1 @@ -45,7 +44,6 @@ SELECT COUNT(*) as value, 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM visualization), 0) as percent from unused_visualizations; - details_uid: nxpcGQ7h4ey details_sql: >- SELECT c.uid,c.name, CAST(c.last_viewed as text) from ( SELECT a.uid,a.name,b.last_viewed diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_no_options.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_no_options.yaml index 30676714d9bd..2a5960ad6148 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_no_options.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_no_options.yaml @@ -44,8 +44,6 @@ severity: WARNING introduction: > Categories should always have at least one category option. section_order: 1 -summary_uid: YbexVajxsCD -details_uid: ePVVKF1llbU recommendation: > Any categories without category options should either be removed from the system if they are not in use. Otherwise, appropriate category options diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category.yaml index a52416b059f6..a8ea841e03c3 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category.yaml @@ -40,8 +40,6 @@ severity: SEVERE introduction: > There should only exist one category with name and code "default". section_order: 2 -summary_uid: XiEYmQgK5Dd -details_uid: cI5G5z5Yk59 recommendation: > Only the category with UID "GLevLNI9wkl" should be named "default" and have code "default". Either rename the conflicting category or move all references to category "GLevLNI9wkl" diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_combo.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_combo.yaml index 0e660aa859bc..5bb22da2e5bf 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_combo.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_combo.yaml @@ -40,8 +40,6 @@ severity: SEVERE introduction: > There should only exist one category combo with name and code "default". section_order: 3 -summary_uid: rMZ8WxkhBWN -details_uid: B7m7Vn0Go8z recommendation: > Only the category combo with UID "bjDvmb4bfuf" should be named "default" and have code "default". Either rename the conflicting category combo or move all references to category combo "bjDvmb4bfuf" diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option.yaml index e8b76a8214bc..72465253f354 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option.yaml @@ -40,8 +40,6 @@ severity: SEVERE introduction: > There should only exist one category option with name and code "default". section_order: 4 -summary_uid: iDLix1p75AM -details_uid: Q0GuHB94wjd recommendation: > Only the category option with UID "xYerKDKCefk" should be named "default" and have code "default". Either rename the conflicting category option or move all references to category option "xYerKDKCefk" diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option_combo.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option_combo.yaml index cb3bf5210c75..8b5fbdc76d38 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option_combo.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_one_default_category_option_combo.yaml @@ -40,8 +40,6 @@ severity: SEVERE introduction: > There should only exist one category option with name and code "default". section_order: 4 -summary_uid: GmX5ZiyrlCL -details_uid: COP7dnqnfJH recommendation: > Only the category option combo with UID "HllvX50cXC0" should be named "default" and have code "default". Either rename the conflicting category option combo or move all references to category option combo "HllvX50cXC0" diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_same_category_options.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_same_category_options.yaml index 8221c30af406..09bcc3ab03ac 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_same_category_options.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_same_category_options.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- - summary_uid: SkFAjPsMHJc name: categories_same_category_options description: Categories with the same category options section: Categories @@ -42,7 +41,6 @@ 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM dataelementcategory),0 ) percent FROM duplicative_categories; - details_uid: yKInwm9p54H details_sql: >- SELECT x.uid,'(' || b.rank || ') ' || x.name as name from dataelementcategory x INNER JOIN ( diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_shared_category_options_in_combo.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_shared_category_options_in_combo.yaml index cb56fa4fca50..229002d24706 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_shared_category_options_in_combo.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_shared_category_options_in_combo.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: s9rs1qOwn0C name: category_options_shared_within_category_combo description: Category combinations with categories which share the same category options. section: Categories @@ -41,7 +40,6 @@ summary_sql: >- COUNT(*)as value, 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM categoryoptioncombo),0) as percent FROM category_option_multiple_member_category; -details_uid: SqWFiRS39cz details_sql: >- WITH category_option_multiple_member_category as ( select cc.uid, cc.name as cc_name, co.categoryoptionid, co.name as co_name from categorycombo cc diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_unique_category_combo.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_unique_category_combo.yaml index 8de018854fa3..45c5ee2cbece 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_unique_category_combo.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/categories_unique_category_combo.yaml @@ -66,7 +66,5 @@ introduction: > contain the exact same set of categories, this would be considered to be duplicative and potentially confusing to users. section_order: 8 -summary_uid: fAsLcyf2Ea2 -details_uid: EFCJQ8uLFgM recommendation: > One category combo is kept, all references are updated to use this combo and other combos are removed. \ No newline at end of file diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_combos_unused.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_combos_unused.yaml index d192111e27e3..205fe0720607 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_combos_unused.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_combos_unused.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- - summary_uid: yJ73aZmXuz1 name: category_combos_unused description: Category combinations not used by other metadata objects section: Categories @@ -49,7 +48,6 @@ SELECT COUNT(*) as value, 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM categorycombo), 0) as percent FROM unused_category_combo where type IS NULL; - details_uid: mG05UTfKZVz details_sql: >- WITH unused_category_combo as ( SELECT a.uid, a.name, a.created, b.type FROM categorycombo a diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_cardinality.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_cardinality.yaml index 48ab74482eb5..40f3916a4103 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_cardinality.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_cardinality.yaml @@ -24,7 +24,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #--- -summary_uid: VBIs20Adkyf name: cocs_wrong_cardinality description: Category option combinations with incorrect cardinality. section: Categories @@ -43,7 +42,6 @@ summary_sql: >- SELECT COUNT(*) as value, 100.0 * COUNT(*) / NULLIF( ( SELECT COUNT(*) FROM categoryoptioncombo),0) as percent FROM cocs_wrong_cardinality; -details_uid: ZMsFbuKljqn details_sql: >- WITH baz as ( SELECT foo.categorycomboid,foo.categoryoptioncomboid,foo.actual_cardnality,bar.theoretical_cardnality FROM ( diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_disjoint.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_disjoint.yaml index a5170ac136d9..2d34c79447c3 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_disjoint.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_disjoint.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- - summary_uid: ubUZycuJ8VM name: category_option_combos_disjoint description: Category option combinations with disjoint associations. section: Categories @@ -46,7 +45,6 @@ SELECT COUNT(*) as value, 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM categoryoptioncombo), 0) as percent FROM cocs_disjoint_associations; - details_uid: VFqTzfnhQoy details_sql: >- SELECT DISTINCT d.uid, d.name, e.name as comment from categoryoptioncombos_categoryoptions a diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_excess_members.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_excess_members.yaml index f4104602e805..7a865aab41e8 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_excess_members.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_excess_members.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: hFeGvP851tv name: category_options_excess_groupset_membership description: Category options which belong to multiple groups in a category option group set. section: Categories @@ -57,7 +56,6 @@ summary_sql: >- SELECT COUNT(*), 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM dataelementcategoryoption), 0) as percent FROM cos_multiple_groups; -details_uid: ARXenW2aL62 details_sql: >- SELECT x.uid,x.name, x.cogs_name || ' :{'|| string_agg(x.cog_name,',') || '}' as comment from ( diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_sets_incomplete.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_sets_incomplete.yaml index 1826501cd3fb..abc7b2eeb11c 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_sets_incomplete.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_groups_sets_incomplete.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: vTuF0zbwTcd name: category_option_group_sets_incomplete description: Category option group sets which which do not contain all category options. section: Categories @@ -61,7 +60,6 @@ summary_sql: >- 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) from categoryoptiongroupmembers where categoryoptiongroupid in (SELECT categoryoptiongroupid from categoryoptiongroupsetmembers)), 0) as percent FROM cat_option_group_complete; -details_uid: xrA4Mhz6yjH details_sql: >- SELECT cogs.uid,cogs.name , cats.name || ':{' || opt.name || '}' as comment FROM ( diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_options_no_categories.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_options_no_categories.yaml index 69cc9950639e..26f1ecb871b0 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_options_no_categories.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_options_no_categories.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- - summary_uid: O1VwQPDJZCy name: category_options_no_categories description: Category options with no categories. section: Categories @@ -41,7 +40,6 @@ 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM dataelementcategoryoption), 0 ) as percent FROM category_options_no_categories; - details_uid: Fp9jUG2txCV details_sql: >- SELECT uid,name FROM dataelementcategoryoption WHERE categoryoptionid diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/coc_no_category_combo.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/coc_no_category_combo.yaml index 55c1efaae9c1..8b0a941450cf 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/coc_no_category_combo.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/coc_no_category_combo.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- - summary_uid: hNesjuZiLI7 name: catoptioncombos_no_catcombo description: Category options combinations with no category combination. section: Categories @@ -39,7 +38,6 @@ 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM categoryoptioncombo), 0 ) percent FROM catoptioncombos_no_catcombo; - details_uid: JVz8Rh6xgaB details_sql: >- SELECT uid,name FROM categoryoptioncombo WHERE categoryoptioncomboid NOT IN (SELECT categoryoptioncomboid diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_abandoned.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_abandoned.yaml index 39b4964e359b..37e925da78d1 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_abandoned.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_abandoned.yaml @@ -29,7 +29,6 @@ description: Aggregate data elements that have not been changed in last 100 days and do not have any data values. section: Data elements (aggregate) section_order: 5 - summary_uid: HA8SfI5Ww6L summary_sql: >- SELECT COUNT(*) as value, 100*COUNT(*) / NULLIF( ( select COUNT(*) from dataelement where domaintype = 'AGGREGATE'), 0) as percent @@ -37,7 +36,6 @@ and dataelementid not in (select dataelementid from datavalue group by dataelementid) and AGE( now(), lastupdated) > INTERVAL '100 days'; - details_uid: eBfvtecrvqU details_sql: >- select uid,name,lastupdated from dataelement where domaintype = 'AGGREGATE' diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_datasets_different_period_types.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_datasets_different_period_types.yaml index b3c4c06f691e..207283092b0b 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_datasets_different_period_types.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_datasets_different_period_types.yaml @@ -29,7 +29,6 @@ description: Aggregate data elements which belong to datasets with different period types. section: Data elements (aggregate) section_order: 7 - summary_uid: DnXxP2Nfn6i summary_sql: >- WITH des_different_periodtypes as ( SELECT uid,dataelement_name as name, @@ -58,7 +57,6 @@ SELECT COUNT(*) as value, 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM dataelement), 0) as percent FROM des_different_periodtypes; - details_uid: N45TSlWpBwu details_sql: >- SELECT uid,dataelement_name as name, STRING_AGG(dataset_name, ';') as comment from ( diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_excess_groupset_membership.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_excess_groupset_membership.yaml index cd9221d79088..7c1b1b70668d 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_excess_groupset_membership.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_excess_groupset_membership.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: kjZFYBcH2L2 name: data_elements_excess_groupset_membership description: Data elements which belong to multiple groups in a group set. section: data_elements @@ -57,7 +56,6 @@ summary_sql: >- SELECT COUNT(*), 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM dataelement), 0) as percent FROM des_multiple_groups; -details_uid: TcSqU3CY5mG details_sql: >- SELECT x.uid,x.name, x.degs_name || ' :{'|| string_agg(x.deg_name,',') || '}' as comment from ( diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_inconsistent_agg_operator.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_inconsistent_agg_operator.yaml index ab22dc76fe56..a8fe12d60682 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_inconsistent_agg_operator.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_inconsistent_agg_operator.yaml @@ -29,7 +29,6 @@ description: Non-numeric data elements which have an aggregation operator other than NONE. section: Data elements (aggregate) section_order: 8 - summary_uid: kaVLxU71NCX summary_sql: >- with des_aggregation_op as ( SELECT uid,name,valuetype,aggregationtype from dataelement @@ -48,7 +47,6 @@ SELECT COUNT(*) as value, 100*COUNT(*) / NULLIF( ( SELECT COUNT(*) from dataelement), 0 ) as percent from des_aggregation_op; - details_uid: BYmPyQUj12q details_sql: >- SELECT uid,name,valuetype || ' (' || aggregationtype || ')' as comment FROM dataelement diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_no_analysis.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_no_analysis.yaml index 8259d0ac3b73..522ed12105f4 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_no_analysis.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_no_analysis.yaml @@ -29,7 +29,6 @@ description: Aggregate data elements not used in any favourites (directly or through indicators) section: Data elements (aggregate) section_order: 2 - summary_uid: lY1DmLxMl7h summary_sql: >- WITH des_no_analysis AS ( select name,uid from dataelement where domaintype = 'AGGREGATE' and @@ -44,7 +43,6 @@ SELECT COUNT(*) as value, 100 * COUNT(*) / NULLIF( ( SELECT COUNT(*) FROM dataelement WHERE domaintype = 'AGGREGATE' ),0) as percent FROM des_no_analysis; - details_uid: EFyNHwRu9w8 details_sql: >- select uid,name from dataelement where domaintype = 'AGGREGATE' and dataelementid not in (select de.dataelementid from dataelement de diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_no_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_no_groups.yaml index 73e736f2a5de..69ea171d92f4 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_no_groups.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_no_groups.yaml @@ -29,13 +29,11 @@ description: Aggregate data elements not in any data element groups. section: Data elements (aggregate) section_order: 4 - summary_uid: f4YhSfEAbHN summary_sql: >- SELECT COUNT(*) as value, 100*COUNT(*)/ NULLIF( (SELECT COUNT(*) from dataelement where domaintype = 'AGGREGATE'), 0) as percent from dataelement where domaintype = 'AGGREGATE' and dataelementid not in (select dataelementid from dataelementgroupmembers); - details_uid: bOxvtqRYGXh details_sql: >- select uid,name from dataelement where domaintype = 'AGGREGATE' diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_nodata.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_nodata.yaml index 182b2f636c24..1449c2c88075 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_nodata.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/data_elements/aggregate_des_nodata.yaml @@ -29,7 +29,6 @@ name: data_elements_aggregate_no_data description: Aggregate data elements with NO data values. section: Data elements (aggregate) section_order: 6 -summary_uid: yc9MsMnAGCj summary_sql: >- WITH dataelements_no_data AS ( SELECT uid,name from dataelement where dataelementid in ( @@ -42,7 +41,6 @@ summary_sql: >- 100*COUNT(*)/ NULLIF( ( select count(*) from dataelement where domaintype = 'AGGREGATE'), 0) as percent from dataelements_no_data; -details_uid: MzPQlUsApVO details_sql: >- SELECT uid,name from dataelement where dataelementid in ( SELECT dataelementid from dataelement diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/datasets/datasets_empty.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/datasets/datasets_empty.yaml index 2e9cefe01d2c..f80a9d615d7c 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/datasets/datasets_empty.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/datasets/datasets_empty.yaml @@ -29,13 +29,11 @@ name: datasets_empty description: Datasets with no data elements. section: Data sets section_order: 1 -summary_uid: sLnHsNHH6ha summary_sql: >- select count(*) as value, 100*count(*) / NULLIF( (select count(*) from dataset),0) as percent from dataset where datasetid NOT IN ( SELECT datasetid from datasetelement); -details_uid: ISwQ9oYQBmp details_sql: >- SELECT uid,name from dataset WHERE datasetid NOT IN (SELECT datasetid from datasetelement); diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_category_option_group_sets.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_category_option_group_sets.yaml index d300c08bf057..60e1eb4293ba 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_category_option_group_sets.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_category_option_group_sets.yaml @@ -29,7 +29,6 @@ name: category_option_group_sets_scarce description: Category option groups should have at least two members. section: Group size section_order: 8 -summary_uid: So63EA8N4BX summary_sql: >- WITH group_size as ( SELECT uid,name, COALESCE(x.count,0) as count from @@ -42,7 +41,6 @@ summary_sql: >- SELECT COUNT(*) as value, 100*COUNT(*) / NULLIF( ( select COUNT(*) from categoryoptiongroupset), 0) as percent from group_size where count < 2; -details_uid: sChquZNGbUJ details_sql: >- WITH group_size as ( SELECT uid,name, COALESCE(x.count,0) as count from diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_category_option_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_category_option_groups.yaml index 23d17efa7429..72f545c64a80 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_category_option_groups.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_category_option_groups.yaml @@ -29,7 +29,6 @@ name: category_option_groups_scarce description: Category option groups should have at least two members. section: Group size section_order: 8 -summary_uid: sJf2PDB8eBy summary_sql: >- WITH group_size as ( SELECT uid,name, COALESCE(x.count,0) as count from @@ -42,7 +41,6 @@ summary_sql: >- SELECT COUNT(*) as value, 100*COUNT(*) / NULLIF( ( select COUNT(*) from categoryoptiongroup), 0) as percent from group_size where count < 2; -details_uid: Yp7F6tXw3w2 details_sql: >- WITH group_size as ( SELECT uid,name, COALESCE(x.count,0) as count from diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_data_element_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_data_element_groups.yaml index a5ba837e363c..2de76cb19d99 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_data_element_groups.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_data_element_groups.yaml @@ -29,7 +29,6 @@ name: data_element_groups_scarce description: Data element groups should have at least two members. section: Group size section_order: 1 -summary_uid: sup7m0e5Ngc summary_sql: >- WITH group_size as ( SELECT uid,name, 'dataElementGroups' as comment, COALESCE(x.count,0) as count from @@ -42,7 +41,6 @@ summary_sql: >- SELECT COUNT(*) as value, 100*COUNT(*) / NULLIF( ( select COUNT(*) from dataelementgroup), 0) as percent from group_size WHERE count < 2; -details_uid: b5e4mmhTjQA details_sql: >- WITH group_size as ( SELECT uid,name, 'dataElementGroups' as comment, COALESCE(x.count,0) as count from diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_indicator_group_sets.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_indicator_group_sets.yaml index 6707a18f076d..758f5baaccd9 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_indicator_group_sets.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_indicator_group_sets.yaml @@ -29,7 +29,6 @@ name: indicator_group_sets_scarce description: Indicator groups sets should have at least two members. section: Group size section_order: 2 -summary_uid: dkxYkOAgUbS summary_sql: >- WITH group_size as ( SELECT uid,name, COALESCE(x.count,0) as count from @@ -42,7 +41,6 @@ summary_sql: >- SELECT COUNT(*) as value, 100*COUNT(*) / NULLIF( ( select COUNT(*) from indicatorgroupset), 0) as percent from group_size where count < 2; -details_uid: zbWRFdq5Sjq details_sql: >- WITH group_size as ( SELECT uid,name, COALESCE(x.count,0) as count from diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_indicator_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_indicator_groups.yaml index f456303f3663..323c02f812ea 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_indicator_groups.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_indicator_groups.yaml @@ -29,7 +29,6 @@ name: indicator_groups_scarce description: Indicator groups should have at least two members. section: Group size section_order: 2 -summary_uid: aQngw1JlEo7 summary_sql: >- WITH group_size as ( SELECT uid,name, COALESCE(x.count,0) as count from @@ -42,7 +41,6 @@ summary_sql: >- SELECT COUNT(*) as value, 100*COUNT(*) / NULLIF( ( select COUNT(*) from indicatorgroup), 0) as percent from group_size where count < 2; -details_uid: mjQxyQbpkED details_sql: >- WITH group_size as ( SELECT uid,name, COALESCE(x.count,0) as count from diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_organisation_unit_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_organisation_unit_groups.yaml index f8c671dcfbde..44da9b02f4cf 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_organisation_unit_groups.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_organisation_unit_groups.yaml @@ -29,7 +29,6 @@ name: orgunit_groups_scarce description: Organisation unit groups should have at least two members. section: Group size section_order: 1 -summary_uid: yfA2yeS6nfP summary_sql: >- WITH group_size as ( SELECT uid,name, COALESCE(x.count,0) as count from @@ -42,7 +41,6 @@ summary_sql: >- SELECT COUNT(*) as value, 100*COUNT(*) / NULLIF( ( select COUNT(*) from orgunitgroup), 0) as percent from group_size where count < 2; -details_uid: Ettes6bOAOq details_sql: >- WITH group_size as ( SELECT uid,name, COALESCE(x.count,0) as count from diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_program_indicator_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_program_indicator_groups.yaml index 1727c109aa04..023d3df66d32 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_program_indicator_groups.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_program_indicator_groups.yaml @@ -29,7 +29,6 @@ name: program_indicator_groups_scarce description: Program indicator groups should have at least two members. section: Group size section_order: 3 -summary_uid: iOZtfxbXjoo summary_sql: >- WITH group_size as ( SELECT uid,name, COALESCE(x.count,0) as count from @@ -43,7 +42,6 @@ summary_sql: >- SELECT COUNT(*) as value, 100*COUNT(*) / NULLIF( ( select COUNT(*) from programindicatorgroup), 0) as percent from group_size where count < 2; -details_uid: rmPoiiAjHNV details_sql: >- WITH group_size as ( SELECT uid,name, COALESCE(x.count,0) as count from diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_user_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_user_groups.yaml index e1699ed51afa..de6dd47ad955 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_user_groups.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_user_groups.yaml @@ -29,7 +29,6 @@ name: user_groups_scarce description: User groups should have at least two members. section: Group size section_order: 9 -summary_uid: UoQRMf3RMMW summary_sql: >- WITH group_size as ( SELECT uid,name, COALESCE(x.count,0) as count from @@ -42,7 +41,6 @@ summary_sql: >- SELECT COUNT(*) as value, 100*COUNT(*) / NULLIF( ( select COUNT(*) from usergroup), 0) as percent from group_size where count < 2; -details_uid: me8PS8d2QSm details_sql: >- WITH group_size as ( SELECT uid,name, COALESCE(x.count,0) as count from diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_validation_rule_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_validation_rule_groups.yaml index 1e8659c71e9d..115b7e684b78 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_validation_rule_groups.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/groups/group_size_validation_rule_groups.yaml @@ -29,7 +29,6 @@ name: validation_rule_groups_scarce description: Validation rule should have at least two members. section: Group size section_order: 2 -summary_uid: adbASuPTWMW summary_sql: >- WITH group_size as ( SELECT uid,name, COALESCE(x.count,0) as count from @@ -43,7 +42,6 @@ summary_sql: >- SELECT COUNT(*) as value, 100*COUNT(*) / NULLIF( ( select COUNT(*) from validationrulegroup), 0) as percent from group_size where count < 2; -details_uid: fXJHZMwuGOh details_sql: >- WITH group_size as ( SELECT uid,name, COALESCE(x.count,0) as count from diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_duplicate_types.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_duplicate_types.yaml index 8e7d117b3416..748a83a01d8a 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_duplicate_types.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_duplicate_types.yaml @@ -29,7 +29,6 @@ name: indicator_types_duplicated description: Indicator types with the same factor. section: Indicators section_order: 5 -summary_uid: ZQkHZ64zNc7 summary_sql: >- WITH duplicate_indicator_factors AS ( SELECT uid,name from indicatortype where @@ -41,7 +40,6 @@ summary_sql: >- select count(*) as value, 100 * COUNT(*) / NULLIF( ( SELECT COUNT(*) FROM indicatortype) ,0) as percent FROM duplicate_indicator_factors; -details_uid: RkNRuWIjT3z details_sql: >- SELECT uid,name from indicatortype where indicatorfactor IN ( diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_duplicated_terms.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_duplicated_terms.yaml index 28167ce6877a..c3197ad6e250 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_duplicated_terms.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_duplicated_terms.yaml @@ -29,7 +29,6 @@ name: indicators_duplicated_terms description: Indicators with the same terms. section: Indicators section_order: 7 -summary_uid: gsOyEd4aHT6 summary_sql: >- WITH duplicated_indicators as ( SELECT unnest(indicatorids) from ( @@ -47,7 +46,6 @@ summary_sql: >- SELECT COUNT(*) as count, 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM indicator), 0) as percent from duplicated_indicators; -details_uid: oqHwQcfFw1W details_sql: >- WITH duplicated_indicators as ( SELECT a.indicatorid, diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_exact_duplicates.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_exact_duplicates.yaml index 52985353cf88..75050df4ed00 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_exact_duplicates.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_exact_duplicates.yaml @@ -29,7 +29,6 @@ name: indicators_exact_duplicates description: Indicators with the same formula. section: Indicators section_order: 8 -summary_uid: PTCwRotFe2R summary_sql: >- WITH duplicated_indicators as ( SELECT uid,name, numerator || '/' || denominator as comment from indicator where indicatorid in ( @@ -43,7 +42,6 @@ summary_sql: >- SELECT COUNT(*) as count, 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM indicator), 0) as percent from duplicated_indicators; -details_uid: B8ZTxmkQ0Cq details_sql: >- SELECT uid,name, numerator || '/' || denominator as comment from indicator where indicatorid in ( SELECT unnest(indicators) as indicatorid from ( diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_noanalysis.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_noanalysis.yaml index 0809ae3e75b1..44ac880a69e3 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_noanalysis.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_noanalysis.yaml @@ -29,7 +29,6 @@ name: indicator_no_analysis description: Indicators not used in analytical objects. section: Indicators section_order: 4 -summary_uid: sgBuVm7KhJC summary_sql: >- select count(*) as value, 100*count(*)/ NULLIF( (select count(*) from indicator),0) as percent @@ -37,7 +36,6 @@ summary_sql: >- where indicatorid not in (SELECT DISTINCT indicatorid from datadimensionitem where indicatorid IS NOT NULL); -details_uid: zqKrnvP2CMp details_sql: >- SELECT uid,name FROM indicator diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_nongrouped.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_nongrouped.yaml index 03712de42e88..a49078552bda 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_nongrouped.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/indicators/indicator_nongrouped.yaml @@ -29,13 +29,11 @@ name: indicators_not_grouped description: Indicators not in any groups. section: Indicators section_order: 2 -summary_uid: yh9JEFsml2C summary_sql: >- select count(*) as value, 100*count(*)/ NULLIF( (select count(*) from indicator), 0) as percent from indicator where indicatorid not in (select indicatorid from indicatorgroupmembers); -details_uid: Is7g6eFibfG details_sql: >- SELECT uid,name FROM indicator diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/integrity_check_schema.json b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/integrity_check_schema.json index c3e0c79e2a18..f855c7a684e8 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/integrity_check_schema.json +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/integrity_check_schema.json @@ -1,5 +1,5 @@ { - "$schema":"http://json-schema.org/draft-04/schema#", + "$schema":"http://json-schema.org/draft/2020-12/schema", "$id": "https://dhis2.org/metadata-assessment.schema.json", "title": "Metadata Assessment", "description": "A DHIS2 metadata assessment object", @@ -9,6 +9,10 @@ "description": "A short and unique name which will represent the name of the set of queries in DHIS2", "type": "string" }, + "description": { + "description": "A description giving information about the data integrity check", + "type": "string" + }, "section" : { "description": "A name which can be used to group related queries.", "type" : "string" @@ -18,19 +22,10 @@ "type": "integer", "minimum": 1 }, - "summary_uid": { - "type": "string", - "description": "A DHIS2 uid used to identify the summary query" - }, "summary_sql": { "description": "An SQL query which should return a single line and consist of four character fields: indicator,value,percent and description", "type": "string" }, - "details_uid":{ - "description": "A DHIS2 uid used to identify the details query", - "type": "string" - - }, "details_sql": { "type" : "string", "description": "A DHIS2 database query which should return all identified objects from this particular issue. Should return at least uid and name" @@ -50,7 +45,6 @@ "introduction": { "description": "This should field describe the issue the queries are seeking to identify.", "type": "string" - }, "recommendation": { "description": "This field should describe the procedure to rectify the problems", @@ -68,16 +62,15 @@ } }, "required": [ + "description", + "details_id_type", + "details_sql", + "introduction", "name", + "recommendation", "section", "section_order", - "summary_uid", - "summary_sql", - "details_uid", - "details_sql", - "details_id_type", "severity", - "introduction", - "recommendation" + "summary_sql" ] } \ No newline at end of file diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/option_sets_empty.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/option_sets_empty.yaml index afd0735dffd8..7e63f394b8a4 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/option_sets_empty.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/option_sets_empty.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: rqwvuIzZcs9 name: options_sets_empty description: Empty option sets section: Option sets @@ -41,7 +40,6 @@ summary_sql: >- COUNT(*) as value, 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM optionset), 0) as percent from options_sets_empty; -details_uid: wjIQ26nrXT8 details_sql: >- SELECT uid,name from optionset where optionsetid diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/option_sets_wrong_sort_order.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/option_sets_wrong_sort_order.yaml index 461251d40fad..b87a14d6f658 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/option_sets_wrong_sort_order.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/option_sets_wrong_sort_order.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: xPFtcXBvQfk name: option_sets_wrong_sort_order description: Option sets with possibly wrong sort order. section: Option sets @@ -44,7 +43,6 @@ summary_sql: >- from option_sets_wrong_sort_order ) as bar) / NULLIF( (SELECT COUNT(*) from optionset), 0) as percent FROM option_sets_wrong_sort_order; -details_uid: NxtEkZcmLl0 details_sql: >- SELECT a.uid, a.name, b.sort_order || ' != ' || b.row_number as comment from optionset a diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/unused_option_sets.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/unused_option_sets.yaml index a0d6c70f6a2b..814e4df89856 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/unused_option_sets.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/option_sets/unused_option_sets.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: X1oWESZRTzY name: options_sets_unused description: Option sets which are not used section: Option sets @@ -41,7 +40,6 @@ summary_sql: >- SELECT COUNT(*) as value, 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM optionset), 0) as percent from options_sets_unused; -details_uid: i4OtFbZmqkp details_sql: >- SELECT DISTINCT uid,name FROM optionset where optionsetid NOT IN ( SELECT DISTINCT optionsetid from attribute where optionsetid IS NOT NULL UNION diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/compulsory_orgunit_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/compulsory_orgunit_groups.yaml index d5956d0fd9b4..79400adc9b81 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/compulsory_orgunit_groups.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/compulsory_orgunit_groups.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: lFkN6LopXcz name: orgunits_compulsory_group_count description: Orgunits that are not in all compulsory orgunit group sets section: Organisation units @@ -50,7 +49,6 @@ summary_sql: >- 100 * COUNT(*)/NULLIF( ( SELECT COUNT(*) FROM organisationunit), 0) as percent FROM ougs_compulsory_members where has_compulsory IS FALSE; -details_uid: HnmKIwXFWy5 details_sql: >- WITH orgunits_not_in_compulsory_groups as ( SELECT ou.uid,ou.name, diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunit_null_island.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunit_null_island.yaml index 26725024afa5..8755a978cab6 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunit_null_island.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunit_null_island.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: fMoHpxuPT18 name: orgunits_null_island description: Organisation units located within 100 km of Null Island (0,0). section: Organisation units @@ -47,7 +46,6 @@ summary_sql: >- COUNT(*) as count, 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM organisationunit), 0) as percent FROM orgunits_null_island; -details_uid: yBAsnuFkmbj details_sql: >- SELECT a.uid,a.name,ST_AsText(a.geometry) as comment from organisationunit a diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunit_open_date_gt_closed_date.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunit_open_date_gt_closed_date.yaml index 07f65f14abfb..cbfed6aaf5e8 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunit_open_date_gt_closed_date.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunit_open_date_gt_closed_date.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: oIK2h41Rc9L name: orgunits_openingdate_gt_closeddate description: Organisation units which have an opening date later than the closed date. section: Organisation units @@ -36,7 +35,6 @@ summary_sql: >- count(*) as count, 100*count(*)/NULLIF((select count(*) from organisationunit), 0) as percent from orgunit_future_opening_date; -details_uid: qGXxCOoeipq details_sql: >- select uid,name from organisationunit where openingdate > closeddate diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_excess_group_memberships.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_excess_group_memberships.yaml index 78a2c8ab0242..fe87a64289b9 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_excess_group_memberships.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_excess_group_memberships.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: zKs07PBTuXM name: orgunit_group_sets_excess_groups description: Organisation units which belong to multiple groups in a group set. section: Organisation units @@ -53,7 +52,6 @@ summary_sql: >- NULLIF((SELECT COUNT(*) from ougs_multiple_groups),0)) as percent from ougs_multiple_groups WHERE array_length(ougs_groups, 1) > 1; -details_uid: dmCuiWwo4KL details_sql: >- WITH ougs_multiple_groups as ( SELECT ou_uid,ou_name,ougs_name, diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_invalid_geometry.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_invalid_geometry.yaml index d9514983c0fa..6ed9972de4a6 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_invalid_geometry.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_invalid_geometry.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: HJoHTRxcaVv name: orgunits_invalid_geometry description: Organisation units with invalid geometry. section: Organisation units @@ -38,7 +37,6 @@ summary_sql: >- COUNT(*) as value, 100.0 * COUNT(*) / NULLIF(( SELECT COUNT(*) FROM organisationunit where geometry IS NOT NULL),0) as percent FROM orgunits_invalid_geometry; -details_uid: lEmJf83tdNR details_sql: >- SELECT uid,name,ST_IsValidReason(geometry) as comment from organisationunit where ST_IsValid(geometry) = FALSE diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_multiple_roots.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_multiple_roots.yaml index 384e59606e19..d0df20dd1da2 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_multiple_roots.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_multiple_roots.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: ezQJxORagqI name: orgunits_multiple_roots description: The organisation unit hierarchy should have a single root. section: Organisation units @@ -41,7 +40,6 @@ summary_sql: >- 100.0 * COUNT(*) / NULLIF( ( SELECT COUNT(*) FROM organisationunit), 0) as percent FROM orgunit_multiple_roots WHERE array_length(comment,1) > 1; -details_uid: CVy8mwGN6Ec details_sql: >- WITH duplicate_roots as ( SELECT uid,name, diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_multiple_spaces.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_multiple_spaces.yaml index 0e0af006ab6c..4bae1195a97b 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_multiple_spaces.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_multiple_spaces.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: BQymdCGR1sz name: orgunits_multiple_spaces description: Organisation units should not have multiple spaces in their names or shortnames. section: Organisation units @@ -39,7 +38,6 @@ summary_sql: >- COUNT(*) as value, 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM organisationunit), 0) as percent from orgunits_multiple_spaces; -details_uid: spwhfsg5Kkn details_sql: >- SELECT uid, name, shortname as comment from organisationunit where name ~('\s{2,}') OR shortname ~('\s{2,}'); diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_no_geometry.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_no_geometry.yaml index f2efa03e5f4c..da3e2b984e6d 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_no_geometry.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_no_geometry.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: KVICQmNL1pc name: orgunits_no_coordinates description: Organisation units with no coordinates. section: Organisation units @@ -38,7 +37,6 @@ summary_sql: >- COUNT(*)as value, 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM organisationunit), 0) as percent FROM orgunits_no_coordinates; -details_uid: O8Hxk6GsUXu details_sql: >- SELECT uid,name, CAST(hierarchylevel as text) as comment from organisationunit where geometry IS NULL diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_not_contained_by_parent.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_not_contained_by_parent.yaml index d77c79419a21..c927a4c914ee 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_not_contained_by_parent.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_not_contained_by_parent.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: IfsKI6Tdz1X name: orgunits_not_contained_by_parent description: Organisation units with point coordinates should be contained by their parent. section: Organisation units @@ -44,7 +43,6 @@ summary_sql: >- 100.0 * COUNT(*) / NULLIF((SELECT COUNT(*) FROM organisationunit WHERE ST_GeometryType(geometry) = 'ST_Point'), 0) as percent FROM parent_contains_point; -details_uid: QC6yuqSTQXa details_sql: >- SELECT a.uid,a.name from organisationunit a INNER JOIN organisationunit b on a.parentid = b.organisationunitid diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_orphaned.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_orphaned.yaml index f58547476e98..b03f7e8bb43a 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_orphaned.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_orphaned.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: dXSwsjWqYd8 name: orgunits_orphaned description: Orphaned organisation units. section: Organisation units @@ -46,7 +45,6 @@ summary_sql: >- COUNT(*) as value, 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM organisationunit), 0) as percent from orgunits_orphaned; -details_uid: wovrWdNIuMb details_sql: >- SELECT uid, name from organisationunit where organisationunitid IN ( diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_same_name_and_parent.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_same_name_and_parent.yaml index 6c9587cdd5fb..965c96accdec 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_same_name_and_parent.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_same_name_and_parent.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: fiXDbSVLunS name: orgunits_same_name_and_parent description: Organisation units should not have the same name and parent. section: Organisation units @@ -49,7 +48,6 @@ summary_sql: >- COUNT(*) as value, 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM organisationunit), 0) as percent from orgunits_same_name_and_parent; -details_uid: Nxb5qywdMYj details_sql: >- SELECT d.uid,d.name,e.name as comment from ( diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_trailing_spaces.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_trailing_spaces.yaml index 506417d6d082..809b25017def 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_trailing_spaces.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_trailing_spaces.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: TrWM0CtEL5N name: orgunits_trailing_spaces description: Organisation units should not have trailing spaces. section: Organisation units @@ -38,7 +37,6 @@ summary_sql: >- COUNT(*) as value, 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM organisationunit), 0) as percent from orgunits_trailing_spaces; -details_uid: N2qHFwW0Yyr details_sql: >- SELECT uid, name, shortname as comment from organisationunit where name ~('\s+$') or shortname ~('\s+$') ORDER BY name; diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_3y_future.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_3y_future.yaml index 108d8350ffef..0ffedb1ffdf8 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_3y_future.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_3y_future.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- - summary_uid: OuOMLbV8AaS name: periods_3y_future description: Periods which are more than three years in the future. section: Periods @@ -36,7 +35,6 @@ SELECT COUNT(*) as value, 100.0* COUNT(*) / NULLIF( (SELECT COUNT(*) FROM period), 0) as percent FROM periods_3y_future; - details_uid: OBtJROkTncz details_sql: >- SELECT NULL as uid, to_char(startdate, 'YYYY-MM-DD') || '-' || to_char(enddate,'YYYY-MM-DD') as name from period where AGE(enddate,now()) >= INTERVAL '3 year' diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_distant_past.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_distant_past.yaml index 1c85cc94f708..8be93651b374 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_distant_past.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_distant_past.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- - summary_uid: VICKLpREFYv name: periods_distant_past description: Periods which are in the distant past. section: Periods @@ -36,7 +35,6 @@ SELECT COUNT(*) as value, 100.0* COUNT(*) / NULLIF( (SELECT COUNT(*) FROM period) , 0) as percent FROM periods_distant_past; - details_uid: wvhK31GYedI details_sql: >- SELECT NULL as uid, to_char(startdate, 'YYYY-MM-DD') || '-' || to_char(enddate, 'YYYY-MM-DD') as name from period where AGE(now(),enddate) >= INTERVAL '30 year'; diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_same_start_end_date.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_same_start_end_date.yaml index 0e8d2ad48056..a69d658f8f44 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_same_start_end_date.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/periods/periods_same_start_end_date.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- - summary_uid: gXmkRzMFQ37 name: periods_same_start_end_date description: Periods with the same start and end dates section: Periods @@ -42,7 +41,6 @@ SELECT COUNT(*) as value, 100.0* COUNT(*) / NULLIF( (SELECT COUNT(*) FROM period), 0) as percent FROM bad_periods; - details_uid: v12ImzFZQCd details_sql: >- select pt.name as uid, CAST( p1.startdate as text) as name diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_message_no_template.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_message_no_template.yaml index d1e62ee63203..9272c62afff4 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_message_no_template.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_message_no_template.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: OHKuZ5NgqT0 name: program_rules_message_no_template description: Program rules actions which should send or schedule a message without a message template. section: Program rules @@ -39,7 +38,6 @@ summary_sql: >- SELECT COUNT(*) as value, 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM programrule),0) as percent FROM program_rules_no_message_template; -details_uid: HRsnqw2VoaS details_sql: >- SELECT uid,name from programrule where programruleid IN ( SELECT programruleid from programruleaction WHERE diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_no_action.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_no_action.yaml index 39a75bb4f494..b29f7648ed3f 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_no_action.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_no_action.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -summary_uid: vDNdEyvMhS9 name: program_rules_no_action description: Program rules with no action. section: Program rules @@ -37,7 +36,6 @@ summary_sql: >- SELECT COUNT(*) as value, 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM programrule),0) as percent FROM program_rules_no_action; -details_uid: oafNmCH85US details_sql: >- SELECT uid,name from programrule where programruleid NOT IN (SELECT DISTINCT programruleid from programruleaction); diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_no_expression.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_no_expression.yaml index 33014996e6c5..e467596d7632 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_no_expression.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/program_rules/program_rules_no_expression.yaml @@ -25,7 +25,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- - summary_uid: xL9ClsJAhx2 name: program_rules_no_expression description: Program rules with no expression. section: Program rules @@ -36,7 +35,6 @@ SELECT COUNT(*) as value, 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM programrule), 0) as percent FROM program_rules_no_expression; - details_uid: tHtxIYabJze details_sql: >- SELECT uid,name from programrule where rulecondition IS NULL; severity: WARNING diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/validation_rules/validation_rules_undefined_missing_value_strategy.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/validation_rules/validation_rules_undefined_missing_value_strategy.yaml index 25e3a7bc84f7..9eed218c9784 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/validation_rules/validation_rules_undefined_missing_value_strategy.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/validation_rules/validation_rules_undefined_missing_value_strategy.yaml @@ -29,12 +29,10 @@ name: validation_rules_missing_value_strategy_null description: All validation rule expressions should have a missing value strategy. section: Validation rules section_order: 1 -summary_uid: r3zB0LV2vbG summary_sql: >- SELECT COUNT(*) as value, 100.0 * count(*) / NULLIF( (select count(*) from expression), 0) as percent from expression where missingvaluestrategy IS NULL; -details_uid: uvj8KSaZHtw details_sql: >- SELECT a.uid,a.name,'RHS' as comment from validationrule a diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityServiceTest.java b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityServiceTest.java index 0846035dd48f..bf5a54541aee 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityServiceTest.java @@ -43,18 +43,21 @@ import static org.hisp.dhis.dataintegrity.DataIntegrityDetails.DataIntegrityIssue.issueName; import static org.hisp.dhis.utils.Assertions.assertContainsOnly; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.clearInvocations; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -72,6 +75,8 @@ import org.hisp.dhis.dataset.DataSet; import org.hisp.dhis.dataset.DataSetService; import org.hisp.dhis.expression.ExpressionService; +import org.hisp.dhis.external.location.DefaultLocationManager; +import org.hisp.dhis.i18n.I18n; import org.hisp.dhis.i18n.I18nManager; import org.hisp.dhis.indicator.Indicator; import org.hisp.dhis.indicator.IndicatorGroup; @@ -100,6 +105,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -113,6 +119,15 @@ class DataIntegrityServiceTest { @Mock private I18nManager i18nManager; + @Mock private I18n i18n; + @Mock private DefaultLocationManager locationManager; + + @Mock private SchemaService schemaService; + + @Mock private CacheProvider cacheProvider; + + @Mock private DataIntegrityStore dataIntegrityStore; + @Mock private DataElementService dataElementService; @Mock private IndicatorService indicatorService; @@ -145,7 +160,7 @@ class DataIntegrityServiceTest { @Mock private ProgramRuleActionService programRuleActionService; - private DefaultDataIntegrityService subject; + @InjectMocks private DefaultDataIntegrityService subject; private DataElementGroup elementGroupA; @@ -197,26 +212,6 @@ class DataIntegrityServiceTest { @BeforeEach public void setUp() { - subject = - new DefaultDataIntegrityService( - i18nManager, - programRuleService, - programRuleActionService, - programRuleVariableService, - dataElementService, - indicatorService, - dataSetService, - organisationUnitService, - organisationUnitGroupService, - validationRuleService, - expressionService, - dataEntryFormService, - categoryService, - periodService, - programIndicatorService, - mock(CacheProvider.class), - mock(DataIntegrityStore.class), - mock(SchemaService.class)); setUpFixtures(); } @@ -614,6 +609,15 @@ void testValidProgramIndicatorFilter() { assertTrue(issues.isEmpty()); } + @Test + void testGetDataIntegrityChecks_ReadDataIntegrityYamlFilesOnClassPath() { + when(i18nManager.getI18n(DataIntegrityService.class)).thenReturn(i18n); + when(i18n.getString(anyString(), anyString())).thenReturn("default"); + when(i18n.getString(contains("severity"), eq("WARNING"))).thenReturn("WARNING"); + Collection dataIntegrityChecks = subject.getDataIntegrityChecks(); + assertFalse(dataIntegrityChecks.isEmpty()); + } + private Map createRandomDataElements(int quantity, String uidSeed) { return IntStream.range(1, quantity + 1) diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java index d084a1dad3ca..86f31d6d5499 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java @@ -28,8 +28,8 @@ package org.hisp.dhis.dataintegrity; import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toUnmodifiableList; -import static org.hisp.dhis.common.CodeGenerator.isValidUid; +import static org.hisp.dhis.dataintegrity.DataIntegrityYamlReader.ResourceLocation.CLASS_PATH; +import static org.hisp.dhis.dataintegrity.DataIntegrityYamlReader.ResourceLocation.FILE_SYSTEM; import static org.hisp.dhis.dataintegrity.DataIntegrityYamlReader.readDataIntegrityYaml; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -41,7 +41,6 @@ import java.util.Set; import java.util.function.Predicate; import java.util.regex.Pattern; -import java.util.stream.Collectors; import java.util.stream.Stream; import org.hisp.dhis.dataintegrity.DataIntegrityDetails.DataIntegrityIssue; import org.junit.jupiter.api.Test; @@ -56,24 +55,11 @@ class DataIntegrityYamlReaderTest { void testReadDataIntegrityYaml() { List checks = new ArrayList<>(); - readDataIntegrityYaml( - "data-integrity-checks.yaml", - checks::add, - (property, defaultValue) -> defaultValue, - sql -> check -> new DataIntegritySummary(check, new Date(), new Date(), null, 1, 100d), - sql -> - check -> - new DataIntegrityDetails( - check, - new Date(), - new Date(), - null, - List.of(new DataIntegrityIssue("id", "name", sql, List.of())))); + readYaml(checks, "data-integrity-checks.yaml", "data-integrity-checks", CLASS_PATH); assertEquals(63, checks.size()); // Names should be unique - List allNames = - checks.stream().map(DataIntegrityCheck::getName).collect(toUnmodifiableList()); + List allNames = checks.stream().map(DataIntegrityCheck::getName).collect(toList()); assertEquals(allNames.size(), Set.copyOf(allNames).size()); // Config checks and Java checks should not have any of the same names @@ -87,32 +73,15 @@ void testReadDataIntegrityYaml() { // Assert that all "codes" are unique. List codeList = - checks.stream().map(DataIntegrityCheck::getCode).sorted().collect(toUnmodifiableList()); + checks.stream().map(DataIntegrityCheck::getCode).sorted().collect(toList()); assertEquals(codeList.size(), Set.copyOf(codeList).size()); // Assert that codes consist of upper case letter and numbers only String regEx = "^[A-Z0-9]+$"; Predicate IS_NOT_CAPS = Pattern.compile(regEx).asPredicate().negate(); - List badCodes = - codeList.stream().filter(IS_NOT_CAPS).collect(Collectors.toUnmodifiableList()); + List badCodes = codeList.stream().filter(IS_NOT_CAPS).collect(toList()); assertEquals(0, badCodes.size()); - // Assert that all checks have details ID and are unique and UIDish - List detailsIDs = - checks.stream().map(DataIntegrityCheck::getDetailsID).collect(toUnmodifiableList()); - assertEquals(detailsIDs.size(), Set.copyOf(detailsIDs).size()); - - List badDetailsUIDs = detailsIDs.stream().filter(e -> !isValidUid(e)).collect(toList()); - assertEquals(0, badDetailsUIDs.size()); - - // Assert that all checks have summary ID and are unique and are UIDish - List summaryIDs = - checks.stream().map(DataIntegrityCheck::getSummaryID).collect(toUnmodifiableList()); - - assertEquals(summaryIDs.size(), Set.copyOf(summaryIDs).size()); - List badSummaryUIDs = summaryIDs.stream().filter(e -> !isValidUid(e)).collect(toList()); - assertEquals(0, badSummaryUIDs.size()); - DataIntegrityCheck check = checks.get(0); assertEquals("categories_no_options", check.getName()); assertEquals("Categories with no category options", check.getDescription()); @@ -136,4 +105,63 @@ void testReadDataIntegrityYaml() { .getComment() .startsWith("SELECT uid,name from dataelementcategory")); } + + @Test + void testWithValidChecksFile() { + List checks = new ArrayList<>(); + readYaml(checks, "test-data-integrity-checks.yaml", "test-data-integrity-checks", CLASS_PATH); + assertEquals(1, checks.size()); + } + + @Test + void testWithInvalidChecksFile() { + List checks = new ArrayList<>(); + readYaml(checks, "invalid-file-checks.yaml", "test-data-integrity-checks", CLASS_PATH); + assertEquals(0, checks.size()); + } + + @Test + void testWithInvalidChecksDirectory() { + List checks = new ArrayList<>(); + readYaml(checks, "test-data-integrity-checks.yaml", "invalid-integrity-checks", CLASS_PATH); + assertEquals(0, checks.size()); + } + + @Test + void testWithInvalidChecksDirectoryFromFileSystem() { + List checks = new ArrayList<>(); + readYaml(checks, "data-integrity-checks.yaml", "invalid-integrity-checks", FILE_SYSTEM); + assertEquals(0, checks.size()); + } + + @Test + void testWithInvalidYamlFormat() { + List checks = new ArrayList<>(); + readYaml( + checks, "test-data-integrity-checks.yaml", "test-data-integrity-checks.yaml", FILE_SYSTEM); + assertEquals(0, checks.size()); + } + + private void readYaml( + List checks, + String fileChecks, + String checksDirectory, + DataIntegrityYamlReader.ResourceLocation resourceLocation) { + readDataIntegrityYaml( + new DefaultDataIntegrityService.DataIntegrityRecord( + resourceLocation, + fileChecks, + checksDirectory, + checks::add, + (property, defaultValue) -> defaultValue, + sql -> check -> new DataIntegritySummary(check, new Date(), new Date(), null, 1, 100d), + sql -> + check -> + new DataIntegrityDetails( + check, + new Date(), + new Date(), + null, + List.of(new DataIntegrityIssue("id", "name", sql, List.of()))))); + } } diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/jsonschema/JsonSchemaValidatorTest.java b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/jsonschema/JsonSchemaValidatorTest.java new file mode 100644 index 000000000000..7abfeb20950e --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/jsonschema/JsonSchemaValidatorTest.java @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.jsonschema; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.networknt.schema.ValidationMessage; +import java.io.IOException; +import java.io.InputStream; +import java.util.Locale; +import java.util.Set; +import java.util.stream.Collectors; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.springframework.core.io.ClassPathResource; + +class JsonSchemaValidatorTest { + + private final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + + @BeforeAll() + static void setup() { + Locale.setDefault(new Locale("en", "US")); + } + + @Test + void validateCheck_ValidFile() { + JsonNode jsonNode = getJsonNodeFromFile("check_with_all_required_fields.yaml"); + Set validationMessages = + JsonSchemaValidator.validateDataIntegrityCheck(jsonNode); + assertTrue(validationMessages.isEmpty()); + } + + @Test + void validateCheck_CheckMissingDescription() { + JsonNode jsonNode = getJsonNodeFromFile("check_without_description.yaml"); + Set validationMessages = + JsonSchemaValidator.validateDataIntegrityCheck(jsonNode); + assertEquals(1, validationMessages.size()); + assertContainsValidationMessages( + validationMessages, Set.of("$.description: is missing but it is required")); + } + + @Test + void validateCheck_CheckMissingName() { + JsonNode jsonNode = getJsonNodeFromFile("check_without_name.yaml"); + Set validationMessages = + JsonSchemaValidator.validateDataIntegrityCheck(jsonNode); + assertEquals(1, validationMessages.size()); + assertContainsValidationMessages( + validationMessages, Set.of("$.name: is missing but it is required")); + } + + @Test + void validateCheck_CheckMissing10Fields() { + JsonNode jsonNode = getJsonNodeFromFile("check_without_10_required_fields.yaml"); + Set validationMessages = + JsonSchemaValidator.validateDataIntegrityCheck(jsonNode); + assertEquals(8, validationMessages.size()); + assertContainsValidationMessages( + validationMessages, + Set.of( + "$.details_id_type: is missing but it is required", + "$.details_sql: is missing but it is required", + "$.introduction: is missing but it is required", + "$.recommendation: is missing but it is required", + "$.section: is missing but it is required", + "$.section_order: is missing but it is required", + "$.severity: is missing but it is required", + "$.summary_sql: is missing but it is required")); + } + + private void assertContainsValidationMessages( + Set validations, Set expected) { + assertTrue( + validations.stream() + .map(ValidationMessage::getMessage) + .collect(Collectors.toSet()) + .containsAll(expected)); + } + + private JsonNode getJsonNodeFromFile(String file) { + try (InputStream is = + new ClassPathResource("test-data-integrity-checks/" + file).getInputStream()) { + return mapper.readTree(is); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks.yaml b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks.yaml new file mode 100644 index 000000000000..5f6b72c696ae --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks.yaml @@ -0,0 +1,6 @@ +checks: + - check_with_all_required_fields.yaml + - check_without_10_required_fields.yaml + - check_without_description.yaml + - check_without_name.yaml + - invalid_yaml_format.yaml diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_with_all_required_fields.yaml b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_with_all_required_fields.yaml new file mode 100644 index 000000000000..607cc2b28da1 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_with_all_required_fields.yaml @@ -0,0 +1,15 @@ +--- +name: test_description_missing +description: test_description +section: test +section_order: 1 +summary_sql: >- + SQL here +details_sql: >- + SQL here +details_id_type: test +severity: CRITICAL +introduction: > + Intro here +recommendation: > + Recommendation here diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_without_10_required_fields.yaml b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_without_10_required_fields.yaml new file mode 100644 index 000000000000..0ade6e80d781 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_without_10_required_fields.yaml @@ -0,0 +1,3 @@ +--- +name: name +description: test_name_missing diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_without_description.yaml b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_without_description.yaml new file mode 100644 index 000000000000..be450286990f --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_without_description.yaml @@ -0,0 +1,14 @@ +--- +name: test_description_missing +section: test +section_order: 1 +summary_sql: >- + SQL here +details_sql: >- + SQL here +details_id_type: test +severity: CRITICAL +introduction: > + Intro here +recommendation: > + Recommendation here diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_without_name.yaml b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_without_name.yaml new file mode 100644 index 000000000000..6344ff376448 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/check_without_name.yaml @@ -0,0 +1,14 @@ +--- +description: test_name_missing +section: test +section_order: 1 +summary_sql: >- + SQL here +details_sql: >- + SQL here +details_id_type: test +severity: CRITICAL +introduction: > + Intro here +recommendation: > + Recommendation here diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/invalid_yaml_format.yaml b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/invalid_yaml_format.yaml new file mode 100644 index 000000000000..9d1fded7d807 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/resources/test-data-integrity-checks/invalid_yaml_format.yaml @@ -0,0 +1,13 @@ +description test_name_missing +section: test +section_order: 1 +summary_sql: >- + SQL here +details_sql: >- + SQL here +details_id_type: test +severity: CRITICAL +introduction: > + Intro here +recommendation: > + Recommendation here diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityCheck.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityCheck.java index cda43cae0b58..6a064f8f8c11 100644 --- a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityCheck.java +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityCheck.java @@ -75,6 +75,10 @@ default boolean getIsSlow() { return getBoolean("isSlow").booleanValue(); } + default boolean getIsProgrammatic() { + return getBoolean("isProgrammatic").booleanValue(); + } + default String getCode() { return getString("code").string(); } diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityChecksControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityChecksControllerTest.java new file mode 100644 index 000000000000..9e0187c82999 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityChecksControllerTest.java @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Set; +import org.hisp.dhis.dataintegrity.DataIntegrityCheckType; +import org.hisp.dhis.jsontree.JsonList; +import org.hisp.dhis.webapi.controller.DataIntegrityController; +import org.hisp.dhis.webapi.json.domain.JsonDataIntegrityCheck; +import org.junit.jupiter.api.Test; + +/** + * Tests the {@link DataIntegrityController} API with focus API returning {@link + * org.hisp.dhis.dataintegrity.DataIntegrityCheck} information. + * + * @author Jan Bernitt + */ +class DataIntegrityChecksControllerTest extends AbstractDataIntegrityIntegrationTest { + @Test + void testGetAvailableChecks() { + JsonList checks = + GET("/dataIntegrity").content().asList(JsonDataIntegrityCheck.class); + assertFalse(checks.isEmpty()); + assertCheckExists("categories_no_options", checks); + assertCheckExists("categories_one_default_category", checks); + assertCheckExists("categories_one_default_category_option", checks); + assertCheckExists("categories_one_default_category_combo", checks); + assertCheckExists("categories_one_default_category_option_combo", checks); + assertCheckExists("categories_unique_category_combo", checks); + for (DataIntegrityCheckType type : DataIntegrityCheckType.values()) { + assertCheckExists(type.getName(), checks); + } + } + + @Test + void testGetAvailableChecksNamesAreUnique() { + JsonList checks = + GET("/dataIntegrity").content().asList(JsonDataIntegrityCheck.class); + + assertEquals(checks.size(), Set.copyOf(checks.toList(JsonDataIntegrityCheck::getName)).size()); + } + + @Test + void testGetAvailableChecksCodesAreUnique() { + JsonList checks = + GET("/dataIntegrity").content().asList(JsonDataIntegrityCheck.class); + + assertEquals(checks.size(), Set.copyOf(checks.toList(JsonDataIntegrityCheck::getCode)).size()); + } + + @Test + void testGetAvailableChecks_FilterUsingCode() { + JsonList checks = + GET("/dataIntegrity?checks=CNO").content().asList(JsonDataIntegrityCheck.class); + assertEquals(1, checks.size()); + assertEquals("categories_no_options", checks.get(0).getName()); + } + + @Test + void testGetAvailableChecks_FilterUsingChecksPatterns() { + JsonList checks = + GET("/dataIntegrity?checks=program*").content().asList(JsonDataIntegrityCheck.class); + assertFalse(checks.isEmpty(), "there should be matches"); + checks.forEach(check -> assertTrue(check.getName().toLowerCase().startsWith("program"))); + } + + @Test + void testGetAvailableChecks_FilterUsingSection() { + JsonList checks = + GET("/dataIntegrity?section=Program Rules").content().asList(JsonDataIntegrityCheck.class); + assertFalse(checks.isEmpty(), "there should be matches"); + checks.forEach(check -> assertEquals("Program Rules", check.getSection())); + } + + @Test + void testGetAvailableChecks_FilterUsingSlow() { + JsonList checks = + GET("/dataIntegrity?slow=true").content().asList(JsonDataIntegrityCheck.class); + assertFalse(checks.isEmpty(), "there should be matches"); + checks.forEach(check -> assertTrue(check.getIsSlow())); + + checks = GET("/dataIntegrity?slow=false").content().asList(JsonDataIntegrityCheck.class); + assertFalse(checks.isEmpty(), "there should be matches"); + checks.forEach(check -> assertFalse(check.getIsSlow())); + } + + @Test + void testGetAvailableChecks_FilterUsingProgrammatic() { + JsonList checks = + GET("/dataIntegrity?programmatic=true").content().asList(JsonDataIntegrityCheck.class); + assertFalse(checks.isEmpty(), "there should be matches"); + checks.forEach(check -> assertTrue(check.getIsProgrammatic())); + + checks = + GET("/dataIntegrity?programmatic=false").content().asList(JsonDataIntegrityCheck.class); + assertFalse(checks.isEmpty(), "there should be matches"); + checks.forEach(check -> assertFalse(check.getIsProgrammatic())); + } + + /** + * The point of this test is to check if the known i18n texts provided are resolved and assigned + * to the correct field + */ + @Test + void testGetAvailableChecks_i18n() { + JsonList checks = + GET("/dataIntegrity?checks=program_rule_variables_without_attribute") + .content() + .asList(JsonDataIntegrityCheck.class); + assertEquals(1, checks.size()); + JsonDataIntegrityCheck check = checks.get(0); + assertEquals("program_rule_variables_without_attribute", check.getName()); + assertEquals("Program rule variables lacking an attribute", check.getDisplayName()); + assertEquals("Program Rules", check.getSection()); + assertEquals( + "Lists all programs with rule variables requiring an attribute source but that is not yet linked to an attribute", + check.getDescription()); + assertEquals( + "Assign an attribute to the variable in question or consider if the variable is not needed", + check.getRecommendation()); + } + + private void assertCheckExists(String name, JsonList checks) { + assertTrue(checks.stream().anyMatch(check -> check.getName().equals(name))); + } +} diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataIntegrityController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataIntegrityController.java index 8a2a8a0bcde6..cf31a2e9b6cd 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataIntegrityController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataIntegrityController.java @@ -27,14 +27,15 @@ */ package org.hisp.dhis.webapi.controller; +import static java.lang.System.currentTimeMillis; import static java.util.stream.Collectors.toList; import static org.hisp.dhis.commons.collection.CollectionUtils.isEmpty; -import static org.hisp.dhis.dxf2.webmessage.WebMessageUtils.conflict; import static org.hisp.dhis.dxf2.webmessage.WebMessageUtils.jobConfigurationReport; import java.util.Collection; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import javax.annotation.CheckForNull; import javax.annotation.Nonnull; import lombok.AllArgsConstructor; @@ -45,9 +46,12 @@ import org.hisp.dhis.dataintegrity.DataIntegrityService; import org.hisp.dhis.dataintegrity.DataIntegritySummary; import org.hisp.dhis.dxf2.webmessage.WebMessage; +import org.hisp.dhis.feedback.ConflictException; import org.hisp.dhis.scheduling.JobConfiguration; +import org.hisp.dhis.scheduling.JobParameters; import org.hisp.dhis.scheduling.JobType; import org.hisp.dhis.scheduling.SchedulingManager; +import org.hisp.dhis.scheduling.parameters.DataIntegrityDetailsJobParameters; import org.hisp.dhis.scheduling.parameters.DataIntegrityJobParameters; import org.hisp.dhis.scheduling.parameters.DataIntegrityJobParameters.DataIntegrityReportType; import org.hisp.dhis.user.CurrentUser; @@ -72,8 +76,8 @@ @ApiVersion({DhisApiVersion.DEFAULT, DhisApiVersion.ALL}) @AllArgsConstructor public class DataIntegrityController { - private final SchedulingManager schedulingManager; + private final SchedulingManager schedulingManager; private final DataIntegrityService dataIntegrityService; @PreAuthorize("hasRole('ALL') or hasRole('F_PERFORM_MAINTENANCE')") @@ -82,29 +86,33 @@ public class DataIntegrityController { public WebMessage runDataIntegrity( @CheckForNull @RequestParam(required = false) Set checks, @CheckForNull @RequestBody(required = false) Set checksBody, - @CurrentUser User currentUser) { + @CurrentUser User currentUser) + throws ConflictException { Set names = getCheckNames(checksBody, checks); - return runDataIntegrityAsync( - names, currentUser, "runDataIntegrity", DataIntegrityReportType.REPORT) + return runDataIntegrityAsync(names, currentUser, DataIntegrityReportType.REPORT) .setLocation("/dataIntegrity/details?checks=" + toChecksList(names)); } private WebMessage runDataIntegrityAsync( - @Nonnull Set checks, - User currentUser, - String description, - DataIntegrityReportType type) { - DataIntegrityJobParameters params = new DataIntegrityJobParameters(); - params.setChecks(checks); - params.setType(type); - JobConfiguration config = - new JobConfiguration(description, JobType.DATA_INTEGRITY, null, params, true, true); + @Nonnull Set checks, User currentUser, DataIntegrityReportType type) + throws ConflictException { + JobType jobType = + type == DataIntegrityReportType.DETAILS + ? JobType.DATA_INTEGRITY_DETAILS + : JobType.DATA_INTEGRITY; + JobParameters params = + type == DataIntegrityReportType.DETAILS + ? new DataIntegrityDetailsJobParameters(checks) + : new DataIntegrityJobParameters(type, checks); + String name = "DATA_INTEGRITY_" + currentTimeMillis(); + JobConfiguration config = new JobConfiguration(name, jobType, null, params, true, true); config.setUserUid(currentUser.getUid()); config.setAutoFields(); if (!schedulingManager.executeNow(config)) { - return conflict("Data integrity check is already running"); + throw new ConflictException("Data integrity check is already running"); } + return jobConfigurationReport(config); } @@ -112,12 +120,16 @@ private WebMessage runDataIntegrityAsync( @ResponseBody public Collection getAvailableChecks( @CheckForNull @RequestParam(required = false) Set checks, - @CheckForNull @RequestParam(required = false) String section) { + @CheckForNull @RequestParam(required = false) String section, + @CheckForNull @RequestParam(required = false) Boolean slow, + @CheckForNull @RequestParam(required = false) Boolean programmatic) { Collection matches = dataIntegrityService.getDataIntegrityChecks(getCheckNames(checks)); - return section == null || section.isBlank() - ? matches - : matches.stream().filter(check -> section.equals(check.getSection())).collect(toList()); + Predicate filter = check -> true; + if (section != null && !section.isBlank()) filter = check -> section.equals(check.getSection()); + if (slow != null) filter = filter.and(check -> check.isSlow() == slow); + if (programmatic != null) filter = filter.and(check -> check.isProgrammatic() == programmatic); + return matches.stream().filter(filter).collect(toList()); } @GetMapping("/summary/running") @@ -147,10 +159,10 @@ public Map getSummaries( public WebMessage runSummariesCheck( @CheckForNull @RequestParam(required = false) Set checks, @CheckForNull @RequestBody(required = false) Set checksBody, - @CurrentUser User currentUser) { + @CurrentUser User currentUser) + throws ConflictException { Set names = getCheckNames(checksBody, checks); - return runDataIntegrityAsync( - names, currentUser, "runSummariesCheck", DataIntegrityReportType.SUMMARY) + return runDataIntegrityAsync(names, currentUser, DataIntegrityReportType.SUMMARY) .setLocation("/dataIntegrity/summary?checks=" + toChecksList(names)); } @@ -181,10 +193,10 @@ public Map getDetails( public WebMessage runDetailsCheck( @CheckForNull @RequestParam(required = false) Set checks, @RequestBody(required = false) Set checksBody, - @CurrentUser User currentUser) { + @CurrentUser User currentUser) + throws ConflictException { Set names = getCheckNames(checksBody, checks); - return runDataIntegrityAsync( - names, currentUser, "runDetailsCheck", DataIntegrityReportType.DETAILS) + return runDataIntegrityAsync(names, currentUser, DataIntegrityReportType.DETAILS) .setLocation("/dataIntegrity/details?checks=" + toChecksList(names)); }