diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheck.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheck.java index 206e71bd3116..71feb0af1b6e 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheck.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheck.java @@ -27,10 +27,13 @@ */ package org.hisp.dhis.dataintegrity; +import static java.util.Comparator.comparingLong; import static java.util.stream.Collectors.joining; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import java.io.Serializable; +import java.util.Comparator; import java.util.function.Function; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -54,25 +57,35 @@ @Builder @AllArgsConstructor(access = AccessLevel.PRIVATE) public final class DataIntegrityCheck implements Serializable { - @JsonProperty private final String name; - @JsonProperty private final String displayName; + public static final Comparator FAST_TO_SLOW = + comparingLong(DataIntegrityCheck::getExecutionTimeIndicator); + @JsonProperty private final String name; + @JsonProperty private final String displayName; @JsonProperty private final String section; - @JsonProperty private final int sectionOrder; - @JsonProperty private final DataIntegritySeverity severity; - @JsonProperty private final String description; - @JsonProperty private final String introduction; - @JsonProperty private final String recommendation; - @JsonProperty private final String issuesIdType; - @JsonProperty private final boolean isSlow; + @JsonProperty private final boolean isProgrammatic; + + private long executionTime; + private int executionCount; + + public @JsonProperty Long getAverageExecutionTime() { + return executionTime <= 0L ? null : executionTime / executionCount; + } + + @JsonIgnore + long getExecutionTimeIndicator() { + Long time = getAverageExecutionTime(); + if (time != null) return time; + return isSlow ? Long.MAX_VALUE : 1000; + } @JsonProperty public String getCode() { @@ -83,6 +96,12 @@ public String getCode() { private final transient Function runDetailsCheck; + public DataIntegrityCheck addExecution(long time) { + executionCount++; + executionTime += time; + return this; + } + /** * Method that takes in a name of a {@link DataIntegrityCheck} and converts it to an acronym of * its name using the first letter from each word e.g. my_data_integrity_check -> MDIC diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfiguration.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfiguration.java index 2ae1d3a13a15..9593f6d4aeae 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfiguration.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfiguration.java @@ -51,6 +51,7 @@ import org.hisp.dhis.scheduling.parameters.AggregateDataExchangeJobParameters; import org.hisp.dhis.scheduling.parameters.AnalyticsJobParameters; import org.hisp.dhis.scheduling.parameters.ContinuousAnalyticsJobParameters; +import org.hisp.dhis.scheduling.parameters.DataIntegrityDetailsJobParameters; import org.hisp.dhis.scheduling.parameters.DataIntegrityJobParameters; import org.hisp.dhis.scheduling.parameters.DataSynchronizationJobParameters; import org.hisp.dhis.scheduling.parameters.DisableInactiveUsersJobParameters; @@ -285,6 +286,9 @@ public boolean hasCronExpression() { value = TrackerTrigramIndexJobParameters.class, name = "TRACKER_SEARCH_OPTIMIZATION"), @JsonSubTypes.Type(value = DataIntegrityJobParameters.class, name = "DATA_INTEGRITY"), + @JsonSubTypes.Type( + value = DataIntegrityDetailsJobParameters.class, + name = "DATA_INTEGRITY_DETAILS"), @JsonSubTypes.Type( value = AggregateDataExchangeJobParameters.class, name = "AGGREGATE_DATA_EXCHANGE"), diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java index 7b51389db2d9..c45ce4f56e0e 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java @@ -42,6 +42,7 @@ import org.hisp.dhis.scheduling.parameters.AggregateDataExchangeJobParameters; import org.hisp.dhis.scheduling.parameters.AnalyticsJobParameters; import org.hisp.dhis.scheduling.parameters.ContinuousAnalyticsJobParameters; +import org.hisp.dhis.scheduling.parameters.DataIntegrityDetailsJobParameters; import org.hisp.dhis.scheduling.parameters.DataIntegrityJobParameters; import org.hisp.dhis.scheduling.parameters.DataSynchronizationJobParameters; import org.hisp.dhis.scheduling.parameters.DisableInactiveUsersJobParameters; @@ -71,6 +72,7 @@ public enum JobType { User defined jobs */ DATA_INTEGRITY(DataIntegrityJobParameters.class), + DATA_INTEGRITY_DETAILS(DataIntegrityDetailsJobParameters.class), RESOURCE_TABLE(), ANALYTICS_TABLE(AnalyticsJobParameters.class), CONTINUOUS_ANALYTICS_TABLE(ContinuousAnalyticsJobParameters.class), @@ -232,7 +234,7 @@ public boolean isUsingErrorNotification() { * the ready jobs per type is attempted to start in a single loop cycle */ public boolean isUsingContinuousExecution() { - return this == METADATA_IMPORT || this == TRACKER_IMPORT_JOB; + return this == METADATA_IMPORT || this == TRACKER_IMPORT_JOB || this == DATA_INTEGRITY_DETAILS; } public boolean hasJobParameters() { @@ -248,7 +250,7 @@ public boolean isUserDefined() { public Map getRelativeApiElements() { return switch (this) { - case DATA_INTEGRITY -> Map.of("checks", "/api/dataIntegrity"); + case DATA_INTEGRITY, DATA_INTEGRITY_DETAILS -> Map.of("checks", "/api/dataIntegrity"); case ANALYTICS_TABLE -> Map.of( "skipTableTypes", "/api/analytics/tableTypes", "skipPrograms", "/api/programs"); diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityDetailsJobParameters.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityDetailsJobParameters.java new file mode 100644 index 000000000000..fbd9b5010f4d --- /dev/null +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityDetailsJobParameters.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.scheduling.parameters; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import java.util.Set; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.hisp.dhis.common.DxfNamespaces; +import org.hisp.dhis.scheduling.JobParameters; + +/** + * @author Jan Bernitt + */ +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class DataIntegrityDetailsJobParameters implements JobParameters { + + @JsonProperty(required = false) + @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0) + private Set checks; +} diff --git a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityCheckTest.java b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityCheckTest.java new file mode 100644 index 000000000000..8c2bf862d2c0 --- /dev/null +++ b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityCheckTest.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.dataintegrity; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import org.junit.jupiter.api.Test; + +/** + * @author Jan Bernitt + */ +class DataIntegrityCheckTest { + + @Test + void testSortingFastToSlow_NoExecutionTime() { + List checks = + List.of( + DataIntegrityCheck.builder().name("a").isSlow(true).build(), + DataIntegrityCheck.builder().name("b").isSlow(false).build(), + DataIntegrityCheck.builder().name("c").isSlow(true).build()); + List actual = + checks.stream().sorted(DataIntegrityCheck.FAST_TO_SLOW).toList(); + assertEquals(List.of("b", "a", "c"), actual.stream().map(DataIntegrityCheck::getName).toList()); + } + + @Test + void testSortingFastToSlow() { + List checks = + List.of( + DataIntegrityCheck.builder().name("a").isSlow(true).build().addExecution(12345), + DataIntegrityCheck.builder().name("b").isSlow(false).build().addExecution(20), + DataIntegrityCheck.builder().name("c").isSlow(true).build().addExecution(500)); + List actual = + checks.stream().sorted(DataIntegrityCheck.FAST_TO_SLOW).toList(); + assertEquals(List.of("b", "c", "a"), actual.stream().map(DataIntegrityCheck::getName).toList()); + } + + @Test + void testSortingFastToSlow_MixedExecutionTime() { + List checks = + List.of( + DataIntegrityCheck.builder().name("a").isSlow(true).build().addExecution(12345), + DataIntegrityCheck.builder().name("b").isSlow(false).build(), + DataIntegrityCheck.builder().name("c").isSlow(true).build().addExecution(500)); + List actual = + checks.stream().sorted(DataIntegrityCheck.FAST_TO_SLOW).toList(); + assertEquals(List.of("c", "b", "a"), actual.stream().map(DataIntegrityCheck::getName).toList()); + } + + @Test + void testAddExecution() { + DataIntegrityCheck check = DataIntegrityCheck.builder().name("a").build(); + + check.addExecution(1000).addExecution(1000); // sum is 2000, 2 times + + assertEquals(1000L, check.getAverageExecutionTime()); + + check.addExecution(4000); // sum is now 6000, 3 times + + assertEquals(2000L, check.getAverageExecutionTime()); + } +} diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java index 2f6286b32fc8..bd25dca42688 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java @@ -491,7 +491,6 @@ private List getInvalidValidationRuleExpressions( issues.add(toIssue(rule, i18n.getString(result.getKey()))); } } - return issues; } @@ -508,11 +507,13 @@ private void registerNonDatabaseIntegrityCheck( try { Schema issueSchema = issueIdType == null ? null : schemaService.getDynamicSchema(issueIdType); String issueIdTypeName = issueSchema == null ? null : issueSchema.getPlural(); - checksByName.put( + checksByName.putIfAbsent( name, DataIntegrityCheck.builder() .name(name) .displayName(info.apply("name", name.replace('_', ' '))) + .isSlow(true) + .isProgrammatic(true) .severity( DataIntegritySeverity.valueOf( info.apply("severity", DataIntegritySeverity.WARNING.name()).toUpperCase())) @@ -942,7 +943,10 @@ private void runDataIntegrityChecks( progress.startingProcess("Data Integrity check"); progress.startingStage(stageDesc, checks.size(), SKIP_ITEM); progress.runStage( - checks.stream().map(checksByName::get).filter(Objects::nonNull), + checks.stream() + .map(checksByName::get) + .filter(Objects::nonNull) + .sorted(DataIntegrityCheck.FAST_TO_SLOW), DataIntegrityCheck::getDescription, check -> { Date startTime = new Date(); @@ -956,6 +960,7 @@ private void runDataIntegrityChecks( running.remove(check.getName()); } if (res != null) { + check.addExecution(currentTimeMillis() - startTime.getTime()); cache.put(check.getName(), res); } }); @@ -981,9 +986,13 @@ private Set expandChecks(Set names) { .map(DataIntegrityCheck::getName) .forEach(expanded::add); } else if (name.contains("*")) { - String pattern = name.toLowerCase().replace('-', '_').replace("*", ".*"); + String pattern = + name.toLowerCase() + .replace('-', '_') // make uniform + .replaceAll("[^*_a-z0-9]+", "") // sanitise against regex attacks + .replace("*", ".*"); // expand regex wildcard match for (DataIntegrityCheck check : checksByName.values()) { - if (!check.isSlow() && check.getName().matches(pattern)) { + if (check.getName().matches(pattern)) { expanded.add(check.getName()); } } @@ -1005,12 +1014,12 @@ private Set getDefaultChecks() { private void ensureConfigurationsAreLoaded() { if (configurationsAreLoaded.compareAndSet(false, true)) { - // programmatic checks - initIntegrityChecks(); - // load system-packaged data integrity checks loadChecks(CLASS_PATH, "data-integrity-checks.yaml", "data-integrity-checks"); + // programmatic checks + initIntegrityChecks(); + // load user-packaged custom data integrity checks try { String dhis2Home = locationManager.getExternalDirectoryPath(); diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityDetailsJob.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityDetailsJob.java new file mode 100644 index 000000000000..d6ed5cfbd70b --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityDetailsJob.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.dataintegrity.jobs; + +import java.util.Set; +import lombok.AllArgsConstructor; +import org.hisp.dhis.dataintegrity.DataIntegrityService; +import org.hisp.dhis.scheduling.Job; +import org.hisp.dhis.scheduling.JobConfiguration; +import org.hisp.dhis.scheduling.JobProgress; +import org.hisp.dhis.scheduling.JobType; +import org.hisp.dhis.scheduling.parameters.DataIntegrityDetailsJobParameters; +import org.springframework.stereotype.Component; + +/** + * @author Jan Bernitt + */ +@Component +@AllArgsConstructor +public class DataIntegrityDetailsJob implements Job { + + private final DataIntegrityService dataIntegrityService; + + @Override + public JobType getJobType() { + return JobType.DATA_INTEGRITY_DETAILS; + } + + @Override + public void execute(JobConfiguration config, JobProgress progress) { + DataIntegrityDetailsJobParameters parameters = + (DataIntegrityDetailsJobParameters) config.getJobParameters(); + Set checks = parameters == null ? Set.of() : parameters.getChecks(); + + dataIntegrityService.runDetailsChecks(checks, progress); + } +} diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityJob.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityJob.java index 70376595dfa3..9e418a69b1cb 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityJob.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/jobs/DataIntegrityJob.java @@ -43,7 +43,8 @@ import org.springframework.stereotype.Component; /** - * @author Halvdan Hoem Grelland + * @author Halvdan Hoem Grelland (original) + * @author Jan Bernitt (refactored) */ @Component @AllArgsConstructor diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityCheck.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityCheck.java index cda43cae0b58..6a064f8f8c11 100644 --- a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityCheck.java +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityCheck.java @@ -75,6 +75,10 @@ default boolean getIsSlow() { return getBoolean("isSlow").booleanValue(); } + default boolean getIsProgrammatic() { + return getBoolean("isProgrammatic").booleanValue(); + } + default String getCode() { return getString("code").string(); } diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/resources/postgresTestConfig.conf b/dhis-2/dhis-support/dhis-support-test/src/main/resources/postgresTestConfig.conf index 1590bec66d54..5297dda9119e 100644 --- a/dhis-2/dhis-support/dhis-support-test/src/main/resources/postgresTestConfig.conf +++ b/dhis-2/dhis-support/dhis-support-test/src/main/resources/postgresTestConfig.conf @@ -1,6 +1,7 @@ filestore.provider = transient filestore.container = files connection.schema=validate +connection.pool.max_size=50 encryption.password=54C73D06-1D34-477F-94B0-8F94E59BE41D hibernate.cache.use_query_cache=false @@ -15,4 +16,3 @@ aggregate.audit.persist=on audit.metadata=CREATE_UPDATE_DELETE audit.tracker=CREATE_UPDATE_DELETE audit.aggregate=CREATE_UPDATE_DELETE -connection.pool.max_size=50 diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityChecksControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityChecksControllerTest.java index 0d9e1311cfa4..9e0187c82999 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityChecksControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityChecksControllerTest.java @@ -89,7 +89,7 @@ void testGetAvailableChecks_FilterUsingCode() { void testGetAvailableChecks_FilterUsingChecksPatterns() { JsonList checks = GET("/dataIntegrity?checks=program*").content().asList(JsonDataIntegrityCheck.class); - assertTrue(checks.size() > 0, "there should be matches"); + assertFalse(checks.isEmpty(), "there should be matches"); checks.forEach(check -> assertTrue(check.getName().toLowerCase().startsWith("program"))); } @@ -97,10 +97,35 @@ void testGetAvailableChecks_FilterUsingChecksPatterns() { void testGetAvailableChecks_FilterUsingSection() { JsonList checks = GET("/dataIntegrity?section=Program Rules").content().asList(JsonDataIntegrityCheck.class); - assertTrue(checks.size() > 0, "there should be matches"); + assertFalse(checks.isEmpty(), "there should be matches"); checks.forEach(check -> assertEquals("Program Rules", check.getSection())); } + @Test + void testGetAvailableChecks_FilterUsingSlow() { + JsonList checks = + GET("/dataIntegrity?slow=true").content().asList(JsonDataIntegrityCheck.class); + assertFalse(checks.isEmpty(), "there should be matches"); + checks.forEach(check -> assertTrue(check.getIsSlow())); + + checks = GET("/dataIntegrity?slow=false").content().asList(JsonDataIntegrityCheck.class); + assertFalse(checks.isEmpty(), "there should be matches"); + checks.forEach(check -> assertFalse(check.getIsSlow())); + } + + @Test + void testGetAvailableChecks_FilterUsingProgrammatic() { + JsonList checks = + GET("/dataIntegrity?programmatic=true").content().asList(JsonDataIntegrityCheck.class); + assertFalse(checks.isEmpty(), "there should be matches"); + checks.forEach(check -> assertTrue(check.getIsProgrammatic())); + + checks = + GET("/dataIntegrity?programmatic=false").content().asList(JsonDataIntegrityCheck.class); + assertFalse(checks.isEmpty(), "there should be matches"); + checks.forEach(check -> assertFalse(check.getIsProgrammatic())); + } + /** * The point of this test is to check if the known i18n texts provided are resolved and assigned * to the correct field diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataIntegrityController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataIntegrityController.java index 96e42cd15c02..1bc565a61fe6 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataIntegrityController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataIntegrityController.java @@ -34,6 +34,7 @@ import java.util.Collection; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import javax.annotation.CheckForNull; import javax.annotation.Nonnull; import lombok.AllArgsConstructor; @@ -48,8 +49,10 @@ import org.hisp.dhis.feedback.NotFoundException; import org.hisp.dhis.scheduling.JobConfiguration; import org.hisp.dhis.scheduling.JobConfigurationService; +import org.hisp.dhis.scheduling.JobParameters; import org.hisp.dhis.scheduling.JobSchedulerService; import org.hisp.dhis.scheduling.JobType; +import org.hisp.dhis.scheduling.parameters.DataIntegrityDetailsJobParameters; import org.hisp.dhis.scheduling.parameters.DataIntegrityJobParameters; import org.hisp.dhis.scheduling.parameters.DataIntegrityJobParameters.DataIntegrityReportType; import org.hisp.dhis.user.CurrentUser; @@ -95,9 +98,17 @@ public WebMessage runDataIntegrity( private WebMessage runDataIntegrityAsync( @Nonnull Set checks, User currentUser, DataIntegrityReportType type) throws ConflictException, NotFoundException { - JobConfiguration config = new JobConfiguration(JobType.DATA_INTEGRITY); + JobType jobType = + type == DataIntegrityReportType.DETAILS + ? JobType.DATA_INTEGRITY_DETAILS + : JobType.DATA_INTEGRITY; + JobConfiguration config = new JobConfiguration(jobType); config.setExecutedBy(currentUser.getUid()); - config.setJobParameters(new DataIntegrityJobParameters(type, checks)); + JobParameters parameters = + type == DataIntegrityReportType.DETAILS + ? new DataIntegrityDetailsJobParameters(checks) + : new DataIntegrityJobParameters(type, checks); + config.setJobParameters(parameters); jobSchedulerService.executeNow(jobConfigurationService.create(config)); @@ -108,12 +119,16 @@ private WebMessage runDataIntegrityAsync( @ResponseBody public Collection getAvailableChecks( @CheckForNull @RequestParam(required = false) Set checks, - @CheckForNull @RequestParam(required = false) String section) { + @CheckForNull @RequestParam(required = false) String section, + @CheckForNull @RequestParam(required = false) Boolean slow, + @CheckForNull @RequestParam(required = false) Boolean programmatic) { Collection matches = dataIntegrityService.getDataIntegrityChecks(getCheckNames(checks)); - return section == null || section.isBlank() - ? matches - : matches.stream().filter(check -> section.equals(check.getSection())).collect(toList()); + Predicate filter = check -> true; + if (section != null && !section.isBlank()) filter = check -> section.equals(check.getSection()); + if (slow != null) filter = filter.and(check -> check.isSlow() == slow); + if (programmatic != null) filter = filter.and(check -> check.isProgrammatic() == programmatic); + return matches.stream().filter(filter).collect(toList()); } @GetMapping("/summary/running")