feat: improved data integrity execution [DHIS2-16223] (2.40) (#15804)

* feat: improved data integrity execution [DHIS2-16223] (2.40)

* fix: compiles again

* fix: update YAML and schema to master

* fix: job creation in controller

* fix: revert table renames

* fix: test setup

dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheck.java

@@ -27,12 +27,16 @@
  */
 package org.hisp.dhis.dataintegrity;
 
+import static java.util.Comparator.comparingLong;
 import static java.util.stream.Collectors.joining;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import java.io.Serializable;
+import java.util.Comparator;
 import java.util.function.Function;
 import java.util.stream.Stream;
+import javax.annotation.Nonnull;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -53,36 +57,58 @@
 @Builder
 @AllArgsConstructor(access = AccessLevel.PRIVATE)
 public final class DataIntegrityCheck implements Serializable {
-  @JsonProperty private final String name;
 
-  @JsonProperty private final String displayName;
+  public static final Comparator<DataIntegrityCheck> FAST_TO_SLOW =
+      comparingLong(DataIntegrityCheck::getExecutionTimeIndicator);
 
+  @JsonProperty private final String name;
+  @JsonProperty private final String displayName;
   @JsonProperty private final String section;
-
+  @JsonProperty private final int sectionOrder;
   @JsonProperty private final DataIntegritySeverity severity;
-
   @JsonProperty private final String description;
-
   @JsonProperty private final String introduction;
-
   @JsonProperty private final String recommendation;
-
   @JsonProperty private final String issuesIdType;
-
   @JsonProperty private final boolean isSlow;
+  @JsonProperty private final boolean isProgrammatic;
 
-  @JsonProperty
-  public String getCode() {
-    return Stream.of(name.split("_"))
-        .map(f -> String.valueOf(f.charAt(0)).toUpperCase())
-        .collect(joining());
+  private long executionTime;
+  private int executionCount;
+
+  public @JsonProperty Long getAverageExecutionTime() {
+    return executionTime <= 0L ? null : executionTime / executionCount;
   }
 
-  private final String detailsID;
+  @JsonIgnore
+  long getExecutionTimeIndicator() {
+    Long time = getAverageExecutionTime();
+    if (time != null) return time;
+    return isSlow ? Long.MAX_VALUE : 1000;
+  }
 
-  private final String summaryID;
+  @JsonProperty
+  public String getCode() {
+    return getCodeFromName(name);
+  }
 
   private final transient Function<DataIntegrityCheck, DataIntegritySummary> runSummaryCheck;
 
   private final transient Function<DataIntegrityCheck, DataIntegrityDetails> runDetailsCheck;
+
+  public DataIntegrityCheck addExecution(long time) {
+    executionCount++;
+    executionTime += time;
+    return this;
+  }
+
+  /**
+   * Method that takes in a name of a {@link DataIntegrityCheck} and converts it to an acronym of
+   * its name using the first letter from each word e.g. my_data_integrity_check -> MDIC
+   */
+  public static String getCodeFromName(@Nonnull String fullName) {
+    return Stream.of(fullName.split("_"))
+        .map(word -> String.valueOf(word.charAt(0)).toUpperCase())
+        .collect(joining());
+  }
 }

dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfiguration.java

@@ -45,6 +45,7 @@
 import org.hisp.dhis.scheduling.parameters.AggregateDataExchangeJobParameters;
 import org.hisp.dhis.scheduling.parameters.AnalyticsJobParameters;
 import org.hisp.dhis.scheduling.parameters.ContinuousAnalyticsJobParameters;
+import org.hisp.dhis.scheduling.parameters.DataIntegrityDetailsJobParameters;
 import org.hisp.dhis.scheduling.parameters.DataIntegrityJobParameters;
 import org.hisp.dhis.scheduling.parameters.DataSynchronizationJobParameters;
 import org.hisp.dhis.scheduling.parameters.DisableInactiveUsersJobParameters;
@@ -346,6 +347,9 @@ public void setDelay(Integer delay) {
             value = TrackerTrigramIndexJobParameters.class,
             name = "TRACKER_SEARCH_OPTIMIZATION"),
         @JsonSubTypes.Type(value = DataIntegrityJobParameters.class, name = "DATA_INTEGRITY"),
+        @JsonSubTypes.Type(
+            value = DataIntegrityDetailsJobParameters.class,
+            name = "DATA_INTEGRITY_DETAILS"),
         @JsonSubTypes.Type(
             value = AggregateDataExchangeJobParameters.class,
             name = "AGGREGATE_DATA_EXCHANGE"),

dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java

@@ -31,6 +31,7 @@
 import org.hisp.dhis.scheduling.parameters.AggregateDataExchangeJobParameters;
 import org.hisp.dhis.scheduling.parameters.AnalyticsJobParameters;
 import org.hisp.dhis.scheduling.parameters.ContinuousAnalyticsJobParameters;
+import org.hisp.dhis.scheduling.parameters.DataIntegrityDetailsJobParameters;
 import org.hisp.dhis.scheduling.parameters.DataIntegrityJobParameters;
 import org.hisp.dhis.scheduling.parameters.DataSynchronizationJobParameters;
 import org.hisp.dhis.scheduling.parameters.DisableInactiveUsersJobParameters;
@@ -62,6 +63,11 @@ public enum JobType {
       SchedulingType.CRON,
       DataIntegrityJobParameters.class,
       Map.of("checks", "/api/dataIntegrity")),
+  DATA_INTEGRITY_DETAILS(
+      true,
+      SchedulingType.CRON,
+      DataIntegrityDetailsJobParameters.class,
+      Map.of("checks", "/api/dataIntegrity")),
   RESOURCE_TABLE(true),
   ANALYTICS_TABLE(
       true,

dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityDetailsJobParameters.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2004-2023, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.hisp.dhis.scheduling.parameters;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
+import java.util.Set;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import org.hisp.dhis.common.DxfNamespaces;
+import org.hisp.dhis.scheduling.JobParameters;
+
+/**
+ * @author Jan Bernitt
+ */
+@Getter
+@Setter
+@NoArgsConstructor
+@AllArgsConstructor
+public class DataIntegrityDetailsJobParameters implements JobParameters {
+
+  @JsonProperty(required = false)
+  @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
+  private Set<String> checks;
+}

dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/DataIntegrityJobParameters.java

@@ -30,7 +30,9 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
 import java.util.Set;
+import lombok.AllArgsConstructor;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
 import lombok.Setter;
 import org.hisp.dhis.common.DxfNamespaces;
 import org.hisp.dhis.scheduling.JobParameters;
@@ -40,6 +42,8 @@
  */
 @Getter
 @Setter
+@NoArgsConstructor
+@AllArgsConstructor
 public class DataIntegrityJobParameters implements JobParameters {
   public enum DataIntegrityReportType {
     REPORT,
@@ -49,9 +53,9 @@ public enum DataIntegrityReportType {
 
   @JsonProperty(required = false)
   @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
-  private Set<String> checks;
+  private DataIntegrityReportType type;
 
   @JsonProperty(required = false)
   @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
-  private DataIntegrityReportType type;
+  private Set<String> checks;
 }

dhis-2/dhis-api/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityCheckTest.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2004-2023, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.hisp.dhis.dataintegrity;
+
+import static java.util.stream.Collectors.toList;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.List;
+import org.junit.jupiter.api.Test;
+
+/**
+ * @author Jan Bernitt
+ */
+class DataIntegrityCheckTest {
+
+  @Test
+  void testSortingFastToSlow_NoExecutionTime() {
+    List<DataIntegrityCheck> checks =
+        List.of(
+            DataIntegrityCheck.builder().name("a").isSlow(true).build(),
+            DataIntegrityCheck.builder().name("b").isSlow(false).build(),
+            DataIntegrityCheck.builder().name("c").isSlow(true).build());
+    List<DataIntegrityCheck> actual =
+        checks.stream().sorted(DataIntegrityCheck.FAST_TO_SLOW).collect(toList());
+    assertEquals(
+        List.of("b", "a", "c"), actual.stream().map(DataIntegrityCheck::getName).collect(toList()));
+  }
+
+  @Test
+  void testSortingFastToSlow() {
+    List<DataIntegrityCheck> checks =
+        List.of(
+            DataIntegrityCheck.builder().name("a").isSlow(true).build().addExecution(12345),
+            DataIntegrityCheck.builder().name("b").isSlow(false).build().addExecution(20),
+            DataIntegrityCheck.builder().name("c").isSlow(true).build().addExecution(500));
+    List<DataIntegrityCheck> actual =
+        checks.stream().sorted(DataIntegrityCheck.FAST_TO_SLOW).collect(toList());
+    assertEquals(
+        List.of("b", "c", "a"), actual.stream().map(DataIntegrityCheck::getName).collect(toList()));
+  }
+
+  @Test
+  void testSortingFastToSlow_MixedExecutionTime() {
+    List<DataIntegrityCheck> checks =
+        List.of(
+            DataIntegrityCheck.builder().name("a").isSlow(true).build().addExecution(12345),
+            DataIntegrityCheck.builder().name("b").isSlow(false).build(),
+            DataIntegrityCheck.builder().name("c").isSlow(true).build().addExecution(500));
+    List<DataIntegrityCheck> actual =
+        checks.stream().sorted(DataIntegrityCheck.FAST_TO_SLOW).collect(toList());
+    assertEquals(
+        List.of("c", "b", "a"), actual.stream().map(DataIntegrityCheck::getName).collect(toList()));
+  }
+
+  @Test
+  void testAddExecution() {
+    DataIntegrityCheck check = DataIntegrityCheck.builder().name("a").build();
+
+    check.addExecution(1000).addExecution(1000); // sum is 2000, 2 times
+
+    assertEquals(1000L, check.getAverageExecutionTime());
+
+    check.addExecution(4000); // sum is now 6000, 3 times
+
+    assertEquals(2000L, check.getAverageExecutionTime());
+  }
+}

dhis-2/dhis-services/dhis-service-administration/pom.xml

@@ -135,6 +135,21 @@
       <groupId>com.google.code.findbugs</groupId>
       <artifactId>jsr305</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.networknt</groupId>
+      <artifactId>json-schema-validator</artifactId>
+      <version>1.0.87</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.commons</groupId>
+          <artifactId>commons-lang3</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-collections4</artifactId>
+    </dependency>
 
     <!-- Test -->
     <dependency>