feat: query API for job configurations with errors [DHIS2-15276]

dhis-2/dhis-api/pom.xml

@@ -97,6 +97,10 @@
       <groupId>org.hisp.dhis.rules</groupId>
       <artifactId>rule-engine</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.hisp.dhis</groupId>
+      <artifactId>json-tree</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.springframework.security</groupId>
       <artifactId>spring-security-core</artifactId>

dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfigurationService.java

@@ -30,7 +30,9 @@
 import java.io.InputStream;
 import java.util.List;
 import java.util.Map;
+import javax.annotation.Nonnull;
 import org.hisp.dhis.feedback.ConflictException;
+import org.hisp.dhis.jsontree.JsonObject;
 import org.hisp.dhis.schema.Property;
 import org.springframework.util.MimeType;
 
@@ -160,6 +162,13 @@ String create(JobConfiguration config, MimeType contentType, InputStream content
    */
   List<JobConfiguration> getStaleConfigurations(int staleForSeconds);
 
+  /**
+   * @param params query parameters (criteria) to find
+   * @return all job configurations that match the query parameters
+   */
+  @Nonnull
+  List<JsonObject> findJobRunErrors(@Nonnull JobRunErrorsParams params);
+
   /**
    * Get a map of parameter classes with appropriate properties This can be used for a frontend app
    * or for other appropriate applications which needs information about the jobs in the system.

dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfigurationStore.java

@@ -115,6 +115,13 @@ public interface JobConfigurationStore extends GenericDimensionalObjectStore<Job
    */
   Stream<JobConfiguration> getDueJobConfigurations(boolean includeWaiting);
 
+  /**
+   * @param params query parameters (criteria) to find
+   * @return all job configurations that match the query parameters
+   */
+  @Nonnull
+  Stream<String> findJobRunErrors(@Nonnull JobRunErrorsParams params);
+
   /**
    * @return A list of all job types that are currently in {@link JobStatus#RUNNING} state.
    */

dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobProgress.java

@@ -142,12 +142,18 @@ default boolean isSkipCurrentStage() {
   Error reporting API:
   */
 
-  default void addError(ErrorCode code, String uid, String type, Integer index, String... args) {
-    addError(code, uid, type, index, List.of(args));
+  default void addError(
+      @Nonnull ErrorCode code, @CheckForNull String uid, @Nonnull String type, String... args) {
+    addError(code, uid, type, List.of(args));
   }
 
-  default void addError(ErrorCode code, String uid, String type, Integer index, List<String> args) {
-    // default implementation is a NOOP, we don't remember or handle the error
+  default void addError(
+      @Nonnull ErrorCode code,
+      @CheckForNull String uid,
+      @Nonnull String type,
+      @Nonnull List<String> args) {
+    // is overridden by a tracker that collects errors
+    // default is to not collect errors
   }
 
   /*
@@ -590,10 +596,13 @@ public Progress(
     }
 
     public void addError(Error error) {
-      errors
-          .computeIfAbsent(error.getId(), key -> new ConcurrentHashMap<>())
-          .computeIfAbsent(error.getCode(), key2 -> new ConcurrentLinkedQueue<>())
-          .add(error);
+      Queue<Error> sameObjectAndCode =
+          errors
+              .computeIfAbsent(error.getId(), key -> new ConcurrentHashMap<>())
+              .computeIfAbsent(error.getCode(), key2 -> new ConcurrentLinkedQueue<>());
+      if (sameObjectAndCode.stream().noneMatch(e -> e.args.equals(error.args))) {
+        sameObjectAndCode.add(error);
+      }
     }
 
     public boolean hasErrors() {
@@ -619,13 +628,6 @@ final class Error {
     /** The type of the object identified by #id that has the error */
     @Nonnull @JsonProperty private final String type;
 
-    /**
-     * The row index in the payload of the import. This is the index in the list of objects of a
-     * single type. This means the same index occurs for each object type. For some imports this
-     * information is not available.
-     */
-    @CheckForNull @JsonProperty private final Integer index;
-
     /** The arguments used in the {@link #code}'s {@link ErrorCode#getMessage()} template */
     @Nonnull @JsonProperty private final List<String> args;
 
@@ -642,12 +644,10 @@ public Error(
         @Nonnull @JsonProperty("code") ErrorCode code,
         @Nonnull @JsonProperty("id") String id,
         @Nonnull @JsonProperty("type") String type,
-        @CheckForNull @JsonProperty("index") Integer index,
         @Nonnull @JsonProperty("args") List<String> args) {
       this.code = code;
       this.id = id;
       this.type = type;
-      this.index = index;
       this.args = args;
     }
   }

dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobRunErrorsParams.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2004-2023, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.hisp.dhis.scheduling;
+
+import java.util.Date;
+import java.util.List;
+import javax.annotation.CheckForNull;
+import lombok.Data;
+import lombok.experimental.Accessors;
+import org.hisp.dhis.common.OpenApi;
+import org.hisp.dhis.common.UID;
+import org.hisp.dhis.feedback.ErrorCode;
+import org.hisp.dhis.user.User;
+
+/**
+ * Query params when searching for {@link JobConfiguration}s with errors.
+ *
+ * <p>A match has to satisfy all filters (AND logic) but only one of the given codes or object
+ * {@link UID} (OR logic).
+ *
+ * <p>If any of the criteria is not defined it has no filter effect.
+ *
+ * @author Jan Bernitt
+ */
+@Data
+@Accessors(chain = true)
+public class JobRunErrorsParams {
+
+  @OpenApi.Ignore @CheckForNull private UID job;
+
+  /** The user that ran the job */
+  @OpenApi.Property({UID.class, User.class})
+  @CheckForNull
+  private UID user;
+
+  /** The earliest date the job ran that should be included */
+  @CheckForNull private Date from;
+
+  /** The latest date the job ran that should be included */
+  @CheckForNull private Date to;
+
+  /** The codes to select, any match combined */
+  @CheckForNull private List<ErrorCode> code;
+
+  /** The object with errors to select, any match combined */
+  @CheckForNull private List<UID> object;
+
+  /** The {@link JobType} with errors to select, any match combined */
+  @CheckForNull private List<JobType> type;
+}

dhis-2/dhis-services/dhis-service-core/pom.xml

@@ -117,6 +117,11 @@
       <groupId>org.hibernate</groupId>
       <artifactId>hibernate-core</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.vladmihalcea</groupId>
+      <artifactId>hibernate-types-52</artifactId>
+      <version>${hibernate-types.version}</version>
+    </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-core</artifactId>

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobConfigurationService.java

@@ -40,6 +40,7 @@
 import java.lang.reflect.Field;
 import java.lang.reflect.ParameterizedType;
 import java.lang.reflect.Type;
+import java.text.MessageFormat;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
@@ -48,8 +49,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import javax.annotation.Nonnull;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.beanutils.PropertyUtils;
@@ -60,9 +63,11 @@
 import org.hisp.dhis.common.NameableObject;
 import org.hisp.dhis.commons.util.TextUtils;
 import org.hisp.dhis.feedback.ConflictException;
+import org.hisp.dhis.feedback.ErrorCode;
 import org.hisp.dhis.fileresource.FileResource;
 import org.hisp.dhis.fileresource.FileResourceDomain;
 import org.hisp.dhis.fileresource.FileResourceService;
+import org.hisp.dhis.jsontree.*;
 import org.hisp.dhis.scheduling.JobType.Defaults;
 import org.hisp.dhis.schema.Property;
 import org.hisp.dhis.setting.SettingKey;
@@ -247,6 +252,54 @@ public List<JobConfiguration> getStaleConfigurations(int staleForSeconds) {
     return jobConfigurationStore.getStaleConfigurations(staleForSeconds);
   }
 
+  @Nonnull
+  @Override
+  @Transactional(readOnly = true)
+  public List<JsonObject> findJobRunErrors(@Nonnull JobRunErrorsParams params) {
+    Function<String, JsonObject> toObject =
+        json -> {
+          JsonObject obj = JsonMixed.of(json);
+          List<JsonNode> flatErrors = new ArrayList<>();
+          JsonObject errors = obj.getObject("errors");
+          errors
+              .node()
+              .members()
+              .forEach(
+                  byObject ->
+                      byObject
+                          .getValue()
+                          .members()
+                          .forEach(
+                              byCode ->
+                                  byCode
+                                      .getValue()
+                                      .elements()
+                                      .forEach(
+                                          error -> {
+                                            ErrorCode code =
+                                                ErrorCode.valueOf(
+                                                    JsonMixed.of(error).getString("code").string());
+                                            Object[] args =
+                                                JsonMixed.of(error)
+                                                    .getArray("args")
+                                                    .stringValues()
+                                                    .toArray(new String[0]);
+                                            String msg =
+                                                MessageFormat.format(code.getMessage(), args);
+                                            flatErrors.add(
+                                                error
+                                                    .extract()
+                                                    .addMembers(e -> e.addString("message", msg)));
+                                          })));
+          return JsonMixed.of(
+              errors
+                  .node()
+                  .replaceWith(
+                      JsonBuilder.createArray(arr -> flatErrors.forEach(arr::addElement))));
+        };
+    return jobConfigurationStore.findJobRunErrors(params).map(toObject).toList();
+  }
+
   @Override
   @Transactional(readOnly = true)
   public Map<String, Map<String, Property>> getJobParametersSchema() {

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerLoopService.java

@@ -285,7 +285,7 @@ private void updateProgress(@Nonnull String jobId) {
     try {
       JobProgress.Progress progress = job.getProgress();
       String errorCodes =
-          progress.getErrorCodes().stream().map(ErrorCode::name).collect(joining(" "));
+          progress.getErrorCodes().stream().map(ErrorCode::name).sorted().collect(joining(" "));
       jobConfigurationStore.updateProgress(
           jobId, jsonMapper.writeValueAsString(progress), errorCodes);
     } catch (JsonProcessingException ex) {