Merge branch 'master' into DHIS2-13779_TEI_PI_aggregation

.github/workflows/deploy-instance.yml

@@ -21,9 +21,6 @@ jobs:
       INSTANCE_HOST: 'https://dev.im.dhis2.org'
       INSTANCE_NAME: pr-${{ github.event.pull_request.number }}
     steps:
-      - name: Dump context
-        uses: crazy-max/ghaction-dump-context@v2
-
       - name: Wait for API tests
         # Using this fork of the upstream https://github.com/lewagon/wait-on-check-action,
         # as it will filter out and check only the latest run of a workflow when checking for the allowed conclusions,

dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java

@@ -146,6 +146,9 @@ public enum ErrorCode {
   E2206("Max results exceeds the allowed max limit: `{0}`"),
   E2207("Data start date must be before data end date"),
   E2208("Non-numeric data values encountered during outlier value detection"),
+  E2209("Data start date not allowed"),
+  E2210("Data end date not allowed"),
+  E2211("Algorithm min-max values not allowed"),
 
   /* Followup analysis */
   E2300("At least one data element or data set must be specified"),

dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/AnalyticsJobParameters.java

@@ -55,15 +55,18 @@ public class AnalyticsJobParameters implements JobParameters {
   private Set<String> skipPrograms = new HashSet<>();
 
   @JsonProperty private boolean skipResourceTables = false;
+  @JsonProperty private boolean skipOutliers = false;
 
   public AnalyticsJobParameters(
       Integer lastYears,
       Set<AnalyticsTableType> skipTableTypes,
       Set<String> skipPrograms,
-      boolean skipResourceTables) {
+      boolean skipResourceTables,
+      boolean skipOutliers) {
     this.lastYears = lastYears;
     this.skipTableTypes = skipTableTypes;
     this.skipPrograms = skipPrograms;
     this.skipResourceTables = skipResourceTables;
+    this.skipOutliers = skipOutliers;
   }
 }

dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/ContinuousAnalyticsJobParameters.java

@@ -52,10 +52,17 @@ public class ContinuousAnalyticsJobParameters implements JobParameters {
   /** The types of analytics tables for which to skip update. */
   @JsonProperty private Set<AnalyticsTableType> skipTableTypes = new HashSet<>();
 
+  /** Outliers statistics columns of Analytics tables will be skipped. */
+  @JsonProperty private Boolean skipOutliers = false;
+
   public ContinuousAnalyticsJobParameters(
-      Integer fullUpdateHourOfDay, Integer lastYears, Set<AnalyticsTableType> skipTableTypes) {
+      Integer fullUpdateHourOfDay,
+      Integer lastYears,
+      Set<AnalyticsTableType> skipTableTypes,
+      Boolean skipOutliers) {
     this.fullUpdateHourOfDay = fullUpdateHourOfDay;
     this.lastYears = lastYears;
     this.skipTableTypes = skipTableTypes;
+    this.skipOutliers = skipOutliers;
   }
 }

dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/SqlExceptionUtils.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2004-2023, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.hisp.dhis.util;
+
+import java.sql.SQLException;
+import java.util.Optional;
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class SqlExceptionUtils {
+  public static final String ERR_MSG_TABLE_NOT_EXISTING =
+      "Query failed, likely because the requested analytics table does not exist: ";
+
+  public static final String ERR_MSG_SQL_SYNTAX_ERROR =
+      "An error occurred during the execution of an analytics query: ";
+
+  public static final String ERR_MSG_SILENT_FALLBACK =
+      "An exception occurred - silently fallback since it's multiple analytics query: ";
+
+  /**
+   * Utility method to detect if the {@link SQLException} refers to a missing relation in the
+   * database.
+   *
+   * @param ex a {@link SQLException} to analyze
+   * @return true if the error is a missing relation error, false otherwise
+   */
+  public static boolean relationDoesNotExist(SQLException ex) {
+    if (ex != null) {
+      return Optional.of(ex).map(SQLException::getSQLState).filter("42P01"::equals).isPresent();
+    }
+
+    return false;
+  }
+}

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableUpdateParams.java

@@ -55,7 +55,10 @@ public class AnalyticsTableUpdateParams {
   private Integer lastYears;
 
   /** Indicates whether to skip update of resource tables. */
-  boolean skipResourceTables;
+  private boolean skipResourceTables;
+
+  /** Indicates whether to skip update of analytics tables, outliers stats columns. */
+  private boolean skipOutliers;
 
   /** Analytics table types to skip. */
   private Set<AnalyticsTableType> skipTableTypes = new HashSet<>();
@@ -101,6 +104,10 @@ public boolean isSkipResourceTables() {
     return skipResourceTables;
   }
 
+  public boolean isSkipOutliers() {
+    return skipOutliers;
+  }
+
   public Set<AnalyticsTableType> getSkipTableTypes() {
     return skipTableTypes;
   }
@@ -149,6 +156,7 @@ public String toString() {
         .add("skip resource tables", skipResourceTables)
         .add("skip table types", skipTableTypes)
         .add("skip programs", skipPrograms)
+        .add("skip outliers statistics", skipOutliers)
         .add("start time", DateUtils.getLongDateString(startTime))
         .toString();
   }
@@ -186,6 +194,7 @@ public AnalyticsTableUpdateParams instance() {
 
     params.lastYears = this.lastYears;
     params.skipResourceTables = this.skipResourceTables;
+    params.skipOutliers = this.skipOutliers;
     params.skipTableTypes = new HashSet<>(this.skipTableTypes);
     params.skipPrograms = new HashSet<>(this.skipPrograms);
     params.jobId = this.jobId;
@@ -230,6 +239,11 @@ public Builder withSkipResourceTables(boolean skipResourceTables) {
       return this;
     }
 
+    public Builder withSkipOutliers(boolean skipOutliers) {
+      this.params.skipOutliers = skipOutliers;
+      return this;
+    }
+
     public Builder withSkipTableTypes(Set<AnalyticsTableType> skipTableTypes) {
       this.params.skipTableTypes = skipTableTypes;
       return this;

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java

@@ -47,15 +47,15 @@
 import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quoteAliasCommaSeparate;
 import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quoteWithFunction;
 import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quotedListOf;
-import static org.hisp.dhis.analytics.util.AnalyticsUtils.ERR_MSG_SILENT_FALLBACK;
 import static org.hisp.dhis.analytics.util.AnalyticsUtils.throwIllegalQueryEx;
 import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling;
 import static org.hisp.dhis.common.DimensionalObject.DIMENSION_SEP;
 import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids;
 import static org.hisp.dhis.commons.collection.CollectionUtils.concat;
 import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString;
-import static org.hisp.dhis.dxf2.webmessage.WebMessageUtils.relationDoesNotExist;
 import static org.hisp.dhis.util.DateUtils.getMediumDateString;
+import static org.hisp.dhis.util.SqlExceptionUtils.ERR_MSG_SILENT_FALLBACK;
+import static org.hisp.dhis.util.SqlExceptionUtils.relationDoesNotExist;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;