Merge branch 'master' into lars-dev

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java

@@ -49,8 +49,8 @@
 import static org.hisp.dhis.analytics.event.data.EnrollmentQueryHelper.getHeaderColumns;
 import static org.hisp.dhis.analytics.event.data.EnrollmentQueryHelper.getOrgUnitLevelColumns;
 import static org.hisp.dhis.analytics.event.data.EnrollmentQueryHelper.getPeriodColumns;
-import static org.hisp.dhis.analytics.table.JdbcEventAnalyticsTableManager.OU_GEOMETRY_COL_SUFFIX;
-import static org.hisp.dhis.analytics.table.JdbcEventAnalyticsTableManager.OU_NAME_COL_SUFFIX;
+import static org.hisp.dhis.analytics.table.AbstractEventJdbcTableManager.OU_GEOMETRY_COL_SUFFIX;
+import static org.hisp.dhis.analytics.table.AbstractEventJdbcTableManager.OU_NAME_COL_SUFFIX;
 import static org.hisp.dhis.analytics.util.AnalyticsUtils.replaceStringBetween;
 import static org.hisp.dhis.analytics.util.AnalyticsUtils.throwIllegalQueryEx;
 import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling;
@@ -375,8 +375,9 @@ private void addDimensionSelectColumns(
                     singleQuote(period.getIsoDate()) + " as " + period.getPeriodType().getName());
               } else {
                 throw new IllegalStateException(
-                    "Program indicator non-default boundary query must have "
-                        + "exactly one period, or no periods and a period filter");
+                    """
+                    Program indicator non-default boundary query must have \"
+                    exactly one period, or no periods and a period filter""");
               }
             });
   }
@@ -455,7 +456,6 @@ private ColumnAndAlias getColumnAndAlias(
     } else if (queryItem.getValueType() == ValueType.NUMBER && !isGroupByClause) {
       ColumnAndAlias columnAndAlias =
           getColumnAndAlias(queryItem, isAggregated, queryItem.getItemName());
-
       return ColumnAndAlias.ofColumnAndAlias(
           columnAndAlias.getColumn(),
           defaultIfNull(columnAndAlias.getAlias(), queryItem.getItemName()));
@@ -532,14 +532,10 @@ protected Optional<String> getAlias(QueryItem queryItem) {
   @Transactional(readOnly = true, propagation = REQUIRES_NEW)
   public Grid getAggregatedEventData(EventQueryParams params, Grid grid, int maxLimit) {
     String aggregateClause = getAggregateClause(params);
+    String columns = StringUtils.join(getSelectColumns(params, true), ",");
 
     String sql =
-        TextUtils.removeLastComma(
-            "select "
-                + aggregateClause
-                + " as value,"
-                + StringUtils.join(getSelectColumns(params, true), ",")
-                + " ");
+        TextUtils.removeLastComma("select " + aggregateClause + " as value," + columns + " ");
 
     // ---------------------------------------------------------------------
     // Criteria

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java

@@ -214,6 +214,10 @@ protected List<AnalyticsTableColumn> getColumnForAttribute(TrackedEntityAttribut
    */
   private List<AnalyticsTableColumn> getColumnForOrgUnitAttribute(
       TrackedEntityAttribute attribute) {
+    if (!sqlBuilder.supportsCorrelatedSubquery()) {
+      return List.of();
+    }
+
     Validate.isTrue(attribute.getValueType().isOrganisationUnit());
     List<AnalyticsTableColumn> columns = new ArrayList<>();
 
@@ -283,7 +287,7 @@ private String getOrgUnitSelectSubquery(TrackedEntityAttribute attribute, String
   protected String getAttributeValueJoinClause(Program program) {
     String template =
         """
-        left join ${trackedentityattributevalue} as ${uid} \
+        left join trackedentityattributevalue as ${uid} \
         on en.trackedentityid=${uid}.trackedentityid \
         and ${uid}.trackedentityattributeid = ${id}\s""";
 

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java

@@ -27,6 +27,9 @@
  */
 package org.hisp.dhis.analytics.table;
 
+import static org.hisp.dhis.analytics.AnalyticsTableType.ENROLLMENT;
+import static org.hisp.dhis.analytics.AnalyticsTableType.EVENT;
+import static org.hisp.dhis.analytics.AnalyticsTableType.TRACKED_ENTITY_INSTANCE;
 import static org.hisp.dhis.common.collection.CollectionUtils.emptyIfNull;
 import static org.hisp.dhis.scheduling.JobProgress.FailurePolicy.SKIP_STAGE;
 import static org.hisp.dhis.util.DateUtils.toLongDate;
@@ -51,6 +54,7 @@
 import org.hisp.dhis.setting.SystemSettings;
 import org.hisp.dhis.setting.SystemSettingsService;
 import org.hisp.dhis.system.util.Clock;
+import org.hisp.dhis.tablereplication.TableReplicationService;
 import org.springframework.stereotype.Service;
 
 /**
@@ -64,6 +68,8 @@ public class DefaultAnalyticsTableGenerator implements AnalyticsTableGenerator {
 
   private final ResourceTableService resourceTableService;
 
+  private final TableReplicationService tableReplicationService;
+
   private final SystemSettingsService settingsService;
 
   private final AnalyticsTableSettings settings;
@@ -84,23 +90,30 @@ public void generateAnalyticsTables(AnalyticsTableUpdateParams params0, JobProgr
     log.info("Found analytics table types: {}", getAvailableTableTypes());
     log.info("Analytics table update params: {}", params);
     log.info("Last successful analytics table update: {}", toLongDate(lastSuccessfulUpdate));
-    log.debug("Skipping table types: {}", skipTypes);
+    log.info("Analytics database: {}", settings.isAnalyticsDatabase());
+    log.info("Skipping table types: {}", skipTypes);
 
     progress.startingProcess(
         "Analytics table update process{}", (params.isLatestUpdate() ? " (latest partition)" : ""));
 
     if (!params.isSkipResourceTables() && !params.isLatestUpdate()) {
       generateResourceTablesInternal(progress);
 
-      if (settings.isAnalyticsDatabaseConfigured()) {
+      if (settings.isAnalyticsDatabase()) {
         log.info("Replicating resource tables in analytics database");
         resourceTableService.replicateAnalyticsResourceTables();
       }
     }
 
+    if (!params.isLatestUpdate() && settings.isAnalyticsDatabase()) {
+      if (!skipTypes.containsAll(Set.of(EVENT, ENROLLMENT, TRACKED_ENTITY_INSTANCE))) {
+        log.info("Replicating tracked entity attribute value table");
+        tableReplicationService.replicateTrackedEntityAttributeValue();
+      }
+    }
+
     for (AnalyticsTableService service : analyticsTableServices) {
       AnalyticsTableType tableType = service.getAnalyticsTableType();
-
       if (!skipTypes.contains(tableType)) {
         service.create(params, progress);
       }

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java

@@ -534,6 +534,10 @@ private String getSelectExpression(DataElement dataElement, String columnExpress
    * @return a list of {@link AnalyticsTableColumn}.
    */
   private List<AnalyticsTableColumn> getColumnForOrgUnitDataElement(DataElement dataElement) {
+    if (!sqlBuilder.supportsCorrelatedSubquery()) {
+      return List.of();
+    }
+
     List<AnalyticsTableColumn> columns = new ArrayList<>();
 
     if (isSpatialSupport()) {
@@ -612,12 +616,16 @@ private List<AnalyticsTableColumn> getAttributeColumns(Program program) {
    */
   private List<AnalyticsTableColumn> getColumnForAttributeWithLegendSet(
       TrackedEntityAttribute attribute) {
+    if (!sqlBuilder.supportsCorrelatedSubquery()) {
+      return List.of();
+    }
+
     String columnExpression = getColumnExpression(attribute.getValueType(), "value");
     String numericClause = getNumericClause("value");
     String query =
         """
         \s(select l.uid from ${maplegend} l \
-        inner join ${trackedentityattributevalue} av on l.startvalue <= ${selectClause} \
+        inner join trackedentityattributevalue av on l.startvalue <= ${selectClause} \
         and l.endvalue > ${selectClause} \
         and l.maplegendsetid=${legendSetId} \
         and av.trackedentityid=en.trackedentityid \
@@ -656,6 +664,10 @@ private List<AnalyticsTableColumn> getColumnForAttributeWithLegendSet(
    */
   private List<AnalyticsTableColumn> getColumnFromDataElementWithLegendSet(
       DataElement dataElement, String selectExpression, String dataFilterClause) {
+    if (!sqlBuilder.supportsCorrelatedSubquery()) {
+      return List.of();
+    }
+
     String query =
         """
         (select l.uid from ${maplegend} l \

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java

@@ -179,7 +179,7 @@ private List<AnalyticsTableColumn> getColumns(
     List<AnalyticsTableColumn> columns = new ArrayList<>(getFixedColumns());
     List<Program> programs = programsByTetUid.get(trackedEntityType.getUid());
 
-    if (isNotEmpty(programs)) {
+    if (isNotEmpty(programs) && sqlBuilder.supportsCorrelatedSubquery()) {
       String enrolledInProgramExpression =
           """
           \s exists(select 1 from ${enrollment} en_0 \
@@ -320,7 +320,7 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti
               sql.append(
                   replaceQualify(
                       """
-                      \s left join ${trackedentityattributevalue} ${teaUid} on ${teaUid}.trackedentityid=te.trackedentityid \
+                      \s left join trackedentityattributevalue ${teaUid} on ${teaUid}.trackedentityid=te.trackedentityid \
                       and ${teaUid}.trackedentityattributeid = ${teaId}""",
                       Map.of(
                           "teaUid", quote(tea.getUid()),

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTableColumn.java

@@ -80,10 +80,6 @@ public class AnalyticsTableColumn {
   /** Date of creation of the underlying data dimension. */
   private final Date created;
 
-  //  // -------------------------------------------------------------------------
-  //  // Logic
-  //  // -------------------------------------------------------------------------
-
   /** Indicates whether this column is not null. */
   public boolean isNotNull() {
     return Nullable.NOT_NULL == nullable;

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/setting/AnalyticsTableSettings.java

@@ -111,7 +111,7 @@ public PeriodSource getPeriodSource() {
    *
    * @return true if an analytics database instance is configured.
    */
-  public boolean isAnalyticsDatabaseConfigured() {
+  public boolean isAnalyticsDatabase() {
     return config.isAnalyticsDatabaseConfigured();
   }
 

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/trackedentity/query/context/sql/SqlQueryBuilders.java

@@ -61,22 +61,22 @@ select json_agg(json_build_object(
           // incidentDate is kept to support a deprecated field, will be removed when FE will only
           // use occurreddate
           """
-              select json_agg(
-                         json_build_object(
-                             'programUid', en.program,
-                             'enrollmentUid', en.enrollment,
-                             'enrollmentDate', en.enrollmentdate,
-                             'incidentDate', en.occurreddate,
-                             'occurredDate', en.occurreddate,
-                             'completedDate', en.completeddate,
-                             'orgUnitUid', en.ou,
-                             'orgUnitName', en.ouname,
-                             'orgUnitCode', en.oucode,
-                             'orgUnitNameHierarchy', en.ounamehierarchy,
-                             'enrollmentStatus', en.enrollmentstatus,
-                             'events', ${eventQuery}))
-                    from analytics_te_enrollment_${trackedEntityType} en
-                    where en.trackedentity = t_1.trackedentity""",
+          select json_agg(
+               json_build_object(
+                   'programUid', en.program,
+                   'enrollmentUid', en.enrollment,
+                   'enrollmentDate', en.enrollmentdate,
+                   'incidentDate', en.occurreddate,
+                   'occurredDate', en.occurreddate,
+                   'completedDate', en.completeddate,
+                   'orgUnitUid', en.ou,
+                   'orgUnitName', en.ouname,
+                   'orgUnitCode', en.oucode,
+                   'orgUnitNameHierarchy', en.ounamehierarchy,
+                   'enrollmentStatus', en.enrollmentstatus,
+                   'events', ${eventQuery}))
+          from analytics_te_enrollment_${trackedEntityType} en
+          where en.trackedentity = t_1.trackedentity""",
           Map.of("eventQuery", coalesceToEmptyArray(EVENT_QUERY)));
 
   private static final String JSON_AGGREGATION_QUERY = coalesceToEmptyArray(ENROLLMENT_QUERY);

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/DisplayNameUtils.java

@@ -124,22 +124,22 @@ public static String getDisplayName(
         isNotEmpty.apply(surname),
         isNotEmpty.apply(firstName),
         isEmpty.apply(username),
-        formatNames(sqlBuilder, surname, "', '", firstName),
+        sqlBuilder.concat(surname, "', '", firstName),
 
         // FirstName and Username
         isEmpty.apply(surname),
         isNotEmpty.apply(firstName),
         isNotEmpty.apply(username),
-        formatNames(sqlBuilder, firstName, "' ('", username, "')'"),
+        sqlBuilder.concat(firstName, "' ('", username, "')'"),
 
         // Surname and Username
         isNotEmpty.apply(surname),
         isEmpty.apply(firstName),
         isNotEmpty.apply(username),
-        formatNames(sqlBuilder, surname, "' ('", username, "')'"),
+        sqlBuilder.concat(surname, "' ('", username, "')'"),
 
         // All fields
-        formatNames(sqlBuilder, surname, "', '", firstName, "' ('", username, "')'"),
+        sqlBuilder.concat(surname, "', '", firstName, "' ('", username, "')'"),
         columnAlias);
   }
 
@@ -149,8 +149,4 @@ private static String extractJsonValue(
     String jsonExtracted = sqlBuilder.jsonExtract(json, path);
     return sqlBuilder.trim(jsonExtracted);
   }
-
-  private static String formatNames(SqlBuilder sqlBuilder, String... elements) {
-    return sqlBuilder.concat(elements);
-  }
 }

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/DefaultTableReplicationService.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2024, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.hisp.dhis.tablereplication;
+
+import java.util.List;
+import lombok.RequiredArgsConstructor;
+import org.hisp.dhis.db.model.Column;
+import org.hisp.dhis.db.model.DataType;
+import org.hisp.dhis.db.model.Table;
+import org.hisp.dhis.db.model.constraint.Nullable;
+import org.springframework.stereotype.Service;
+
+@Service
+@RequiredArgsConstructor
+public class DefaultTableReplicationService implements TableReplicationService {
+  private static final Table TABLE_TRACKED_ENTIY_ATTRIBUTE_VALUE =
+      new Table(
+          "trackedentityattributevalue",
+          List.of(
+              new Column("trackedentityid", DataType.BIGINT, Nullable.NOT_NULL),
+              new Column("trackedentityattributeid", DataType.BIGINT, Nullable.NOT_NULL),
+              new Column("value", DataType.TEXT, Nullable.NULL)),
+          List.of("trackedentityid", "trackedentityattributeid"));
+
+  private final TableReplicationStore store;
+
+  @Override
+  public void replicateTrackedEntityAttributeValue() {
+    store.replicateAnalyticsDatabaseTable(TABLE_TRACKED_ENTIY_ATTRIBUTE_VALUE);
+  }
+}

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/TableReplicationService.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2004-2024, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.hisp.dhis.tablereplication;
+
+public interface TableReplicationService {
+  void replicateTrackedEntityAttributeValue();
+}

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/JdbcTableReplicationStore.java → dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/jdbc/JdbcTableReplicationStore.java

@@ -25,13 +25,14 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-package org.hisp.dhis.tablereplication;
+package org.hisp.dhis.tablereplication.jdbc;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.hisp.dhis.db.model.Table;
 import org.hisp.dhis.db.sql.SqlBuilder;
 import org.hisp.dhis.system.util.Clock;
+import org.hisp.dhis.tablereplication.TableReplicationStore;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Service;