feat: Download is missing support for NAME, CODE, UID and ID (#16049)

* feat: Download is missing support for NAME, CODE, UID and ID

* redundant _name vars

* post code review impl

* post code review impl

* post code review impl - e2e tests

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/outlier/Order.java

@@ -28,12 +28,8 @@
 package org.hisp.dhis.analytics.outlier;
 
 import static org.hisp.dhis.analytics.common.ColumnHeader.ABSOLUTE_DEVIATION;
-import static org.hisp.dhis.analytics.common.ColumnHeader.ATTRIBUTE_OPTION_COMBO_NAME;
-import static org.hisp.dhis.analytics.common.ColumnHeader.CATEGORY_OPTION_COMBO_NAME;
-import static org.hisp.dhis.analytics.common.ColumnHeader.DIMENSION_NAME;
 import static org.hisp.dhis.analytics.common.ColumnHeader.LOWER_BOUNDARY;
 import static org.hisp.dhis.analytics.common.ColumnHeader.MEDIAN_ABS_DEVIATION;
-import static org.hisp.dhis.analytics.common.ColumnHeader.ORG_UNIT_NAME;
 import static org.hisp.dhis.analytics.common.ColumnHeader.STANDARD_DEVIATION;
 import static org.hisp.dhis.analytics.common.ColumnHeader.UPPER_BOUNDARY;
 import static org.hisp.dhis.analytics.common.ColumnHeader.ZSCORE;
@@ -51,10 +47,6 @@
 public enum Order {
   Z_SCORE(ZSCORE.getItem(), "z_score"),
   MODIFIED_ZSCORE(ColumnHeader.MODIFIED_ZSCORE.getItem(), "z_score"),
-  DE_NAME(DIMENSION_NAME.getItem(), "de_name"),
-  OU_NAME(ORG_UNIT_NAME.getItem(), "ou_name"),
-  COC_NAME(CATEGORY_OPTION_COMBO_NAME.getItem(), "coc_name"),
-  AOC_NAME(ATTRIBUTE_OPTION_COMBO_NAME.getItem(), "aoc_name"),
   VALUE("value", "value"),
   MEDIAN(ColumnHeader.MEDIAN.getItem(), "middle_value"),
   MEAN(ColumnHeader.MEAN.getItem(), "middle_value"),

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/outlier/data/OutlierQueryParams.java

@@ -38,6 +38,7 @@
 import org.hisp.dhis.analytics.QueryKey;
 import org.hisp.dhis.analytics.SortOrder;
 import org.hisp.dhis.analytics.outlier.Order;
+import org.hisp.dhis.common.IdScheme;
 
 /** Encapsulation of a web API request for outlier value detection. */
 @Data
@@ -72,6 +73,8 @@ public class OutlierQueryParams {
 
   private Integer maxResults;
 
+  private IdScheme outputIdScheme = IdScheme.UID;
+
   public boolean hasHeaders() {
     return headers != null && !headers.isEmpty();
   }
@@ -88,6 +91,7 @@ public String queryKey() {
     key.add(threshold);
     key.add(orderBy);
     key.add(sortOrder);
+    key.add(outputIdScheme);
 
     if (ds != null) {
       ds.forEach(e -> key.add("ds", "[" + e + "]"));

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/outlier/data/OutlierQueryParser.java

@@ -79,6 +79,7 @@ public OutlierRequest getFromQuery(OutlierQueryParams queryParams, boolean analy
             .analyzeOnly(analyzeOnly)
             .dataStartDate(queryParams.getDataStartDate())
             .dataEndDate(queryParams.getDataEndDate())
+            .outputIdScheme(queryParams.getOutputIdScheme())
             .queryKey(queryParams.queryKey());
 
     if (queryParams.getAlgorithm() != null) {

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/outlier/data/OutlierRequest.java

@@ -42,6 +42,7 @@
 import org.hisp.dhis.analytics.OutlierDetectionAlgorithm;
 import org.hisp.dhis.analytics.SortOrder;
 import org.hisp.dhis.analytics.outlier.Order;
+import org.hisp.dhis.common.IdScheme;
 import org.hisp.dhis.common.OrganisationUnitDescendants;
 import org.hisp.dhis.dataelement.DataElement;
 import org.hisp.dhis.organisationunit.OrganisationUnit;
@@ -82,6 +83,8 @@ public class OutlierRequest {
 
   @Default private double threshold = 3.0d;
 
+  @Default private IdScheme outputIdScheme = IdScheme.UID;
+
   private boolean analyzeOnly;
 
   private String explainOrderId;

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/outlier/service/AnalyticsOutlierService.java

@@ -48,6 +48,8 @@
 import static org.hisp.dhis.analytics.common.ColumnHeader.UPPER_BOUNDARY;
 import static org.hisp.dhis.analytics.common.ColumnHeader.VALUE;
 import static org.hisp.dhis.analytics.common.ColumnHeader.ZSCORE;
+import static org.hisp.dhis.common.IdentifiableProperty.CODE;
+import static org.hisp.dhis.common.IdentifiableProperty.ID;
 import static org.hisp.dhis.common.ValueType.NUMBER;
 import static org.hisp.dhis.common.ValueType.TEXT;
 
@@ -62,10 +64,15 @@
 import org.hisp.dhis.analytics.common.TableInfoReader;
 import org.hisp.dhis.analytics.outlier.data.Outlier;
 import org.hisp.dhis.analytics.outlier.data.OutlierRequest;
+import org.hisp.dhis.category.CategoryOptionCombo;
 import org.hisp.dhis.common.ExecutionPlan;
 import org.hisp.dhis.common.Grid;
 import org.hisp.dhis.common.GridHeader;
+import org.hisp.dhis.common.IdScheme;
+import org.hisp.dhis.common.IdentifiableObject;
+import org.hisp.dhis.common.IdentifiableObjectManager;
 import org.hisp.dhis.common.IllegalQueryException;
+import org.hisp.dhis.dataelement.DataElement;
 import org.hisp.dhis.feedback.ErrorCode;
 import org.hisp.dhis.feedback.ErrorMessage;
 import org.hisp.dhis.organisationunit.OrganisationUnit;
@@ -91,6 +98,8 @@ public class AnalyticsOutlierService {
 
   private final TableInfoReader tableInfoReader;
 
+  private final IdentifiableObjectManager idObjectManager;
+
   /**
    * Transform the incoming request into api response (json).
    *
@@ -103,7 +112,7 @@ public Grid getOutliers(OutlierRequest request) throws IllegalQueryException {
 
     Grid grid = new ListGrid();
     setHeaders(grid, request);
-    setMetaData(grid, request, outliers);
+    setMetaData(grid, outliers, request);
     setRows(grid, outliers, request);
 
     return grid;
@@ -247,14 +256,28 @@ private void setHeaders(Grid grid, OutlierRequest request) {
         new GridHeader(UPPER_BOUNDARY.getItem(), UPPER_BOUNDARY.getName(), NUMBER, false, false));
   }
 
-  private void setMetaData(Grid grid, OutlierRequest request, List<Outlier> outliers) {
+  /**
+   * The method add the metadata into the response grid.
+   *
+   * @param grid the {@link Grid}
+   * @param outliers the list of {@link Outlier}
+   * @param request the {@link OutlierRequest}
+   */
+  private void setMetaData(Grid grid, List<Outlier> outliers, OutlierRequest request) {
     grid.addMetaData("algorithm", request.getAlgorithm());
     grid.addMetaData("threshold", request.getThreshold());
     grid.addMetaData("orderBy", request.getOrderBy().getColumnName());
     grid.addMetaData("maxResults", request.getMaxResults());
     grid.addMetaData("count", outliers.size());
   }
 
+  /**
+   * The method add the rows into the response grid.
+   *
+   * @param grid the {@link Grid}
+   * @param outliers the list of {@link Outlier}
+   * @param request the {@link OutlierRequest}
+   */
   private void setRows(Grid grid, List<Outlier> outliers, OutlierRequest request) {
     outliers.forEach(
         v -> {
@@ -264,16 +287,27 @@ private void setRows(Grid grid, List<Outlier> outliers, OutlierRequest request)
           Collection<OrganisationUnit> roots = user != null ? user.getOrganisationUnits() : null;
 
           grid.addRow();
-          grid.addValue(v.getDx());
-          grid.addValue(v.getDxName());
+
+          IdentifiableObject object = idObjectManager.get(DataElement.class, v.getDx());
+          grid.addValue(getIdProperty(object, v.getDx(), request.getOutputIdScheme()));
+          grid.addValue(getIdProperty(object, v.getDx(), IdScheme.NAME));
+
           grid.addValue(v.getPe());
-          grid.addValue(v.getOu());
-          grid.addValue(v.getOuName());
+
+          object = idObjectManager.get(OrganisationUnit.class, v.getOu());
+          grid.addValue(getIdProperty(object, v.getOu(), request.getOutputIdScheme()));
+          grid.addValue(getIdProperty(object, v.getOu(), IdScheme.NAME));
+
           grid.addValue(ou.getParentNameGraph(roots, true));
-          grid.addValue(v.getCoc());
-          grid.addValue(v.getCocName());
-          grid.addValue(v.getAoc());
-          grid.addValue(v.getAocName());
+
+          object = idObjectManager.get(CategoryOptionCombo.class, v.getCoc());
+          grid.addValue(getIdProperty(object, v.getCoc(), request.getOutputIdScheme()));
+          grid.addValue(getIdProperty(object, v.getCoc(), IdScheme.NAME));
+
+          object = idObjectManager.get(CategoryOptionCombo.class, v.getAoc());
+          grid.addValue(getIdProperty(object, v.getAoc(), request.getOutputIdScheme()));
+          grid.addValue(getIdProperty(object, v.getAoc(), IdScheme.NAME));
+
           grid.addValue(v.getValue());
           grid.addValue(isModifiedZScore ? v.getMedian() : v.getMean());
           grid.addValue(v.getStdDev());
@@ -283,4 +317,28 @@ private void setRows(Grid grid, List<Outlier> outliers, OutlierRequest request)
           grid.addValue(v.getUpperBound());
         });
   }
+
+  /**
+   * The method retrieves ID Property. Depend on the IdScheme parameter it could be ID, UID, UUID,
+   * Code or Name. The default property is the UID.
+   *
+   * @param object the {@link IdentifiableObject}
+   * @param uid the {@link String}, default UID of the identifiable object (data element,
+   *     organisation unit, category option combo, etc...)
+   * @param idScheme the {@link IdScheme}
+   * @return ID Property of the identifiable object (ID, UID, UUID, Code or Name)
+   */
+  private String getIdProperty(IdentifiableObject object, String uid, IdScheme idScheme) {
+    if (object == null || idScheme == IdScheme.UID || idScheme == IdScheme.UUID) {
+      return uid;
+    }
+    if (idScheme.getIdentifiableProperty() == ID) {
+      return Long.toString(object.getId());
+    }
+    if (idScheme.getIdentifiableProperty() == CODE) {
+      return object.getCode();
+    }
+
+    return object.getName();
+  }
 }

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/outlier/service/AnalyticsZScoreOutlierDetector.java

@@ -123,14 +123,10 @@ private Outlier getOutlier(Calendar calendar, ResultSet rs) throws SQLException
 
     Outlier outlier = new Outlier();
     outlier.setDx(rs.getString("de_uid"));
-    outlier.setDxName(rs.getString("de_name"));
     outlier.setPe(isoPeriod);
     outlier.setOu(rs.getString("ou_uid"));
-    outlier.setOuName(rs.getString("ou_name"));
     outlier.setCoc(rs.getString("coc_uid"));
-    outlier.setCocName(rs.getString("coc_name"));
     outlier.setAoc(rs.getString("aoc_uid"));
-    outlier.setAocName(rs.getString("aoc_name"));
     outlier.setValue(rs.getDouble("value"));
 
     return outlier;

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/outlier/service/AnalyticsZScoreSqlStatementProcessor.java

@@ -135,10 +135,6 @@ private String getSqlStatement(OutlierRequest request, boolean withParams) {
             + "ax.ou as ou_uid, "
             + "ax.co as coc_uid, "
             + "ax.ao as aoc_uid, "
-            + "ax.de_name, "
-            + "ax.ou_name, "
-            + "ax.coc_name, "
-            + "ax.aoc_name, "
             + "ax.value, "
             + "ax.pestartdate as pe_start_date, "
             + "ax.petype as pt_name, "

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java

@@ -571,12 +571,6 @@ private List<AnalyticsTableColumn> getOutlierStatsColumns() {
             quote("attributeoptioncomboid"), INTEGER, NOT_NULL, "dv.attributeoptioncomboid"),
         new AnalyticsTableColumn(quote("dataelementid"), INTEGER, NOT_NULL, "dv.dataelementid")
             .withIndexColumns(List.of(quote("dataelementid"))),
-
-        // TODO: Remove all these name columns from here. Analytics tables should not have them.
-        new AnalyticsTableColumn(quote("de_name"), VARCHAR_255, "de.name"),
-        new AnalyticsTableColumn(quote("ou_name"), VARCHAR_255, "ou.name"),
-        new AnalyticsTableColumn(quote("coc_name"), VARCHAR_255, "co.name"),
-        new AnalyticsTableColumn(quote("aoc_name"), VARCHAR_255, "ao.name"),
         new AnalyticsTableColumn(quote("petype"), VARCHAR_255, "pt.name"),
         new AnalyticsTableColumn(quote("path"), VARCHAR_255, "ou.path"),
         // mean

dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/outlier/service/AnalyticsZscoreSqlStatementProcessorTest.java

@@ -86,7 +86,7 @@ void testGetSqlStatement() {
         .build();
     String sql = subject.getSqlStatement(builder.build());
     String expected =
-        "select * from (select ax.dx as de_uid, ax.ou as ou_uid, ax.co as coc_uid, ax.ao as aoc_uid, ax.de_name, ax.ou_name, ax.coc_name, ax.aoc_name, ax.value, ax.pestartdate as pe_start_date, ax.petype as pt_name,  ax.avg_middle_value as middle_value, ax.std_dev, ax.mad, abs(ax.value::double precision -  ax.avg_middle_value) as middle_value_abs_dev, (case when ax.std_dev = 0 then 0       else abs(ax.value::double precision -  ax.avg_middle_value ) / ax.std_dev        end) as z_score,  ax.avg_middle_value - (ax.std_dev * :threshold) as lower_bound,  ax.avg_middle_value + (ax.std_dev * :threshold) as upper_bound from analytics ax where dataelementid in  (:data_element_ids) and (ax.\"path\" like '/ouabcdefghA%' or ax.\"path\" like '/ouabcdefghB%') and ax.pestartdate >= :start_date and ax.peenddate <= :end_date) t1 where t1.z_score > :threshold order by middle_value_abs_dev desc limit :max_results ";
+        "select * from (select ax.dx as de_uid, ax.ou as ou_uid, ax.co as coc_uid, ax.ao as aoc_uid, ax.value, ax.pestartdate as pe_start_date, ax.petype as pt_name,  ax.avg_middle_value as middle_value, ax.std_dev, ax.mad, abs(ax.value::double precision -  ax.avg_middle_value) as middle_value_abs_dev, (case when ax.std_dev = 0 then 0       else abs(ax.value::double precision -  ax.avg_middle_value ) / ax.std_dev        end) as z_score,  ax.avg_middle_value - (ax.std_dev * :threshold) as lower_bound,  ax.avg_middle_value + (ax.std_dev * :threshold) as upper_bound from analytics ax where dataelementid in  (:data_element_ids) and (ax.\"path\" like '/ouabcdefghA%' or ax.\"path\" like '/ouabcdefghB%') and ax.pestartdate >= :start_date and ax.peenddate <= :end_date) t1 where t1.z_score > :threshold order by middle_value_abs_dev desc limit :max_results ";
     assertEquals(expected, sql);
   }
 

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/analytics/common/ColumnHeader.java

@@ -57,7 +57,7 @@ public enum ColumnHeader {
   EVENT_STATUS("eventstatus", "Event status"),
   DIMENSION("dx", "Data"),
   DIMENSION_NAME("dxname", "Data name"),
-  PERIOD("pe", "Event status"),
+  PERIOD("pe", "Period"),
   CATEGORY_OPTION_COMBO("coc", "Category option combo"),
   CATEGORY_OPTION_COMBO_NAME("cocname", "Category option combo name"),
   ATTRIBUTE_OPTION_COMBO("aoc", "Attribute option combo"),

dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/enrollment/aggregate/EnrollmentsAggregate3AutoTest.java

@@ -212,7 +212,8 @@ public void queryAggregatedenrollmentsbirthbcgdosegenderthisyearlastyearlevel4or
             .add("totalPages=false")
             .add("outputType=ENROLLMENT")
             .add(
-                "dimension=ou:DiszpKrYNg8,pe:THIS_YEAR;LAST_YEAR,A03MvHHogjR.bx6fsa0t90x,A03MvHHogjR.cejWyOfXge6");
+                "dimension=ou:DiszpKrYNg8,pe:THIS_YEAR;LAST_YEAR,A03MvHHogjR.bx6fsa0t90x,A03MvHHogjR.cejWyOfXge6")
+            .add("relativePeriodDate=2023-08-01");
 
     // When
     ApiResponse response = actions.aggregate().get("IpHINAT79UW", JSON, JSON, params);
@@ -276,7 +277,8 @@ public void queryAggregatedenrollmentsbirthgenderthisyearlevel4orgwithheaders()
             .add("displayProperty=SHORTNAME")
             .add("totalPages=false")
             .add("outputType=ENROLLMENT")
-            .add("dimension=ou:DiszpKrYNg8,pe:THIS_YEAR,A03MvHHogjR.cejWyOfXge6");
+            .add("dimension=ou:DiszpKrYNg8,pe:THIS_YEAR,A03MvHHogjR.cejWyOfXge6")
+            .add("relativePeriodDate=2023-08-01");
 
     // When
     ApiResponse response = actions.aggregate().get("IpHINAT79UW", JSON, JSON, params);