From a1261b9949a3018892ec854e09fb665f0e610849 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 09:31:26 +0100 Subject: [PATCH 01/48] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 19 ++++++++++++++++++- .../JdbcEnrollmentAnalyticsTableManager.java | 5 +---- .../table/JdbcEventAnalyticsTableManager.java | 14 -------------- ...dbcTrackedEntityAnalyticsTableManager.java | 2 -- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 0a60a2d5042f..8a5fb3d1532a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -27,6 +27,7 @@ */ package org.hisp.dhis.analytics.table; +import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getClosingParentheses; import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; @@ -170,6 +171,22 @@ private String getSelectExpression(ValueType valueType, String columnExpression, } } + /** + * For numeric and date value types, returns a data filter clause for checking whether the value + * is valid according to the value type. For other value types, returns the empty string. + * + * @param attribute the {@link TrackedEntityAttribute}. + * @return a data filter clause. + */ + protected String getDataFilterClause(TrackedEntityAttribute attribute) { + if (attribute.isNumericType()) { + return getNumericClause(); + } else if (attribute.isDateType()) { + return getDateClause(); + } + return EMPTY; + } + /** * Returns a cast expression which includes a value filter for the given value type. * @@ -178,7 +195,7 @@ private String getSelectExpression(ValueType valueType, String columnExpression, * @param dataType the SQL data type. * @return a cast and validate expression. */ - String getCastExpression(String columnExpression, String filterRegex, String dataType) { + protected String getCastExpression(String columnExpression, String filterRegex, String dataType) { String filter = sqlBuilder.regexpMatch(columnExpression, filterRegex); return String.format( "case when %s then cast(%s as %s) else null end", filter, columnExpression, dataType); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index f78f2aad73f6..779538490b34 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -201,10 +201,7 @@ private List getTrackedEntityAttributeColumns(Program prog for (TrackedEntityAttribute attribute : program.getNonConfidentialTrackedEntityAttributes()) { DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); - String dataClause = - attribute.isNumericType() - ? getNumericClause() - : attribute.isDateType() ? getDateClause() : ""; + String dataClause = getDataFilterClause(attribute); String select = getSelectExpressionForAttribute(attribute.getValueType(), "value"); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index aa63a4d5ed99..4bf867f12895 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -788,20 +788,6 @@ private String getDataFilterClause(DataElement dataElement) { return EMPTY; } - /** - * For numeric and date value types, returns a data filter clause for checking whether the value - * is valid according to the value type. For other value types, returns the empty string. - * - * @param attribute the {@link TrackedEntityAttribute}. - * @return a data filter clause. - */ - private String getDataFilterClause(TrackedEntityAttribute attribute) { - if (attribute.isNumericType()) { - return getNumericClause(); - } - return attribute.isDateType() ? getDateClause() : EMPTY; - } - /** * Returns a list of years for which data exist. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index 4071615a7e82..333602b5adf8 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -247,8 +247,6 @@ private Stream getAllTrackedEntityAttributes( /** * Returns the select clause, potentially with a cast statement, based on the given value type. - * (this method is an adapted version of {@link - * JdbcEventAnalyticsTableManager#getSelectExpression(ValueType, String)}) * * @param valueType the value type to represent as database column type. */ From bf145f655097568214cba4e06714c5d384b7eee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 09:32:54 +0100 Subject: [PATCH 02/48] fix: Update code --- .../table/JdbcEnrollmentAnalyticsTableManager.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index 779538490b34..54c11184b977 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -202,25 +202,25 @@ private List getTrackedEntityAttributeColumns(Program prog for (TrackedEntityAttribute attribute : program.getNonConfidentialTrackedEntityAttributes()) { DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); String dataClause = getDataFilterClause(attribute); - String select = getSelectExpressionForAttribute(attribute.getValueType(), "value"); + String selectExpression = getSelectExpressionForAttribute(attribute.getValueType(), "value"); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); String sql = replaceQualify( """ - (select ${select} from ${trackedentityattributevalue} \ + (select ${selectExpression} from ${trackedentityattributevalue} \ where trackedentityid=en.trackedentityid \ and trackedentityattributeid=${attributeId}\ ${dataClause})${closingParentheses} as ${attributeUid}""", Map.of( - "select", - select, + "selectExpression", + selectExpression, "attributeId", String.valueOf(attribute.getId()), "dataClause", dataClause, "closingParentheses", - getClosingParentheses(select), + getClosingParentheses(selectExpression), "attributeUid", quote(attribute.getUid()))); columns.add( From 2c933d82232e88ec41a6c1e8b6c2e0712228f598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 09:33:41 +0100 Subject: [PATCH 03/48] fix: Update code --- .../analytics/table/AbstractEventJdbcTableManager.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 8a5fb3d1532a..04571cbfef67 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -240,23 +240,23 @@ protected void populateTableInternal(AnalyticsTablePartition partition, String f * The select subquery statement. * * @param attribute the {@link TrackedEntityAttribute}. - * @param columnExpression the column expression. + * @param selectExpression the column expression. * @param dataFilterClause the data filter clause. * @return a select statement. */ protected String getSelectSubquery( - TrackedEntityAttribute attribute, String columnExpression, String dataFilterClause) { + TrackedEntityAttribute attribute, String selectExpression, String dataFilterClause) { return replaceQualify( """ - (select ${columnExpression} from ${trackedentityattributevalue} \ + (select ${selectExpression} from ${trackedentityattributevalue} \ where trackedentityid=en.trackedentityid \ and trackedentityattributeid=${attributeId}${dataFilterClause})\ ${closingParentheses} as ${attributeUid}""", Map.of( - "columnExpression", columnExpression, + "selectExpression", selectExpression, "dataFilterClause", dataFilterClause, "attributeId", String.valueOf(attribute.getId()), - "closingParentheses", getClosingParentheses(columnExpression), + "closingParentheses", getClosingParentheses(selectExpression), "attributeUid", quote(attribute.getUid()))); } } From 754da7fbd3c7be529cb1df8de055ef920f966f28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 09:33:48 +0100 Subject: [PATCH 04/48] fix: Update code --- .../dhis/analytics/table/AbstractEventJdbcTableManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 04571cbfef67..1695889fa6a3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -240,7 +240,7 @@ protected void populateTableInternal(AnalyticsTablePartition partition, String f * The select subquery statement. * * @param attribute the {@link TrackedEntityAttribute}. - * @param selectExpression the column expression. + * @param selectExpression the select expression. * @param dataFilterClause the data filter clause. * @return a select statement. */ From 5e164c7ec38e93cb69440e292b8ee69a5f24da5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 09:39:45 +0100 Subject: [PATCH 05/48] fix: Update code --- .../JdbcEnrollmentAnalyticsTableManager.java | 26 +++---------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index 54c11184b977..4d8271cc605d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -28,7 +28,6 @@ package org.hisp.dhis.analytics.table; import static org.hisp.dhis.analytics.table.model.Skip.SKIP; -import static org.hisp.dhis.analytics.util.AnalyticsUtils.getClosingParentheses; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.util.DateUtils.toLongDate; @@ -150,7 +149,7 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti String fromClause = replaceQualify( """ - \s from ${enrollment} en \ + \sfrom ${enrollment} en \ inner join ${program} pr on en.programid=pr.programid \ left join ${trackedentity} te on en.trackedentityid=te.trackedentityid and te.deleted = false \ left join ${organisationunit} registrationou on te.organisationunitid=registrationou.organisationunitid \ @@ -201,28 +200,11 @@ private List getTrackedEntityAttributeColumns(Program prog for (TrackedEntityAttribute attribute : program.getNonConfidentialTrackedEntityAttributes()) { DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); - String dataClause = getDataFilterClause(attribute); String selectExpression = getSelectExpressionForAttribute(attribute.getValueType(), "value"); + String dataFilterClause = getDataFilterClause(attribute); + String sql = getSelectSubquery(attribute, selectExpression, dataFilterClause); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); - String sql = - replaceQualify( - """ - (select ${selectExpression} from ${trackedentityattributevalue} \ - where trackedentityid=en.trackedentityid \ - and trackedentityattributeid=${attributeId}\ - ${dataClause})${closingParentheses} as ${attributeUid}""", - Map.of( - "selectExpression", - selectExpression, - "attributeId", - String.valueOf(attribute.getId()), - "dataClause", - dataClause, - "closingParentheses", - getClosingParentheses(selectExpression), - "attributeUid", - quote(attribute.getUid()))); columns.add( AnalyticsTableColumn.builder() .name(attribute.getUid()) @@ -234,7 +216,7 @@ private List getTrackedEntityAttributeColumns(Program prog if (attribute.getValueType().isOrganisationUnit()) { String fromTypeSql = "ou.name from organisationunit ou where ou.uid = (select value"; - String ouNameSql = getSelectSubquery(attribute, fromTypeSql, dataClause); + String ouNameSql = getSelectSubquery(attribute, fromTypeSql, dataFilterClause); columns.add( AnalyticsTableColumn.builder() From 4edfd49d71bafad0ede753353659473e189337d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 09:50:23 +0100 Subject: [PATCH 06/48] fix: Update code --- .../analytics/table/AbstractEventJdbcTableManager.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 1695889fa6a3..3b3b67598504 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -149,11 +149,9 @@ private String getSelectExpression(ValueType valueType, String columnExpression, } else if (valueType.isInteger()) { return getCastExpression(columnExpression, NUMERIC_REGEXP, sqlBuilder.dataTypeBigInt()); } else if (valueType.isBoolean()) { - return "case when " - + columnExpression - + " = 'true' then 1 when " - + columnExpression - + " = 'false' then 0 else null end"; + return String.format( + "case when %1$s = 'true' then 1 when %1$s = 'false' then 0 else null end", + columnExpression); } else if (valueType.isDate()) { return getCastExpression(columnExpression, DATE_REGEXP, sqlBuilder.dataTypeTimestamp()); } else if (valueType.isGeo() && isSpatialSupport()) { From c82b6f74f382a7d7e9b756b5d05be61e079ac6b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 09:52:18 +0100 Subject: [PATCH 07/48] fix: Update code --- .../analytics/table/AbstractEventJdbcTableManager.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 3b3b67598504..c3dc3b8a729a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -155,9 +155,10 @@ private String getSelectExpression(ValueType valueType, String columnExpression, } else if (valueType.isDate()) { return getCastExpression(columnExpression, DATE_REGEXP, sqlBuilder.dataTypeTimestamp()); } else if (valueType.isGeo() && isSpatialSupport()) { - return "ST_GeomFromGeoJSON('{\"type\":\"Point\", \"coordinates\":' || (" - + columnExpression - + ") || ', \"crs\":{\"type\":\"name\", \"properties\":{\"name\":\"EPSG:4326\"}}}')"; + return String.format( + """ + ST_GeomFromGeoJSON('{\type":"Point", "coordinates":' || (%s) || ', "crs":{"type":"name", "properties":{"name":"EPSG:4326"}}}')""", + columnExpression); } else if (valueType.isOrganisationUnit()) { String ouClause = isTea From aa7625b41f09467e8df880a05e2954f224d32f03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 10:00:23 +0100 Subject: [PATCH 08/48] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 2 +- .../AbstractEventJdbcTableManagerTest.java | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index c3dc3b8a729a..6639da1d405f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -157,7 +157,7 @@ private String getSelectExpression(ValueType valueType, String columnExpression, } else if (valueType.isGeo() && isSpatialSupport()) { return String.format( """ - ST_GeomFromGeoJSON('{\type":"Point", "coordinates":' || (%s) || ', "crs":{"type":"name", "properties":{"name":"EPSG:4326"}}}')""", + ST_GeomFromGeoJSON('{"type":"Point", "coordinates":' || (%s) || ', "crs":{"type":"name", "properties":{"name":"EPSG:4326"}}}')""", columnExpression); } else if (valueType.isOrganisationUnit()) { String ouClause = diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java index 80cf1dbee0bf..035d7b5b0c0c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java @@ -28,10 +28,12 @@ package org.hisp.dhis.analytics.table; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.db.sql.PostgreSqlBuilder; import org.hisp.dhis.db.sql.SqlBuilder; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; @@ -45,6 +47,11 @@ class AbstractEventJdbcTableManagerTest { @InjectMocks private JdbcEventAnalyticsTableManager manager; + @BeforeEach + public void beforeEach() { + manager.spatialSupport = true; + } + @Test void testGetCastExpression() { String expected = @@ -101,4 +108,19 @@ void testGetSelectExpressionText() { assertEquals(expected, actual); } + + @Test + void testGetSelectExpressionGeometry() { + when(manager.isSpatialSupport()).thenReturn(Boolean.TRUE); + + String expected = + """ + ST_GeomFromGeoJSON('{"type":"Point", "coordinates":' || (eventdatavalues #>> '{C6bh7GevJfH, value}') || ', "crs":{"type":"name", "properties":{"name":"EPSG:4326"}}}')"""; + + String actual = + manager.getSelectExpression( + ValueType.GEOJSON, "eventdatavalues #>> '{C6bh7GevJfH, value}'"); + + assertEquals(expected, actual); + } } From c0d12daa8c57549b614498243d5fb37c46e60ef6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 10:02:01 +0100 Subject: [PATCH 09/48] fix: Update code --- .../table/AbstractEventJdbcTableManagerTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java index 035d7b5b0c0c..68decf6d5200 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java @@ -83,6 +83,19 @@ then cast(eventdatavalues #>> '{GieVkTxp4HH, value}' as double precision) \ assertEquals(expected, actual); } + @Test + void testGetSelectExpressionBoolean() { + String expected = + """ + case when eventdatavalues #>> '{Xl3voRRcmpo, value}' = 'true' then 1 when eventdatavalues #>> '{Xl3voRRcmpo, value}' = 'false' then 0 else null end"""; + + String actual = + manager.getSelectExpression( + ValueType.BOOLEAN, "eventdatavalues #>> '{Xl3voRRcmpo, value}'"); + + assertEquals(expected, actual); + } + @Test void testGetSelectExpressionDate() { String expected = From 057cdf0e36dd42b2ec07f331d636ef5e12c57b8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 10:08:03 +0100 Subject: [PATCH 10/48] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 4bf867f12895..34a273f3d9ee 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -672,8 +672,8 @@ private List getColumnsForOrgUnitTrackedEntityAttribute( qualifyVariables("from ${organisationunit} ou where ou.uid = (select value"); if (isSpatialSupport()) { - String fromType = "ou.geometry " + fromClause; - String geoSql = getSelectSubquery(attribute, fromType, dataFilterClause); + String selectExpression = "ou.geometry " + fromClause; + String geoSql = getSelectSubquery(attribute, selectExpression, dataFilterClause); columns.add( AnalyticsTableColumn.builder() .name((attribute.getUid() + OU_GEOMETRY_COL_SUFFIX)) From 60c8ad648abcc325a94cbae2234116d25d7c004e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 10:10:12 +0100 Subject: [PATCH 11/48] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 34a273f3d9ee..c4dfdb463d1f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -673,13 +673,13 @@ private List getColumnsForOrgUnitTrackedEntityAttribute( if (isSpatialSupport()) { String selectExpression = "ou.geometry " + fromClause; - String geoSql = getSelectSubquery(attribute, selectExpression, dataFilterClause); + String ouGeoSql = getSelectSubquery(attribute, selectExpression, dataFilterClause); columns.add( AnalyticsTableColumn.builder() .name((attribute.getUid() + OU_GEOMETRY_COL_SUFFIX)) .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(GEOMETRY) - .selectExpression(geoSql) + .selectExpression(ouGeoSql) .indexType(IndexType.GIST) .build()); } From ee780bbbe56f30cc93f5cf66de1414c6d027409e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 10:17:24 +0100 Subject: [PATCH 12/48] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 86 +++++++++---------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index c4dfdb463d1f..c4008fe6f762 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -600,7 +600,7 @@ private List getColumnForAttribute(TrackedEntityAttribute Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); if (attribute.getValueType().isOrganisationUnit()) { - columns.addAll(getColumnsForOrgUnitTrackedEntityAttribute(attribute, dataFilterClause)); + columns.addAll(getColumnForOrgUnitTrackedEntityAttribute(attribute, dataFilterClause)); } columns.add( @@ -615,6 +615,48 @@ private List getColumnForAttribute(TrackedEntityAttribute return columns; } + /** + * Returns a list of columns based on the given attribute. + * + * @param attribute the {@link TrackedEntityAttribute}. + * @param dataFilterClause the data filter clause. + * @return a list of {@link AnalyticsTableColumn}. + */ + private List getColumnForOrgUnitTrackedEntityAttribute( + TrackedEntityAttribute attribute, String dataFilterClause) { + List columns = new ArrayList<>(); + + String fromClause = + qualifyVariables("from ${organisationunit} ou where ou.uid = (select value"); + + if (isSpatialSupport()) { + String selectExpression = "ou.geometry " + fromClause; + String ouGeoSql = getSelectSubquery(attribute, selectExpression, dataFilterClause); + columns.add( + AnalyticsTableColumn.builder() + .name((attribute.getUid() + OU_GEOMETRY_COL_SUFFIX)) + .dimensionType(AnalyticsDimensionType.DYNAMIC) + .dataType(GEOMETRY) + .selectExpression(ouGeoSql) + .indexType(IndexType.GIST) + .build()); + } + + String fromTypeSql = "ou.name " + fromClause; + String ouNameSql = getSelectSubquery(attribute, fromTypeSql, dataFilterClause); + + columns.add( + AnalyticsTableColumn.builder() + .name((attribute.getUid() + OU_NAME_COL_SUFFIX)) + .dimensionType(AnalyticsDimensionType.DYNAMIC) + .dataType(TEXT) + .selectExpression(ouNameSql) + .skipIndex(SKIP) + .build()); + + return columns; + } + /** * Returns a list of columns based on the given attribute with legend set. * @@ -657,48 +699,6 @@ private List getColumnForAttributeWithLegendSet( .toList(); } - /** - * Returns a list of columns based on the given attribute. - * - * @param attribute the {@link TrackedEntityAttribute}. - * @param dataFilterClause the data filter clause. - * @return a list of {@link AnalyticsTableColumn}. - */ - private List getColumnsForOrgUnitTrackedEntityAttribute( - TrackedEntityAttribute attribute, String dataFilterClause) { - List columns = new ArrayList<>(); - - String fromClause = - qualifyVariables("from ${organisationunit} ou where ou.uid = (select value"); - - if (isSpatialSupport()) { - String selectExpression = "ou.geometry " + fromClause; - String ouGeoSql = getSelectSubquery(attribute, selectExpression, dataFilterClause); - columns.add( - AnalyticsTableColumn.builder() - .name((attribute.getUid() + OU_GEOMETRY_COL_SUFFIX)) - .dimensionType(AnalyticsDimensionType.DYNAMIC) - .dataType(GEOMETRY) - .selectExpression(ouGeoSql) - .indexType(IndexType.GIST) - .build()); - } - - String fromTypeSql = "ou.name " + fromClause; - String ouNameSql = getSelectSubquery(attribute, fromTypeSql, dataFilterClause); - - columns.add( - AnalyticsTableColumn.builder() - .name((attribute.getUid() + OU_NAME_COL_SUFFIX)) - .dimensionType(AnalyticsDimensionType.DYNAMIC) - .dataType(TEXT) - .selectExpression(ouNameSql) - .skipIndex(SKIP) - .build()); - - return columns; - } - /** * Retyrns a select statement for the given select expression. * From db9651bc80ec34b6a20686da2167e883c5040feb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 10:21:36 +0100 Subject: [PATCH 13/48] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 84 +++++++++++++++++++ .../JdbcEnrollmentAnalyticsTableManager.java | 36 +------- .../table/JdbcEventAnalyticsTableManager.java | 76 ----------------- 3 files changed, 86 insertions(+), 110 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 6639da1d405f..197e0c28c9f5 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -28,13 +28,19 @@ package org.hisp.dhis.analytics.table; import static org.apache.commons.lang3.StringUtils.EMPTY; +import static org.hisp.dhis.analytics.table.model.Skip.SKIP; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getClosingParentheses; +import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; +import static org.hisp.dhis.db.model.DataType.GEOMETRY; +import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; +import java.util.ArrayList; import java.util.List; import java.util.Map; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.partition.PartitionManager; +import org.hisp.dhis.analytics.table.model.AnalyticsDimensionType; import org.hisp.dhis.analytics.table.model.AnalyticsTableColumn; import org.hisp.dhis.analytics.table.model.AnalyticsTablePartition; import org.hisp.dhis.analytics.table.model.Skip; @@ -44,6 +50,8 @@ import org.hisp.dhis.common.ValueType; import org.hisp.dhis.commons.util.TextUtils; import org.hisp.dhis.dataapproval.DataApprovalLevelService; +import org.hisp.dhis.db.model.DataType; +import org.hisp.dhis.db.model.IndexType; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.organisationunit.OrganisationUnitService; import org.hisp.dhis.period.PeriodDataProvider; @@ -87,6 +95,8 @@ public AbstractEventJdbcTableManager( sqlBuilder); } + public static final String OU_GEOMETRY_COL_SUFFIX = "_geom"; + public static final String OU_NAME_COL_SUFFIX = "_name"; protected final String getNumericClause() { @@ -258,4 +268,78 @@ protected String getSelectSubquery( "closingParentheses", getClosingParentheses(selectExpression), "attributeUid", quote(attribute.getUid()))); } + + /** + * Returns a list of columns based on the given attribute. + * + * @param attribute the {@link TrackedEntityAttribute}. + * @param withLegendSet indicates whether the attribute has a legend set. + * @return a list of {@link AnaylyticsTableColumn}. + */ + protected List getColumnForAttribute(TrackedEntityAttribute attribute) { + List columns = new ArrayList<>(); + + DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); + String selectExpression = getSelectExpressionForAttribute(attribute.getValueType(), "value"); + String dataFilterClause = getDataFilterClause(attribute); + String sql = getSelectSubquery(attribute, selectExpression, dataFilterClause); + Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); + + if (attribute.getValueType().isOrganisationUnit()) { + columns.addAll(getColumnForOrgUnitTrackedEntityAttribute(attribute, dataFilterClause)); + } + + columns.add( + AnalyticsTableColumn.builder() + .name(attribute.getUid()) + .dimensionType(AnalyticsDimensionType.DYNAMIC) + .dataType(dataType) + .selectExpression(sql) + .skipIndex(skipIndex) + .build()); + + return columns; + } + + /** + * Returns a list of columns based on the given attribute. + * + * @param attribute the {@link TrackedEntityAttribute}. + * @param dataFilterClause the data filter clause. + * @return a list of {@link AnalyticsTableColumn}. + */ + private List getColumnForOrgUnitTrackedEntityAttribute( + TrackedEntityAttribute attribute, String dataFilterClause) { + List columns = new ArrayList<>(); + + String fromClause = + qualifyVariables("from ${organisationunit} ou where ou.uid = (select value"); + + if (isSpatialSupport()) { + String selectExpression = "ou.geometry " + fromClause; + String ouGeoSql = getSelectSubquery(attribute, selectExpression, dataFilterClause); + columns.add( + AnalyticsTableColumn.builder() + .name((attribute.getUid() + OU_GEOMETRY_COL_SUFFIX)) + .dimensionType(AnalyticsDimensionType.DYNAMIC) + .dataType(GEOMETRY) + .selectExpression(ouGeoSql) + .indexType(IndexType.GIST) + .build()); + } + + String selectExpression = "ou.name " + fromClause; + String ouNameSql = getSelectSubquery(attribute, selectExpression, dataFilterClause); + + columns.add( + AnalyticsTableColumn.builder() + .name((attribute.getUid() + OU_NAME_COL_SUFFIX)) + .dimensionType(AnalyticsDimensionType.DYNAMIC) + .dataType(TEXT) + .selectExpression(ouNameSql) + .skipIndex(SKIP) + .build()); + + return columns; + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index 4d8271cc605d..b40e0ce7ff96 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -27,9 +27,6 @@ */ package org.hisp.dhis.analytics.table; -import static org.hisp.dhis.analytics.table.model.Skip.SKIP; -import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; -import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.util.DateUtils.toLongDate; import java.util.ArrayList; @@ -40,17 +37,14 @@ import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; import org.hisp.dhis.analytics.partition.PartitionManager; -import org.hisp.dhis.analytics.table.model.AnalyticsDimensionType; import org.hisp.dhis.analytics.table.model.AnalyticsTable; import org.hisp.dhis.analytics.table.model.AnalyticsTableColumn; import org.hisp.dhis.analytics.table.model.AnalyticsTablePartition; -import org.hisp.dhis.analytics.table.model.Skip; import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; import org.hisp.dhis.category.CategoryService; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.commons.collection.UniqueArrayList; import org.hisp.dhis.dataapproval.DataApprovalLevelService; -import org.hisp.dhis.db.model.DataType; import org.hisp.dhis.db.model.Logged; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.organisationunit.OrganisationUnitService; @@ -199,35 +193,9 @@ private List getTrackedEntityAttributeColumns(Program prog List columns = new ArrayList<>(); for (TrackedEntityAttribute attribute : program.getNonConfidentialTrackedEntityAttributes()) { - DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); - String selectExpression = getSelectExpressionForAttribute(attribute.getValueType(), "value"); - String dataFilterClause = getDataFilterClause(attribute); - String sql = getSelectSubquery(attribute, selectExpression, dataFilterClause); - Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); - - columns.add( - AnalyticsTableColumn.builder() - .name(attribute.getUid()) - .dimensionType(AnalyticsDimensionType.DYNAMIC) - .dataType(dataType) - .selectExpression(sql) - .skipIndex(skipIndex) - .build()); - - if (attribute.getValueType().isOrganisationUnit()) { - String fromTypeSql = "ou.name from organisationunit ou where ou.uid = (select value"; - String ouNameSql = getSelectSubquery(attribute, fromTypeSql, dataFilterClause); - - columns.add( - AnalyticsTableColumn.builder() - .name((attribute.getUid() + OU_NAME_COL_SUFFIX)) - .dimensionType(AnalyticsDimensionType.DYNAMIC) - .dataType(TEXT) - .selectExpression(ouNameSql) - .skipIndex(SKIP) - .build()); - } + columns.addAll(getColumnForAttribute(attribute)); } + return columns; } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index c4008fe6f762..8c652952993f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -97,8 +97,6 @@ @Service("org.hisp.dhis.analytics.EventAnalyticsTableManager") public class JdbcEventAnalyticsTableManager extends AbstractEventJdbcTableManager { - public static final String OU_GEOMETRY_COL_SUFFIX = "_geom"; - static final String[] EXPORTABLE_EVENT_STATUSES = {"'COMPLETED'", "'ACTIVE'", "'SCHEDULE'"}; protected final List fixedColumns; @@ -583,80 +581,6 @@ private List getAttributeColumns(Program program) { return columns; } - /** - * Returns a list of columns based on the given attribute. - * - * @param attribute the {@link TrackedEntityAttribute}. - * @param withLegendSet indicates whether the attribute has a legend set. - * @return a list of {@link AnaylyticsTableColumn}. - */ - private List getColumnForAttribute(TrackedEntityAttribute attribute) { - List columns = new ArrayList<>(); - - DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); - String selectExpression = getSelectExpressionForAttribute(attribute.getValueType(), "value"); - String dataFilterClause = getDataFilterClause(attribute); - String sql = getSelectSubquery(attribute, selectExpression, dataFilterClause); - Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); - - if (attribute.getValueType().isOrganisationUnit()) { - columns.addAll(getColumnForOrgUnitTrackedEntityAttribute(attribute, dataFilterClause)); - } - - columns.add( - AnalyticsTableColumn.builder() - .name(attribute.getUid()) - .dimensionType(AnalyticsDimensionType.DYNAMIC) - .dataType(dataType) - .selectExpression(sql) - .skipIndex(skipIndex) - .build()); - - return columns; - } - - /** - * Returns a list of columns based on the given attribute. - * - * @param attribute the {@link TrackedEntityAttribute}. - * @param dataFilterClause the data filter clause. - * @return a list of {@link AnalyticsTableColumn}. - */ - private List getColumnForOrgUnitTrackedEntityAttribute( - TrackedEntityAttribute attribute, String dataFilterClause) { - List columns = new ArrayList<>(); - - String fromClause = - qualifyVariables("from ${organisationunit} ou where ou.uid = (select value"); - - if (isSpatialSupport()) { - String selectExpression = "ou.geometry " + fromClause; - String ouGeoSql = getSelectSubquery(attribute, selectExpression, dataFilterClause); - columns.add( - AnalyticsTableColumn.builder() - .name((attribute.getUid() + OU_GEOMETRY_COL_SUFFIX)) - .dimensionType(AnalyticsDimensionType.DYNAMIC) - .dataType(GEOMETRY) - .selectExpression(ouGeoSql) - .indexType(IndexType.GIST) - .build()); - } - - String fromTypeSql = "ou.name " + fromClause; - String ouNameSql = getSelectSubquery(attribute, fromTypeSql, dataFilterClause); - - columns.add( - AnalyticsTableColumn.builder() - .name((attribute.getUid() + OU_NAME_COL_SUFFIX)) - .dimensionType(AnalyticsDimensionType.DYNAMIC) - .dataType(TEXT) - .selectExpression(ouNameSql) - .skipIndex(SKIP) - .build()); - - return columns; - } - /** * Returns a list of columns based on the given attribute with legend set. * From 80021185adbe8f27aef13f7ada957e429544c8b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 10:23:49 +0100 Subject: [PATCH 14/48] fix: Update code --- .../table/JdbcEnrollmentAnalyticsTableManager.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index b40e0ce7ff96..009b19645afb 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -30,6 +30,7 @@ import static org.hisp.dhis.util.DateUtils.toLongDate; import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.List; import java.util.Map; @@ -53,7 +54,6 @@ import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingsProvider; import org.hisp.dhis.system.database.DatabaseInfoProvider; -import org.hisp.dhis.trackedentity.TrackedEntityAttribute; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; @@ -190,13 +190,10 @@ private List getColumns(Program program) { * @return a list of {@link AnalyticsTableColumn}. */ private List getTrackedEntityAttributeColumns(Program program) { - List columns = new ArrayList<>(); - - for (TrackedEntityAttribute attribute : program.getNonConfidentialTrackedEntityAttributes()) { - columns.addAll(getColumnForAttribute(attribute)); - } - - return columns; + return program.getNonConfidentialTrackedEntityAttributes().stream() + .map(this::getColumnForAttribute) + .flatMap(Collection::stream) + .toList(); } /** From 7fc3b92e0879aa2a6e44239f98c7154a42ddf346 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 10:27:45 +0100 Subject: [PATCH 15/48] fix: Update code --- .../org/hisp/dhis/db/model/ColumnTest.java | 56 --- .../org/hisp/dhis/db/model/DataTypeTest.java | 53 -- .../org/hisp/dhis/db/model/IndexTest.java | 76 --- .../org/hisp/dhis/db/model/TableTest.java | 145 ------ .../dhis/db/sql/ClickHouseSqlBuilderTest.java | 381 --------------- .../hisp/dhis/db/sql/DorisSqlBuilderTest.java | 367 -------------- .../dhis/db/sql/PostgreSqlBuilderTest.java | 452 ------------------ 7 files changed, 1530 deletions(-) delete mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/ColumnTest.java delete mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/DataTypeTest.java delete mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/IndexTest.java delete mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/TableTest.java delete mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java delete mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java delete mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/ColumnTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/ColumnTest.java deleted file mode 100644 index e1d2f48443c5..000000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/ColumnTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2004-2024, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.db.model; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import org.hisp.dhis.db.model.constraint.Nullable; -import org.junit.jupiter.api.Test; - -class ColumnTest { - @Test - void testIsNotNull() { - Column colA = new Column("dx", DataType.CHARACTER_11, Nullable.NOT_NULL); - Column colB = new Column("value", DataType.DOUBLE, Nullable.NULL); - - assertTrue(colA.isNotNull()); - assertFalse(colB.isNotNull()); - } - - @Test - void testHasCollation() { - Column colA = new Column("dx", DataType.CHARACTER_11, Nullable.NOT_NULL, Collation.DEFAULT); - Column colB = new Column("ou", DataType.CHARACTER_11, Nullable.NOT_NULL, Collation.C); - Column colC = new Column("value", DataType.DOUBLE, Nullable.NULL); - - assertFalse(colA.hasCollation()); - assertTrue(colB.hasCollation()); - assertFalse(colC.hasCollation()); - } -} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/DataTypeTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/DataTypeTest.java deleted file mode 100644 index 1c75b3c3cdda..000000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/DataTypeTest.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2004-2024, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.db.model; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import org.junit.jupiter.api.Test; - -class DataTypeTest { - @Test - void testIsNumeric() { - assertTrue(DataType.BIGINT.isNumeric()); - assertFalse(DataType.CHARACTER_11.isNumeric()); - } - - @Test - void testIsBoolean() { - assertTrue(DataType.BOOLEAN.isBoolean()); - assertFalse(DataType.DOUBLE.isBoolean()); - } - - @Test - void testIsCharacter() { - assertTrue(DataType.VARCHAR_255.isCharacter()); - assertFalse(DataType.DECIMAL.isCharacter()); - } -} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/IndexTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/IndexTest.java deleted file mode 100644 index c7d46a847a70..000000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/IndexTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2004-2024, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.db.model; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; -import org.hisp.dhis.db.model.constraint.Unique; -import org.junit.jupiter.api.Test; - -class IndexTest { - @Test - void testIsUnique() { - Index indexA = - Index.builder() - .name("in_analytics_id") - .tableName("analytics") - .unique(Unique.UNIQUE) - .columns(List.of("id")) - .build(); - - Index indexB = - Index.builder() - .name("in_analytics_dx") - .tableName("analytics") - .columns(List.of("dx")) - .build(); - - assertTrue(indexA.isUnique()); - assertFalse(indexB.isUnique()); - } - - @Test - void testDefaults() { - Index.IndexBuilder builder = Index.builder(); - - Index index = builder.build(); - - assertNull(index.getName()); - assertNull(index.getTableName()); - assertNull(index.getCondition()); - assertNull(index.getFunction()); - assertNull(index.getColumns()); - assertNull(index.getSortOrder()); - assertSame(IndexType.BTREE, index.getIndexType()); - assertFalse(index.isUnique()); - } -} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/TableTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/TableTest.java deleted file mode 100644 index bc57ee6db4cc..000000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/TableTest.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright (c) 2004-2024, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.db.model; - -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; -import org.hisp.dhis.db.model.constraint.Nullable; -import org.junit.jupiter.api.Test; - -class TableTest { - private final Column colA = new Column("dx", DataType.CHARACTER_11, Nullable.NOT_NULL); - private final Column colB = new Column("value", DataType.DOUBLE, Nullable.NULL); - - @Test - void testToStagingTable() { - assertEquals( - "analytics_rs_categorystructure_temp", Table.toStaging("analytics_rs_categorystructure")); - assertEquals("analytics_temp", Table.toStaging("analytics")); - } - - @Test - void testFromStagingTable() { - assertEquals( - "analytics_rs_categorystructure", Table.fromStaging("analytics_rs_categorystructure_temp")); - assertEquals("analytics", Table.fromStaging("analytics_temp")); - } - - @Test - void testIsUnlogged() { - List columns = List.of(colA, colB); - - Table tableA = new Table("analytics", columns, List.of(), Logged.UNLOGGED); - Table tableB = new Table("analytics", columns, List.of(), Logged.LOGGED); - - assertTrue(tableA.isUnlogged()); - assertFalse(tableB.isUnlogged()); - } - - @Test - void testHasColumns() { - Table table = new Table("analytics", List.of(colA, colB), List.of()); - - assertTrue(table.hasColumns()); - } - - @Test - void getFirstColumn() { - Table table = new Table("analytics", List.of(colA, colB), List.of()); - - assertEquals(colA, table.getFirstColumn()); - } - - @Test - void testHasPrimaryKey() { - Table tableA = new Table("analytics", List.of(colA, colB), List.of("dx")); - Table tableB = new Table("analytics", List.of(colA, colB), List.of()); - - assertTrue(tableA.hasPrimaryKey()); - assertFalse(tableB.hasPrimaryKey()); - assertEquals(List.of("dx"), tableA.getPrimaryKey()); - } - - @Test - void testGetFirstPrimaryKey() { - Table table = new Table("analytics", List.of(colA, colB), List.of("dx", "value")); - - assertEquals("dx", table.getFirstPrimaryKey()); - } - - @Test - void testHasSortKey() { - Table tableA = - new Table( - "analytics", - List.of(colA, colB), - List.of("dx", "value"), - List.of("dx"), - List.of(), - Logged.UNLOGGED); - Table tableB = new Table("analytics", List.of(colA, colB), List.of("dx", "value")); - - assertTrue(tableA.hasSortKey()); - assertFalse(tableB.hasSortKey()); - assertEquals(List.of("dx"), tableA.getSortKey()); - } - - @Test - void testSuccessfulValidation() { - List columns = List.of(colA); - List primaryKey = List.of(); - - assertDoesNotThrow(() -> new Table("analytics", columns, primaryKey)); - } - - @Test - void testNameValidation() { - List columns = List.of(colA); - List primaryKey = List.of(); - - assertThrows(NullPointerException.class, () -> new Table(null, columns, primaryKey)); - assertThrows(IllegalArgumentException.class, () -> new Table("", columns, primaryKey)); - } - - @Test - void testColumnsParentValidation() { - List columns = List.of(); - List primaryKey = List.of(); - List checks = List.of(); - - assertThrows(IllegalArgumentException.class, () -> new Table("analytics", columns, primaryKey)); - assertThrows( - IllegalArgumentException.class, - () -> new Table("analytics", columns, primaryKey, checks, Logged.UNLOGGED, null)); - } -} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java deleted file mode 100644 index 5087172105e4..000000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Copyright (c) 2004-2024, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.db.sql; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; -import java.util.Map; -import org.hisp.dhis.db.model.Collation; -import org.hisp.dhis.db.model.Column; -import org.hisp.dhis.db.model.DataType; -import org.hisp.dhis.db.model.Logged; -import org.hisp.dhis.db.model.Table; -import org.hisp.dhis.db.model.constraint.Nullable; -import org.junit.jupiter.api.Test; - -class ClickHouseSqlBuilderTest { - private final ClickHouseSqlBuilder sqlBuilder = new ClickHouseSqlBuilder(); - - private Table getTableA() { - List columns = - List.of( - new Column("id", DataType.BIGINT, Nullable.NOT_NULL), - new Column("data", DataType.CHARACTER_11, Nullable.NOT_NULL), - new Column("period", DataType.VARCHAR_50, Nullable.NOT_NULL), - new Column("created", DataType.TIMESTAMP), - new Column("user", DataType.JSONB), - new Column("value", DataType.DOUBLE)); - - List primaryKey = List.of("id"); - - return new Table("immunization", columns, primaryKey, Logged.LOGGED); - } - - private Table getTableB() { - List columns = - List.of( - new Column("id", DataType.INTEGER, Nullable.NOT_NULL), - new Column("facility_type", DataType.VARCHAR_255, Nullable.NULL, Collation.C), - new Column("bcg_doses", DataType.DOUBLE)); - - return new Table("vaccination", columns, List.of()); - } - - private Table getTableC() { - List columns = - List.of( - new Column("id", DataType.BIGINT, Nullable.NOT_NULL), - new Column("vitamin_a", DataType.BIGINT), - new Column("vitamin_d", DataType.BIGINT)); - - List primaryKey = List.of("id"); - - return new Table("nutrition", columns, primaryKey, List.of(), Logged.LOGGED, getTableB()); - } - - private Table getTableD() { - List columns = - List.of( - new Column("id", DataType.BIGINT, Nullable.NOT_NULL), - new Column("data", DataType.CHARACTER_11, Nullable.NOT_NULL), - new Column("period", DataType.VARCHAR_50, Nullable.NOT_NULL), - new Column("value", DataType.DOUBLE)); - - List sortKey = List.of("data", "period"); - - return new Table("immunization", columns, List.of(), sortKey, List.of(), Logged.LOGGED); - } - - // Data types - - @Test - void testDataTypes() { - assertEquals("Float64", sqlBuilder.dataTypeDouble()); - assertEquals("DateTime64(3)", sqlBuilder.dataTypeTimestamp()); - } - - // Index types - - @Test - void testIndexTypes() { - assertThrows(UnsupportedOperationException.class, () -> sqlBuilder.indexTypeBtree()); - } - - // Capabilities - - @Test - void testSupportsAnalyze() { - assertFalse(sqlBuilder.supportsAnalyze()); - } - - @Test - void testSupportsVacuum() { - assertFalse(sqlBuilder.supportsVacuum()); - } - - // Utilities - - @Test - void testQuote() { - assertEquals( - "\"Treated \"\"malaria\"\" at facility\"", - sqlBuilder.quote("Treated \"malaria\" at facility")); - assertEquals( - "\"\"\"Patients on \"\"treatment\"\" for TB\"\"\"", - sqlBuilder.quote("\"Patients on \"treatment\" for TB\"")); - assertEquals("\"quarterly\"", sqlBuilder.quote("quarterly")); - assertEquals("\"Fully immunized\"", sqlBuilder.quote("Fully immunized")); - } - - @Test - void testQuoteAlias() { - assertEquals( - "ax.\"Treated \"\"malaria\"\" at facility\"", - sqlBuilder.quote("ax", "Treated \"malaria\" at facility")); - assertEquals( - "analytics.\"Patients on \"\"treatment\"\" for TB\"", - sqlBuilder.quote("analytics", "Patients on \"treatment\" for TB")); - assertEquals("analytics.\"quarterly\"", sqlBuilder.quote("analytics", "quarterly")); - assertEquals("dv.\"Fully immunized\"", sqlBuilder.quote("dv", "Fully immunized")); - } - - @Test - void testQuoteAx() { - assertEquals( - "ax.\"Treated \"\"malaria\"\" at facility\"", - sqlBuilder.quoteAx("Treated \"malaria\" at facility")); - assertEquals("ax.\"quarterly\"", sqlBuilder.quoteAx("quarterly")); - assertEquals("ax.\"Fully immunized\"", sqlBuilder.quoteAx("Fully immunized")); - } - - @Test - void testSingleQuote() { - assertEquals("'jkhYg65ThbF'", sqlBuilder.singleQuote("jkhYg65ThbF")); - assertEquals("'Age ''<5'' years'", sqlBuilder.singleQuote("Age '<5' years")); - assertEquals("'Status \"not checked\"'", sqlBuilder.singleQuote("Status \"not checked\"")); - } - - @Test - void testEscape() { - assertEquals("Age group ''under 5'' years", sqlBuilder.escape("Age group 'under 5' years")); - assertEquals("Level ''high'' found", sqlBuilder.escape("Level 'high' found")); - } - - @Test - void testSinqleQuotedCommaDelimited() { - assertEquals( - "'dmPbDBKwXyF', 'zMl4kciwJtz', 'q1Nqu1r1GTn'", - sqlBuilder.singleQuotedCommaDelimited( - List.of("dmPbDBKwXyF", "zMl4kciwJtz", "q1Nqu1r1GTn"))); - assertEquals("'1', '3', '5'", sqlBuilder.singleQuotedCommaDelimited(List.of("1", "3", "5"))); - assertEquals("", sqlBuilder.singleQuotedCommaDelimited(List.of())); - assertEquals("", sqlBuilder.singleQuotedCommaDelimited(null)); - } - - @Test - void testQualifyTable() { - assertEquals("postgresql(\"pg_dhis\", table='category')", sqlBuilder.qualifyTable("category")); - assertEquals( - "postgresql(\"pg_dhis\", table='categories_options')", - sqlBuilder.qualifyTable("categories_options")); - } - - @Test - void testDateTrunc() { - assertEquals( - "date_trunc('month', pe.startdate)", sqlBuilder.dateTrunc("month", "pe.startdate")); - } - - @Test - void testDifferenceInSeconds() { - assertEquals( - "(toUnixTimestamp(a.startdate) - toUnixTimestamp(b.enddate))", - sqlBuilder.differenceInSeconds("a.startdate", "b.enddate")); - assertEquals( - "(toUnixTimestamp(a.\"startdate\") - toUnixTimestamp(b.\"enddate\"))", - sqlBuilder.differenceInSeconds( - sqlBuilder.quote("a", "startdate"), sqlBuilder.quote("b", "enddate"))); - } - - @Test - void testRegexpMatch() { - assertEquals("match(value, 'test')", sqlBuilder.regexpMatch("value", "'test'")); - assertEquals("match(number, '\\d')", sqlBuilder.regexpMatch("number", "'\\d'")); - assertEquals("match(color, '^Blue$')", sqlBuilder.regexpMatch("color", "'^Blue$'")); - assertEquals("match(id, '[a-z]\\w+\\d{3}')", sqlBuilder.regexpMatch("id", "'[a-z]\\w+\\d{3}'")); - } - - @Test - void testJsonExtract() { - assertEquals( - "JSONExtractString(value, 'D7m8vpzxHDJ')", sqlBuilder.jsonExtract("value", "D7m8vpzxHDJ")); - } - - @Test - void testJsonExtractNested() { - assertEquals( - "JSONExtractString(eventdatavalues, 'D7m8vpzxHDJ.value')", - sqlBuilder.jsonExtractNested("eventdatavalues", "D7m8vpzxHDJ", "value")); - } - - // Statements - - @Test - void testCreateTableA() { - Table table = getTableA(); - - String expected = - """ - create table "immunization" ("id" Int64 not null,"data" String not null,\ - "period" String not null,"created" DateTime64(3) null,"user" JSON null,\ - "value" Float64 null) \ - engine = MergeTree() \ - order by ("id");"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - @Test - void testCreateTableB() { - Table table = getTableB(); - - String expected = - """ - create table "vaccination" ("id" Int32 not null,\ - "facility_type" String null,"bcg_doses" Float64 null) \ - engine = MergeTree() \ - order by ("id");"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - @Test - void testCreateTableC() { - Table table = getTableC(); - - String expected = - """ - create table "nutrition" ("id" Int64 not null,"vitamin_a" Int64 null,\ - "vitamin_d" Int64 null) \ - engine = MergeTree() order by ("id");"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - @Test - void testCreateTableD() { - Table table = getTableD(); - - String expected = - """ - create table "immunization" ("id" Int64 not null,"data" String not null,\ - "period" String not null,"value" Float64 null) \ - engine = MergeTree() \ - order by ("data","period");"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - @Test - void testRenameTable() { - Table table = getTableA(); - - String expected = "rename table \"immunization\" to \"vaccination\";"; - - assertEquals(expected, sqlBuilder.renameTable(table, "vaccination")); - } - - @Test - void testDropTableIfExists() { - Table table = getTableA(); - - String expected = "drop table if exists \"immunization\";"; - - assertEquals(expected, sqlBuilder.dropTableIfExists(table)); - } - - @Test - void testDropTableIfExistsString() { - String expected = "drop table if exists \"immunization\";"; - - assertEquals(expected, sqlBuilder.dropTableIfExists("immunization")); - } - - @Test - void testDropTableIfExistsCascade() { - Table table = getTableA(); - - String expected = "drop table if exists \"immunization\";"; - - assertEquals(expected, sqlBuilder.dropTableIfExistsCascade(table)); - } - - @Test - void testDropTableIfExistsCascadeString() { - String expected = "drop table if exists \"immunization\";"; - - assertEquals(expected, sqlBuilder.dropTableIfExistsCascade("immunization")); - } - - @Test - void testSwapTable() { - String expected = - """ - drop table if exists "vaccination"; \ - rename table "immunization" to "vaccination";"""; - - assertEquals(expected, sqlBuilder.swapTable(getTableA(), "vaccination")); - } - - @Test - void testTableExists() { - String expected = - """ - select t.name as table_name \ - from system.tables t \ - where t.database = 'default' \ - and t.name = 'immunization' \ - and engine not in ('View', 'Materialized View');"""; - - assertEquals(expected, sqlBuilder.tableExists("immunization")); - } - - @Test - void testCountRows() { - String expected = - """ - select count(*) as row_count from \"immunization\";"""; - - assertEquals(expected, sqlBuilder.countRows(getTableA())); - } - - // Named collection - - @Test - void testCreateNamedCollection() { - String expected = - """ - create named collection "pg_dhis" as """; - - assertTrue( - sqlBuilder - .createNamedCollection("pg_dhis", Map.of("host", "mydomain.org")) - .startsWith(expected)); - } - - @Test - void testDropNamedCollectionIfExists() { - assertEquals( - "drop named collection if exists \"pg_dhis\";", - sqlBuilder.dropNamedCollectionIfExists("pg_dhis")); - } -} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java deleted file mode 100644 index ad372141c9bc..000000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java +++ /dev/null @@ -1,367 +0,0 @@ -/* - * Copyright (c) 2004-2024, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.db.sql; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import java.util.List; -import org.hisp.dhis.db.model.Collation; -import org.hisp.dhis.db.model.Column; -import org.hisp.dhis.db.model.DataType; -import org.hisp.dhis.db.model.Logged; -import org.hisp.dhis.db.model.Table; -import org.hisp.dhis.db.model.constraint.Nullable; -import org.junit.jupiter.api.Test; - -class DorisSqlBuilderTest { - private final SqlBuilder sqlBuilder = new DorisSqlBuilder("pg_dhis", "postgresql.jar"); - - private Table getTableA() { - List columns = - List.of( - new Column("id", DataType.BIGINT, Nullable.NOT_NULL), - new Column("data", DataType.CHARACTER_11, Nullable.NOT_NULL), - new Column("period", DataType.VARCHAR_50, Nullable.NOT_NULL), - new Column("created", DataType.TIMESTAMP), - new Column("user", DataType.JSONB), - new Column("value", DataType.DOUBLE)); - - List primaryKey = List.of("id"); - - return new Table("immunization", columns, primaryKey, Logged.LOGGED); - } - - private Table getTableB() { - List columns = - List.of( - new Column("id", DataType.INTEGER, Nullable.NOT_NULL), - new Column("facility_type", DataType.VARCHAR_255, Nullable.NULL, Collation.C), - new Column("bcg_doses", DataType.DOUBLE)); - - return new Table("vaccination", columns, List.of(), Logged.UNLOGGED); - } - - private Table getTableC() { - List columns = - List.of( - new Column("id", DataType.BIGINT, Nullable.NOT_NULL), - new Column("vitamin_a", DataType.BIGINT), - new Column("vitamin_d", DataType.BIGINT)); - - List primaryKey = List.of("id"); - - return new Table("nutrition", columns, primaryKey, List.of(), Logged.LOGGED, getTableB()); - } - - // Data types - - @Test - void testDataTypes() { - assertEquals("double", sqlBuilder.dataTypeDouble()); - assertEquals("datetime", sqlBuilder.dataTypeTimestamp()); - } - - // Index types - - @Test - void testIndexTypes() { - assertThrows(UnsupportedOperationException.class, () -> sqlBuilder.indexTypeBtree()); - } - - // Capabilities - - @Test - void testSupportsAnalyze() { - assertFalse(sqlBuilder.supportsAnalyze()); - } - - @Test - void testSupportsVacuum() { - assertFalse(sqlBuilder.supportsVacuum()); - } - - // Utilities - - @Test - void testQuote() { - assertEquals( - "`Treated \"malaria\" at facility`", sqlBuilder.quote("Treated \"malaria\" at facility")); - assertEquals( - "`Patients on ``treatment`` for TB`", sqlBuilder.quote("Patients on `treatment` for TB")); - assertEquals("`quarterly`", sqlBuilder.quote("quarterly")); - assertEquals("`Fully immunized`", sqlBuilder.quote("Fully immunized")); - } - - @Test - void testQuoteAlias() { - assertEquals( - "ax.`Treated \"malaria\" at facility`", - sqlBuilder.quote("ax", "Treated \"malaria\" at facility")); - assertEquals( - "analytics.`Patients on ``treatment`` for TB`", - sqlBuilder.quote("analytics", "Patients on `treatment` for TB")); - assertEquals("analytics.`quarterly`", sqlBuilder.quote("analytics", "quarterly")); - assertEquals("dv.`Fully immunized`", sqlBuilder.quote("dv", "Fully immunized")); - } - - @Test - void testQuoteAx() { - assertEquals( - "ax.`Treated ``malaria`` at facility`", - sqlBuilder.quoteAx("Treated `malaria` at facility")); - assertEquals("ax.`quarterly`", sqlBuilder.quoteAx("quarterly")); - assertEquals("ax.`Fully immunized`", sqlBuilder.quoteAx("Fully immunized")); - } - - @Test - void testSingleQuote() { - assertEquals("'jkhYg65ThbF'", sqlBuilder.singleQuote("jkhYg65ThbF")); - assertEquals("'Age ''<5'' years'", sqlBuilder.singleQuote("Age '<5' years")); - assertEquals("'Status \"not checked\"'", sqlBuilder.singleQuote("Status \"not checked\"")); - } - - @Test - void testEscape() { - assertEquals("Age group ''under 5'' years", sqlBuilder.escape("Age group 'under 5' years")); - assertEquals("Level ''high'' found", sqlBuilder.escape("Level 'high' found")); - assertEquals("C:\\\\Downloads\\\\File.doc", sqlBuilder.escape("C:\\Downloads\\File.doc")); - } - - @Test - void testSinqleQuotedCommaDelimited() { - assertEquals( - "'dmPbDBKwXyF', 'zMl4kciwJtz', 'q1Nqu1r1GTn'", - sqlBuilder.singleQuotedCommaDelimited( - List.of("dmPbDBKwXyF", "zMl4kciwJtz", "q1Nqu1r1GTn"))); - assertEquals("'1', '3', '5'", sqlBuilder.singleQuotedCommaDelimited(List.of("1", "3", "5"))); - assertEquals("", sqlBuilder.singleQuotedCommaDelimited(List.of())); - assertEquals("", sqlBuilder.singleQuotedCommaDelimited(null)); - } - - @Test - void testQualifyTable() { - assertEquals("pg_dhis.public.`category`", sqlBuilder.qualifyTable("category")); - assertEquals( - "pg_dhis.public.`categories_options`", sqlBuilder.qualifyTable("categories_options")); - } - - @Test - void testDateTrunc() { - assertEquals( - "date_trunc(pe.startdate, 'month')", sqlBuilder.dateTrunc("month", "pe.startdate")); - } - - @Test - void testDifferenceInSeconds() { - assertEquals( - "(unix_timestamp(a.startdate) - unix_timestamp(b.enddate))", - sqlBuilder.differenceInSeconds("a.startdate", "b.enddate")); - assertEquals( - "(unix_timestamp(a.`startdate`) - unix_timestamp(b.`enddate`))", - sqlBuilder.differenceInSeconds( - sqlBuilder.quote("a", "startdate"), sqlBuilder.quote("b", "enddate"))); - } - - @Test - void testRegexpMatch() { - assertEquals("value regexp 'test'", sqlBuilder.regexpMatch("value", "'test'")); - assertEquals("number regexp '\\d'", sqlBuilder.regexpMatch("number", "'\\d'")); - assertEquals("color regexp '^Blue$'", sqlBuilder.regexpMatch("color", "'^Blue$'")); - assertEquals("id regexp '[a-z]\\w+\\d{3}'", sqlBuilder.regexpMatch("id", "'[a-z]\\w+\\d{3}'")); - } - - @Test - void testJsonExtract() { - assertEquals( - "json_unquote(json_extract(value, '$.D7m8vpzxHDJ'))", - sqlBuilder.jsonExtract("value", "D7m8vpzxHDJ")); - } - - @Test - void testJsonExtractNested() { - assertEquals( - "json_unquote(json_extract(eventdatavalues, '$.D7m8vpzxHDJ.value'))", - sqlBuilder.jsonExtractNested("eventdatavalues", "D7m8vpzxHDJ", "value")); - } - - // Statements - - @Test - void testCreateTableA() { - Table table = getTableA(); - - String expected = - """ - create table `immunization` (`id` bigint not null,\ - `data` char(11) not null,`period` varchar(50) not null,\ - `created` datetime null,`user` json null,`value` double null) \ - engine = olap \ - unique key (`id`) \ - distributed by hash(`id`) buckets 10 \ - properties ("replication_num" = "1");"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - @Test - void testCreateTableB() { - Table table = getTableB(); - - String expected = - """ - create table `vaccination` (`id` int not null,\ - `facility_type` varchar(255) null,`bcg_doses` double null) \ - engine = olap \ - duplicate key (`id`) \ - distributed by hash(`id`) buckets 10 \ - properties ("replication_num" = "1");"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - // void testCreateTableB() - - @Test - void testCreateTableC() { - Table table = getTableC(); - - String expected = - """ - create table `nutrition` (`id` bigint not null,\ - `vitamin_a` bigint null,`vitamin_d` bigint null) \ - engine = olap \ - unique key (`id`) \ - distributed by hash(`id`) buckets 10 \ - properties ("replication_num" = "1");"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - @Test - void testCreateCatalog() { - String expected = - """ - create catalog `pg_dhis` \ - properties ( - "type" = "jdbc", \ - "user" = "dhis", \ - "password" = "kH7g", \ - "jdbc_url" = "jdbc:mysql://127.18.0.1:9030/dev?useUnicode=true&characterEncoding=UTF-8", \ - "driver_url" = "postgresql.jar", \ - "driver_class" = "org.postgresql.Driver" - );"""; - - assertEquals( - expected, - sqlBuilder.createCatalog( - "jdbc:mysql://127.18.0.1:9030/dev?useUnicode=true&characterEncoding=UTF-8", - "dhis", - "kH7g")); - } - - @Test - void testDropCatalogIfExists() { - String expected = "drop catalog if exists `pg_dhis`;"; - - assertEquals(expected, sqlBuilder.dropCatalogIfExists()); - } - - @Test - void testRenameTable() { - Table table = getTableA(); - - String expected = - """ - alter table `immunization` rename `immunization_main`;"""; - - assertEquals(expected, sqlBuilder.renameTable(table, "immunization_main")); - } - - @Test - void testDropTableIfExists() { - Table table = getTableA(); - - String expected = "drop table if exists `immunization`;"; - - assertEquals(expected, sqlBuilder.dropTableIfExists(table)); - } - - @Test - void testDropTableIfExistsString() { - String expected = "drop table if exists `immunization`;"; - - assertEquals(expected, sqlBuilder.dropTableIfExists("immunization")); - } - - @Test - void testDropTableIfExistsCascade() { - Table table = getTableA(); - - String expected = "drop table if exists `immunization`;"; - - assertEquals(expected, sqlBuilder.dropTableIfExistsCascade(table)); - } - - @Test - void testDropTableIfExistsCascadeString() { - String expected = "drop table if exists `immunization`;"; - - assertEquals(expected, sqlBuilder.dropTableIfExistsCascade("immunization")); - } - - @Test - void testSwapTable() { - String expected = - """ - drop table if exists `vaccination`; \ - alter table `immunization` rename `vaccination`;"""; - - assertEquals(expected, sqlBuilder.swapTable(getTableA(), "vaccination")); - } - - @Test - void testTableExists() { - String expected = - """ - select t.table_name from information_schema.tables t \ - where t.table_schema = 'public' and t.table_name = 'immunization';"""; - - assertEquals(expected, sqlBuilder.tableExists("immunization")); - } - - @Test - void testCountRows() { - String expected = - """ - select count(*) as row_count from `immunization`;"""; - - assertEquals(expected, sqlBuilder.countRows(getTableA())); - } -} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java deleted file mode 100644 index 2d621ae766e4..000000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java +++ /dev/null @@ -1,452 +0,0 @@ -/* - * Copyright (c) 2004-2024, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.db.sql; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; -import org.hisp.dhis.db.model.Collation; -import org.hisp.dhis.db.model.Column; -import org.hisp.dhis.db.model.DataType; -import org.hisp.dhis.db.model.Index; -import org.hisp.dhis.db.model.IndexFunction; -import org.hisp.dhis.db.model.IndexType; -import org.hisp.dhis.db.model.Logged; -import org.hisp.dhis.db.model.Table; -import org.hisp.dhis.db.model.constraint.Nullable; -import org.hisp.dhis.db.model.constraint.Unique; -import org.junit.jupiter.api.Test; - -class PostgreSqlBuilderTest { - private final SqlBuilder sqlBuilder = new PostgreSqlBuilder(); - - private Table getTableA() { - List columns = - List.of( - new Column("id", DataType.BIGINT, Nullable.NOT_NULL), - new Column("data", DataType.CHARACTER_11, Nullable.NOT_NULL), - new Column("period", DataType.VARCHAR_50, Nullable.NOT_NULL), - new Column("created", DataType.TIMESTAMP), - new Column("user", DataType.JSONB), - new Column("value", DataType.DOUBLE)); - - List primaryKey = List.of("id"); - - return new Table("immunization", columns, primaryKey, Logged.LOGGED); - } - - private List getIndexesA() { - return List.of( - Index.builder() - .name("in_immunization_data") - .tableName("immunization") - .columns(List.of("data")) - .build(), - Index.builder() - .name("in_immunization_period_created") - .tableName("immunization") - .columns(List.of("period", "created")) - .build(), - Index.builder() - .name("in_immunization_user") - .tableName("immunization") - .indexType(IndexType.GIN) - .columns(List.of("user")) - .build(), - Index.builder() - .name("in_immunization_data_period") - .tableName("immunization") - .columns(List.of("data", "period")) - .function(IndexFunction.LOWER) - .build()); - } - - private Table getTableB() { - List columns = - List.of( - new Column("id", DataType.INTEGER, Nullable.NOT_NULL), - new Column("facility_type", DataType.VARCHAR_255, Nullable.NULL, Collation.C), - new Column("bcg_doses", DataType.DOUBLE)); - - List checks = List.of("\"id\">0", "\"bcg_doses\">0"); - - return new Table("vaccination", columns, List.of(), List.of(), checks, Logged.UNLOGGED); - } - - private Table getTableC() { - List columns = - List.of( - new Column("id", DataType.BIGINT, Nullable.NOT_NULL), - new Column("vitamin_a", DataType.BIGINT), - new Column("vitamin_d", DataType.BIGINT)); - - List primaryKey = List.of("id"); - - return new Table("nutrition", columns, primaryKey, List.of(), Logged.LOGGED, getTableB()); - } - - // Data types - - @Test - void testDataTypes() { - assertEquals("double precision", sqlBuilder.dataTypeDouble()); - assertEquals("geometry", sqlBuilder.dataTypeGeometry()); - } - - // Index types - - @Test - void testIndexTypes() { - assertEquals("btree", sqlBuilder.indexTypeBtree()); - assertEquals("gist", sqlBuilder.indexTypeGist()); - assertEquals("gin", sqlBuilder.indexTypeGin()); - } - - // Capabilities - - @Test - void testSupportsAnalyze() { - assertTrue(sqlBuilder.supportsAnalyze()); - } - - @Test - void testSupportsVacuum() { - assertTrue(sqlBuilder.supportsVacuum()); - } - - // Utilities - - @Test - void testQuote() { - assertEquals( - "\"Treated \"\"malaria\"\" at facility\"", - sqlBuilder.quote("Treated \"malaria\" at facility")); - assertEquals("\"quarterly\"", sqlBuilder.quote("quarterly")); - assertEquals("\"Fully immunized\"", sqlBuilder.quote("Fully immunized")); - } - - @Test - void testQuoteAlias() { - assertEquals( - "ax.\"Treated \"\"malaria\"\" at facility\"", - sqlBuilder.quote("ax", "Treated \"malaria\" at facility")); - assertEquals("analytics.\"quarterly\"", sqlBuilder.quote("analytics", "quarterly")); - assertEquals("dv.\"Fully immunized\"", sqlBuilder.quote("dv", "Fully immunized")); - } - - @Test - void testQuoteAx() { - assertEquals( - "ax.\"Treated \"\"malaria\"\" at facility\"", - sqlBuilder.quoteAx("Treated \"malaria\" at facility")); - assertEquals("ax.\"quarterly\"", sqlBuilder.quoteAx("quarterly")); - assertEquals("ax.\"Fully immunized\"", sqlBuilder.quoteAx("Fully immunized")); - } - - @Test - void testSingleQuote() { - assertEquals("'jkhYg65ThbF'", sqlBuilder.singleQuote("jkhYg65ThbF")); - assertEquals("'Age ''<5'' years'", sqlBuilder.singleQuote("Age '<5' years")); - assertEquals("'Status \"not checked\"'", sqlBuilder.singleQuote("Status \"not checked\"")); - } - - @Test - void testEscape() { - assertEquals("Age group ''under 5'' years", sqlBuilder.escape("Age group 'under 5' years")); - assertEquals("Level ''high'' found", sqlBuilder.escape("Level 'high' found")); - assertEquals("C:\\\\Downloads\\\\File.doc", sqlBuilder.escape("C:\\Downloads\\File.doc")); - } - - @Test - void testSinqleQuotedCommaDelimited() { - assertEquals( - "'dmPbDBKwXyF', 'zMl4kciwJtz', 'q1Nqu1r1GTn'", - sqlBuilder.singleQuotedCommaDelimited( - List.of("dmPbDBKwXyF", "zMl4kciwJtz", "q1Nqu1r1GTn"))); - assertEquals("'1', '3', '5'", sqlBuilder.singleQuotedCommaDelimited(List.of("1", "3", "5"))); - assertEquals("", sqlBuilder.singleQuotedCommaDelimited(List.of())); - assertEquals("", sqlBuilder.singleQuotedCommaDelimited(null)); - } - - @Test - void testQualifyTable() { - assertEquals("\"category\"", sqlBuilder.qualifyTable("category")); - assertEquals("\"categories_options\"", sqlBuilder.qualifyTable("categories_options")); - } - - @Test - void testDateTrunc() { - assertEquals( - "date_trunc('month', pe.startdate)", sqlBuilder.dateTrunc("month", "pe.startdate")); - } - - @Test - void testDifferenceInSeconds() { - assertEquals( - "extract(epoch from (a.startdate - b.enddate))", - sqlBuilder.differenceInSeconds("a.startdate", "b.enddate")); - assertEquals( - "extract(epoch from (a.\"startdate\" - b.\"enddate\"))", - sqlBuilder.differenceInSeconds( - sqlBuilder.quote("a", "startdate"), sqlBuilder.quote("b", "enddate"))); - } - - @Test - void testRegexpMatch() { - assertEquals("value ~* 'test'", sqlBuilder.regexpMatch("value", "'test'")); - assertEquals("number ~* '\\d'", sqlBuilder.regexpMatch("number", "'\\d'")); - assertEquals("color ~* '^Blue$'", sqlBuilder.regexpMatch("color", "'^Blue$'")); - assertEquals("id ~* '[a-z]\\w+\\d{3}'", sqlBuilder.regexpMatch("id", "'[a-z]\\w+\\d{3}'")); - } - - @Test - void testJsonExtract() { - assertEquals("value ->> 'D7m8vpzxHDJ'", sqlBuilder.jsonExtract("value", "D7m8vpzxHDJ")); - } - - @Test - void testJsonExtractNested() { - assertEquals( - "eventdatavalues #>> '{D7m8vpzxHDJ, value}'", - sqlBuilder.jsonExtractNested("eventdatavalues", "D7m8vpzxHDJ", "value")); - } - - // Statements - - @Test - void testCreateTableA() { - Table table = getTableA(); - - String expected = - """ - create table "immunization" ("id" bigint not null, "data" char(11) not null, \ - "period\" varchar(50) not null, "created" timestamp null, "user" jsonb null, \ - "value" double precision null, primary key ("id"));"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - @Test - void testCreateTableB() { - Table table = getTableB(); - - String expected = - """ - create unlogged table "vaccination" ("id" integer not null, \ - "facility_type" varchar(255) null collate "C", "bcg_doses" double precision null, \ - check("id">0), check("bcg_doses">0));"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - @Test - void testCreateTableC() { - Table table = getTableC(); - - String expected = - """ - create table "nutrition" ("id" bigint not null, "vitamin_a" bigint null, \ - "vitamin_d" bigint null, primary key ("id")) inherits ("vaccination");"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - @Test - void testAnalyzeTable() { - Table table = getTableA(); - - String expected = "analyze \"immunization\";"; - - assertEquals(expected, sqlBuilder.analyzeTable(table)); - } - - @Test - void testVacuumTable() { - Table table = getTableA(); - - String expected = "vacuum \"immunization\";"; - - assertEquals(expected, sqlBuilder.vacuumTable(table)); - } - - @Test - void testRenameTable() { - Table table = getTableA(); - - String expected = "alter table \"immunization\" rename to \"vaccination\";"; - - assertEquals(expected, sqlBuilder.renameTable(table, "vaccination")); - } - - @Test - void testDropTableIfExists() { - Table table = getTableA(); - - String expected = "drop table if exists \"immunization\";"; - - assertEquals(expected, sqlBuilder.dropTableIfExists(table)); - } - - @Test - void testDropTableIfExistsString() { - String expected = "drop table if exists \"immunization\";"; - - assertEquals(expected, sqlBuilder.dropTableIfExists("immunization")); - } - - @Test - void testDropTableIfExistsCascade() { - Table table = getTableA(); - - String expected = "drop table if exists \"immunization\" cascade;"; - - assertEquals(expected, sqlBuilder.dropTableIfExistsCascade(table)); - } - - @Test - void testDropTableIfExistsCascadeString() { - String expected = "drop table if exists \"immunization\" cascade;"; - - assertEquals(expected, sqlBuilder.dropTableIfExistsCascade("immunization")); - } - - @Test - void testSwapTable() { - String expected = - """ - drop table if exists "vaccination" cascade; \ - alter table "immunization" rename to "vaccination";"""; - - assertEquals(expected, sqlBuilder.swapTable(getTableA(), "vaccination")); - } - - @Test - void testSetParent() { - String expected = "alter table \"immunization\" inherit \"vaccination\";"; - - assertEquals(expected, sqlBuilder.setParentTable(getTableA(), "vaccination")); - } - - @Test - void testRemoveParent() { - String expected = "alter table \"immunization\" no inherit \"vaccination\";"; - - assertEquals(expected, sqlBuilder.removeParentTable(getTableA(), "vaccination")); - } - - @Test - void testSwapParentTable() { - String expected = - """ - alter table "immunization" no inherit "vaccination"; \ - alter table "immunization" inherit \"nutrition\";"""; - - assertEquals(expected, sqlBuilder.swapParentTable(getTableA(), "vaccination", "nutrition")); - } - - @Test - void testTableExists() { - String expected = - """ - select t.table_name from information_schema.tables t \ - where t.table_schema = 'public' and t.table_name = 'immunization';"""; - - assertEquals(expected, sqlBuilder.tableExists("immunization")); - } - - @Test - void testCountRows() { - String expected = - """ - select count(*) as row_count from "immunization";"""; - - assertEquals(expected, sqlBuilder.countRows(getTableA())); - } - - @Test - void testCreateIndexA() { - List indexes = getIndexesA(); - - String expected = - "create index \"in_immunization_data\" on \"immunization\" using btree(\"data\");"; - - assertEquals(expected, sqlBuilder.createIndex(indexes.get(0))); - } - - @Test - void testCreateIndexB() { - List indexes = getIndexesA(); - - String expected = - "create index \"in_immunization_period_created\" on \"immunization\" using btree(\"period\",\"created\");"; - - assertEquals(expected, sqlBuilder.createIndex(indexes.get(1))); - } - - @Test - void testCreateIndexC() { - List indexes = getIndexesA(); - - String expected = - "create index \"in_immunization_user\" on \"immunization\" using gin(\"user\");"; - - assertEquals(expected, sqlBuilder.createIndex(indexes.get(2))); - } - - @Test - void testCreateIndexD() { - List indexes = getIndexesA(); - - String expected = - "create index \"in_immunization_data_period\" on \"immunization\" using btree(lower(\"data\"),lower(\"period\"));"; - - assertEquals(expected, sqlBuilder.createIndex(indexes.get(3))); - } - - @Test - void testCreateIndexWithDescNullsLast() { - // given - String expected = - "create unique index \"index_a\" on \"table_a\" using btree(\"column_a\" desc nulls last);"; - Index index = - Index.builder() - .name("index_a") - .tableName("table_a") - .unique(Unique.UNIQUE) - .columns(List.of("column_a")) - .sortOrder("desc nulls last") - .build(); - - // when - String createIndexStmt = sqlBuilder.createIndex(index); - - // then - assertEquals(expected, createIndexStmt); - } -} From 43a69e36c51d67a37e929d7b4b423a29056d0d40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 10:55:21 +0100 Subject: [PATCH 16/48] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 49 +++++++++---------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 197e0c28c9f5..5807a308ae4e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -245,35 +245,10 @@ protected void populateTableInternal(AnalyticsTablePartition partition, String f invokeTimeAndLog(sql, "Populating table: '{}'", tableName); } - /** - * The select subquery statement. - * - * @param attribute the {@link TrackedEntityAttribute}. - * @param selectExpression the select expression. - * @param dataFilterClause the data filter clause. - * @return a select statement. - */ - protected String getSelectSubquery( - TrackedEntityAttribute attribute, String selectExpression, String dataFilterClause) { - return replaceQualify( - """ - (select ${selectExpression} from ${trackedentityattributevalue} \ - where trackedentityid=en.trackedentityid \ - and trackedentityattributeid=${attributeId}${dataFilterClause})\ - ${closingParentheses} as ${attributeUid}""", - Map.of( - "selectExpression", selectExpression, - "dataFilterClause", dataFilterClause, - "attributeId", String.valueOf(attribute.getId()), - "closingParentheses", getClosingParentheses(selectExpression), - "attributeUid", quote(attribute.getUid()))); - } - /** * Returns a list of columns based on the given attribute. * * @param attribute the {@link TrackedEntityAttribute}. - * @param withLegendSet indicates whether the attribute has a legend set. * @return a list of {@link AnaylyticsTableColumn}. */ protected List getColumnForAttribute(TrackedEntityAttribute attribute) { @@ -342,4 +317,28 @@ private List getColumnForOrgUnitTrackedEntityAttribute( return columns; } + + /** + * The select subquery statement. + * + * @param attribute the {@link TrackedEntityAttribute}. + * @param selectExpression the select expression. + * @param dataFilterClause the data filter clause. + * @return a select statement. + */ + private String getSelectSubquery( + TrackedEntityAttribute attribute, String selectExpression, String dataFilterClause) { + return replaceQualify( + """ + (select ${selectExpression} from ${trackedentityattributevalue} \ + where trackedentityid=en.trackedentityid \ + and trackedentityattributeid=${attributeId}${dataFilterClause})\ + ${closingParentheses} as ${attributeUid}""", + Map.of( + "selectExpression", selectExpression, + "dataFilterClause", dataFilterClause, + "attributeId", String.valueOf(attribute.getId()), + "closingParentheses", getClosingParentheses(selectExpression), + "attributeUid", quote(attribute.getUid()))); + } } From 3f162fb86528155f28cf52043d9430754a7dcbac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 17:25:19 +0100 Subject: [PATCH 17/48] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 40 +++---------------- 1 file changed, 6 insertions(+), 34 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 5807a308ae4e..ca0f7bf120ca 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -38,6 +38,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.partition.PartitionManager; import org.hisp.dhis.analytics.table.model.AnalyticsDimensionType; @@ -64,6 +65,7 @@ /** * @author Markus Bekken */ +@Slf4j public abstract class AbstractEventJdbcTableManager extends AbstractJdbcTableManager { public AbstractEventJdbcTableManager( IdentifiableObjectManager idObjectManager, @@ -119,41 +121,15 @@ protected Skip skipIndex(ValueType valueType, boolean hasOptionSet) { return skipIndex ? Skip.SKIP : Skip.INCLUDE; } - /** - * Returns a select expression for a data element value, handling casting to the appropriate data - * type based on the given value type. - * - * @param valueType the {@link ValueType}. - * @param columnName the column name. - * @return a select expression. - */ - protected String getSelectExpression(ValueType valueType, String columnName) { - return getSelectExpression(valueType, columnName, false); - } - - /** - * Returns a select expression for a tracked entity attribute, handling casting to the appropriate - * data type based on the given value type. - * - * @param valueType the {@link ValueType}. - * @param columnName the column name. - * @return a select expression. - */ - protected String getSelectExpressionForAttribute(ValueType valueType, String columnName) { - return getSelectExpression(valueType, columnName, true); - } - /** * Returns a select expression, potentially with a cast statement, based on the given value type. * Handles data element and tracked entity attribute select expressions. * * @param valueType the {@link ValueType} to represent as database column type. * @param columnExpression the expression or name of the column to be selected. - * @param isTea whether the selection is in the context of a tracked entity attribute. When true, - * organisation unit selections will include an additional subquery wrapper. * @return a select expression appropriate for the given value type and context. */ - private String getSelectExpression(ValueType valueType, String columnExpression, boolean isTea) { + protected String getSelectExpression(ValueType valueType, String columnExpression) { if (valueType.isDecimal()) { return getCastExpression(columnExpression, NUMERIC_REGEXP, sqlBuilder.dataTypeDouble()); } else if (valueType.isInteger()) { @@ -169,12 +145,6 @@ private String getSelectExpression(ValueType valueType, String columnExpression, """ ST_GeomFromGeoJSON('{"type":"Point", "coordinates":' || (%s) || ', "crs":{"type":"name", "properties":{"name":"EPSG:4326"}}}')""", columnExpression); - } else if (valueType.isOrganisationUnit()) { - String ouClause = - isTea - ? "ou.uid from ${organisationunit} ou where ou.uid = (select ${columnName}" - : "ou.uid from ${organisationunit} ou where ou.uid = ${columnName}"; - return replaceQualify(ouClause, Map.of("columnName", columnExpression)); } else { return columnExpression; } @@ -242,6 +212,8 @@ protected void populateTableInternal(AnalyticsTablePartition partition, String f sql += fromClause; + log.info("Populate table SQL: '{}'", sql); + invokeTimeAndLog(sql, "Populating table: '{}'", tableName); } @@ -255,7 +227,7 @@ protected List getColumnForAttribute(TrackedEntityAttribut List columns = new ArrayList<>(); DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); - String selectExpression = getSelectExpressionForAttribute(attribute.getValueType(), "value"); + String selectExpression = getSelectExpression(attribute.getValueType(), "value"); String dataFilterClause = getDataFilterClause(attribute); String sql = getSelectSubquery(attribute, selectExpression, dataFilterClause); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); From 6024282469f5e1f2245f08107b25f03ca13b665d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 17:26:42 +0100 Subject: [PATCH 18/48] fix: Update code --- .../dhis/analytics/table/AbstractEventJdbcTableManager.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index ca0f7bf120ca..c9eed3e1bbc2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -38,7 +38,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.partition.PartitionManager; import org.hisp.dhis.analytics.table.model.AnalyticsDimensionType; @@ -65,7 +64,6 @@ /** * @author Markus Bekken */ -@Slf4j public abstract class AbstractEventJdbcTableManager extends AbstractJdbcTableManager { public AbstractEventJdbcTableManager( IdentifiableObjectManager idObjectManager, @@ -212,8 +210,6 @@ protected void populateTableInternal(AnalyticsTablePartition partition, String f sql += fromClause; - log.info("Populate table SQL: '{}'", sql); - invokeTimeAndLog(sql, "Populating table: '{}'", tableName); } From 35a8daa9d2d4e9390f4665fca682c2fb27d882aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 17:39:32 +0100 Subject: [PATCH 19/48] fix: Update code --- .../table/JdbcEnrollmentAnalyticsTableManagerTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java index 12e6110c1646..7a355952e8be 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java @@ -147,10 +147,10 @@ void verifyTeiTypeOrgUnitFetchesOuUidWhenPopulatingEventAnalyticsTable() { String ouQuery = format( """ - (select ou.%s from \"organisationunit\" ou where ou.uid = \ - (select value from \"trackedentityattributevalue\" where trackedentityid=en.trackedentityid and \ - trackedentityattributeid=9999)) as %s""", - "uid", quote(tea.getUid())); + (select value from "trackedentityattributevalue" \ + where trackedentityid=en.trackedentityid and \ + trackedentityattributeid=9999) as %s""", + quote(tea.getUid())); assertThat(sql.getValue(), containsString(ouQuery)); } From 41ca7107d5c7810d861d323331575b7a3e4393f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 18:17:20 +0100 Subject: [PATCH 20/48] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 32 ++--- .../JdbcEventAnalyticsTableManagerTest.java | 111 +++++++++++------- 2 files changed, 82 insertions(+), 61 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 8c652952993f..976872fecffd 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -51,6 +51,7 @@ import java.util.Map; import java.util.Objects; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.Validate; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -492,7 +493,7 @@ private List getColumnForDataElement( sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); String selectExpression = getSelectExpression(dataElement.getValueType(), columnExpression); String dataFilterClause = getDataFilterClause(dataElement); - String sql = getSelectForInsert(dataElement, selectExpression, dataFilterClause); + String sql = String.format("%s as %s", selectExpression, quote(dataElement.getUid())); Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); if (withLegendSet) { @@ -533,7 +534,7 @@ private List getColumnForOrgUnitDataElement( if (isSpatialSupport()) { String fromType = "ou.geometry " + fromClause; - String geoSql = getSelectForInsert(dataElement, fromType, dataFilterClause); + String geoSql = getOrgUnitSelectExpression(dataElement, fromType, dataFilterClause); columns.add( AnalyticsTableColumn.builder() @@ -546,7 +547,7 @@ private List getColumnForOrgUnitDataElement( } String fromTypeSql = "ou.name " + fromClause; - String ouNameSql = getSelectForInsert(dataElement, fromTypeSql, dataFilterClause); + String ouNameSql = getOrgUnitSelectExpression(dataElement, fromTypeSql, dataFilterClause); columns.add( AnalyticsTableColumn.builder() @@ -624,31 +625,24 @@ private List getColumnForAttributeWithLegendSet( } /** - * Retyrns a select statement for the given select expression. + * Returns a select statement for the given data element with value type org unit. * * @param dataElement the data element to create the select statement for. * @param selectExpression the select expression. * @param dataFilterClause the data filter clause. * @return a select expression. */ - private String getSelectForInsert( + private String getOrgUnitSelectExpression( DataElement dataElement, String selectExpression, String dataFilterClause) { - String sqlTemplate = - dataElement.getValueType().isOrganisationUnit() - ? "(select ${selectExpression} ${dataClause})${closingParentheses} as ${uid}" - : "${selectExpression}${closingParentheses} as ${uid}"; - + Validate.isTrue(dataElement.getValueType().isOrganisationUnit()); + String prts = getClosingParentheses(selectExpression); return replaceQualify( - sqlTemplate, + "(select ${selectExpression} ${dataFilterClause})${closingParentheses} as ${uid}", Map.of( - "selectExpression", - selectExpression, - "dataClause", - dataFilterClause, - "closingParentheses", - getClosingParentheses(selectExpression), - "uid", - quote(dataElement.getUid()))); + "selectExpression", selectExpression, + "dataFilterClause", dataFilterClause, + "closingParentheses", prts, + "uid", quote(dataElement.getUid()))); } /** diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java index 4e64792c8546..662fd57c6854 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java @@ -316,10 +316,15 @@ void verifyClientSideTimestampsColumns() { assertThat( lastUpdated.getSelectExpression(), is( - "case when ev.lastupdatedatclient is not null then ev.lastupdatedatclient else ev.lastupdated end")); + """ + case when ev.lastupdatedatclient is not null \ + then ev.lastupdatedatclient else ev.lastupdated end""")); assertThat( created.getSelectExpression(), - is("case when ev.createdatclient is not null then ev.createdatclient else ev.created end")); + is( + """ + case when ev.createdatclient is not null \ + then ev.createdatclient else ev.created end""")); } @Test @@ -389,13 +394,7 @@ void verifyGetTableWithDataElements() { case when eventdatavalues #>> '{deabcdefghW, value}' ~* '^\\d{4}-\\d{2}-\\d{2}(\\s|T)?((\\d{2}:)(\\d{2}:)?(\\d{2}))?(|.(\\d{3})|.(\\d{3})Z)?$' \ then cast(eventdatavalues #>> '{deabcdefghW, value}' as timestamp) else null end as "deabcdefghW\""""; String aliasD5 = - "(select ou.uid from \"organisationunit\" ou where ou.uid = " - + "eventdatavalues #>> '{" - + d5.getUid() - + ", value}' " - + ") as \"" - + d5.getUid() - + "\""; + "eventdatavalues #>> '{" + d5.getUid() + ", value}' as \"" + d5.getUid() + "\""; String aliasD6 = """ case when eventdatavalues #>> '{deabcdefghH, value}' ~* '^(-?[0-9]+)(\\.[0-9]+)?$' \ @@ -509,10 +508,16 @@ void verifyGetTableWithTrackedEntityAttribute() { String aliasD1 = """ eventdatavalues #>> '{deabcdefghZ, value}' as "deabcdefghZ\""""; + String aliasTeaUid = + """ + (select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid \ + and trackedentityattributeid=%d) as "%s\""""; + String aliasTea1 = - "(select %s from \"organisationunit\" ou where ou.uid = (select value from " - + "\"trackedentityattributevalue\" where trackedentityid=en.trackedentityid and " - + "trackedentityattributeid=%d)) as \"%s\""; + """ + (select %s from "organisationunit" ou where ou.uid = \ + (select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid \ + and trackedentityattributeid=%d)) as "%s\""""; AnalyticsTableUpdateParams params = AnalyticsTableUpdateParams.newBuilder() @@ -544,14 +549,14 @@ void verifyGetTableWithTrackedEntityAttribute() { TEXT, toSelectExpression(aliasD1, d1.getUid()), Skip.SKIP) // ValueType.TEXT - .addColumn( - tea1.getUid(), TEXT, String.format(aliasTea1, "ou.uid", tea1.getId(), tea1.getUid())) - // Second Geometry column created from the OU column above + .addColumn(tea1.getUid(), TEXT, String.format(aliasTeaUid, tea1.getId(), tea1.getUid())) + // Org unit geometry column .addColumn( tea1.getUid() + "_geom", GEOMETRY, String.format(aliasTea1, "ou.geometry", tea1.getId(), tea1.getUid()), IndexType.GIST) + // Org unit name column .addColumn( tea1.getUid() + "_name", TEXT, @@ -599,16 +604,21 @@ void verifyDataElementTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable( subject.populateTable(params, partition); verify(jdbcTemplate).execute(sql.capture()); - String ouQuery = - "(select ou.%s from \"organisationunit\" ou where ou.uid = " - + "eventdatavalues #>> '{" - + d5.getUid() - + ", value}' ) as \"" - + d5.getUid() - + "\""; - - assertThat(sql.getValue(), containsString(String.format(ouQuery, "uid"))); - assertThat(sql.getValue(), containsString(String.format(ouQuery, "name"))); + String ouUidQuery = + String.format( + """ + eventdatavalues #>> '{%s, value}' as %s""", + d5.getUid(), quote(d5.getUid())); + + String ouNameQuery = + String.format( + """ + (select ou.name from "organisationunit" ou where ou.uid = \ + eventdatavalues #>> '{%s, value}' ) as %s""", + d5.getUid(), quote(d5.getUid())); + + assertThat(sql.getValue(), containsString(ouUidQuery)); + assertThat(sql.getValue(), containsString(ouNameQuery)); } @Test @@ -649,15 +659,23 @@ void verifyTeiTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() { subject.populateTable(params, partition); verify(jdbcTemplate).execute(sql.capture()); - String ouQuery = - "(select ou.%s from \"organisationunit\" ou where ou.uid = " - + "(select value from \"trackedentityattributevalue\" where trackedentityid=en.trackedentityid and " - + "trackedentityattributeid=9999)) as \"" - + tea.getUid() - + "\""; - - assertThat(sql.getValue(), containsString(String.format(ouQuery, "uid"))); - assertThat(sql.getValue(), containsString(String.format(ouQuery, "name"))); + String ouUidQuery = + String.format( + """ + (select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid and \ + trackedentityattributeid=9999) as %s""", + quote(tea.getUid())); + + String ouNameQuery = + String.format( + """ + (select ou.name from "organisationunit" ou where ou.uid = \ + (select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid and \ + trackedentityattributeid=9999)) as %s""", + quote(tea.getUid())); + + assertThat(sql.getValue(), containsString(ouUidQuery)); + assertThat(sql.getValue(), containsString(ouNameQuery)); } @Test @@ -937,14 +955,23 @@ void verifyTeaTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() { verify(jdbcTemplate).execute(sql.capture()); - String ouQuery = - """ - (select ou.%s from \"organisationunit\" ou where ou.uid = \ - (select value from \"trackedentityattributevalue\" where trackedentityid=en.trackedentityid and \ - trackedentityattributeid=9999)) as %s"""; - - assertThat(sql.getValue(), containsString(String.format(ouQuery, "uid", quote(tea.getUid())))); - assertThat(sql.getValue(), containsString(String.format(ouQuery, "name", quote(tea.getUid())))); + String ouUidQuery = + String.format( + """ + select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid \ + and trackedentityattributeid=9999) as %s""", + quote(tea.getUid())); + + String ouNameQuery = + String.format( + """ + (select ou.name from "organisationunit" ou where ou.uid = \ + (select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid \ + and trackedentityattributeid=9999)) as %s""", + quote(tea.getUid())); + + assertThat(sql.getValue(), containsString(ouUidQuery)); + assertThat(sql.getValue(), containsString(ouNameQuery)); } private String toSelectExpression(String template, String uid) { From 7d7360d11addb3eb857764508c02026a2d0db5a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 18:54:04 +0100 Subject: [PATCH 21/48] fix: Update code --- .../table/JdbcEventAnalyticsTableManagerDorisTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java index 351950d9f48c..54bdc761b617 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java @@ -174,8 +174,7 @@ void verifyGetTableWithDataElements() { "case when json_unquote(json_extract(eventdatavalues, '$.%s.value')) = 'true' then 1 when json_unquote(json_extract(eventdatavalues, '$.%s.value')) = 'false' then 0 else null end as `%s`"; String aliasD = "case when json_unquote(json_extract(eventdatavalues, '$.%s.value')) regexp '^\\d{4}-\\d{2}-\\d{2}(\\s|T)?((\\d{2}:)(\\d{2}:)?(\\d{2}))?(|.(\\d{3})|.(\\d{3})Z)?$' then cast(json_unquote(json_extract(eventdatavalues, '$.%s.value')) as datetime) else null end as `%s`"; - String aliasE = - "(select ou.uid from dhis2.public.`organisationunit` ou where ou.uid = json_unquote(json_extract(eventdatavalues, '$.%s.value')) ) as `%s`"; + String aliasE = "json_unquote(json_extract(eventdatavalues, '$.%s.value')) as `%s`"; String aliasF = "(select ou.name from dhis2.public.`organisationunit` ou where ou.uid = json_unquote(json_extract(eventdatavalues, '$.%s.value')) ) as `%s`"; String aliasG = From 4fff29ee7ee16da30f83d086f7ddb872364c6ecf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 18:55:27 +0100 Subject: [PATCH 22/48] fix: Update code --- .../table/JdbcEventAnalyticsTableManagerTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java index 662fd57c6854..c24088b87470 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java @@ -662,16 +662,16 @@ void verifyTeiTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() { String ouUidQuery = String.format( """ - (select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid and \ - trackedentityattributeid=9999) as %s""", + (select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid and \ + trackedentityattributeid=9999) as %s""", quote(tea.getUid())); String ouNameQuery = String.format( """ - (select ou.name from "organisationunit" ou where ou.uid = \ - (select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid and \ - trackedentityattributeid=9999)) as %s""", + (select ou.name from "organisationunit" ou where ou.uid = \ + (select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid and \ + trackedentityattributeid=9999)) as %s""", quote(tea.getUid())); assertThat(sql.getValue(), containsString(ouUidQuery)); From 2125b913acabb5a008cb4f5621c7e2978cc39bd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 18:56:18 +0100 Subject: [PATCH 23/48] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManagerTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java index c24088b87470..9e624f88547e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java @@ -323,8 +323,8 @@ void verifyClientSideTimestampsColumns() { created.getSelectExpression(), is( """ - case when ev.createdatclient is not null \ - then ev.createdatclient else ev.created end""")); + case when ev.createdatclient is not null \ + then ev.createdatclient else ev.created end""")); } @Test From 3858aa78f4ba617bd451491d432ff20d47a92d66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 18:56:56 +0100 Subject: [PATCH 24/48] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManagerTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java index 9e624f88547e..dc9b0903986c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java @@ -387,8 +387,8 @@ void verifyGetTableWithDataElements() { then cast(eventdatavalues #>> '{deabcdefghP, value}' as double precision) else null end as "deabcdefghP\""""; String aliasD3 = """ - case when eventdatavalues #>> '{deabcdefghY, value}' = 'true' then 1 \ - when eventdatavalues #>> '{deabcdefghY, value}' = 'false' then 0 else null end as "deabcdefghY\""""; + case when eventdatavalues #>> '{deabcdefghY, value}' = 'true' then 1 \ + when eventdatavalues #>> '{deabcdefghY, value}' = 'false' then 0 else null end as "deabcdefghY\""""; String aliasD4 = """ case when eventdatavalues #>> '{deabcdefghW, value}' ~* '^\\d{4}-\\d{2}-\\d{2}(\\s|T)?((\\d{2}:)(\\d{2}:)?(\\d{2}))?(|.(\\d{3})|.(\\d{3})Z)?$' \ From 4a7452f3726833875e88f41031d5b462e3cef80a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 4 Dec 2024 09:44:37 +0100 Subject: [PATCH 25/48] Revert "feat: skipAnalytics and ConfidentialFlag only for QUERY Analytics [DHIS2-18259] (#19350)" This reverts commit dc0fa5ef4343ad2eba3ddd8c059735edbf7b8d4a. --- .../dhis/common/DimensionalItemObject.java | 4 - .../org/hisp/dhis/feedback/ErrorCode.java | 3 - .../java/org/hisp/dhis/program/Program.java | 34 +++- .../org/hisp/dhis/program/ProgramTest.java | 5 + .../analytics/event/EventQueryParams.java | 5 - ...tEnrollmentAnalyticsDimensionsService.java | 41 ++--- ...efaultEventAnalyticsDimensionsService.java | 15 +- .../data/DefaultEventDataQueryService.java | 69 -------- .../table/AbstractEventJdbcTableManager.java | 2 +- .../JdbcEnrollmentAnalyticsTableManager.java | 2 +- .../table/JdbcEventAnalyticsTableManager.java | 8 +- ...dbcTrackedEntityAnalyticsTableManager.java | 12 +- .../DefaultEventDataQueryServiceTest.java | 151 ------------------ ...rackedEntityAnalyticsTableManagerTest.java | 7 +- .../analytics/AnalyticsDimensionsTest.java | 2 +- 15 files changed, 89 insertions(+), 271 deletions(-) delete mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryServiceTest.java diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalItemObject.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalItemObject.java index 4134b7314387..c12c4e954df7 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalItemObject.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalItemObject.java @@ -87,8 +87,4 @@ public interface DimensionalItemObject extends NameableObject { default int getPeriodOffset() { return (getQueryMods() != null) ? getQueryMods().getPeriodOffset() : 0; } - - default boolean isOfType(DimensionItemType type) { - return type == getDimensionItemType(); - } } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java index 9dcf3008b561..1c1cf44d0afe 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java @@ -505,9 +505,6 @@ public enum ErrorCode { E7236("Program stage '{0}' is not associated to program '{0}'"), E7237("Sorting must have a valid dimension and a direction"), E7238("Sorting dimension ‘{0}’ is not a column"), - E7239( - "Tracked Entity Attributes marked as 'confidential' can only be used in aggregate analytics: `{0}`"), - E7240("Data Elements marked as 'skipAnalytics' can only be used in aggregate analytics: `{0}`"), /* TE analytics */ E7250("Dimension is not a fully qualified: `{0}`"), diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/Program.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/Program.java index 8d9b8da95009..d6ca3575963d 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/Program.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/Program.java @@ -37,9 +37,11 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import org.hisp.dhis.category.CategoryCombo; @@ -260,12 +262,26 @@ public Set getDataElements() { return programStages.stream().flatMap(ps -> ps.getDataElements().stream()).collect(toSet()); } + /** + * Returns all data elements which are part of the stages of this program and is not skipped in + * analytics. + */ + public Set getAnalyticsDataElements() { + return programStages.stream() + .map(ProgramStage::getProgramStageDataElements) + .flatMap(Collection::stream) + .filter(Objects::nonNull) + .filter(psde -> !psde.getSkipAnalytics()) + .map(ProgramStageDataElement::getDataElement) + .collect(toSet()); + } + /** * Returns data elements which are part of the stages of this program which have a legend set and * is of numeric value type. */ - public Set getDataElementsWithLegendSet() { - return getDataElements().stream() + public Set getAnalyticsDataElementsWithLegendSet() { + return getAnalyticsDataElements().stream() .filter(de -> de.hasLegendSet() && de.isNumericType()) .collect(toSet()); } @@ -280,13 +296,23 @@ public List getTrackedEntityAttributes() { .collect(Collectors.toList()); } + /** + * Returns non-confidential TrackedEntityAttributes from ProgramTrackedEntityAttributes. Use + * getAttributes() to access the persisted attribute list. + */ + public List getNonConfidentialTrackedEntityAttributes() { + return getTrackedEntityAttributes().stream() + .filter(a -> !a.isConfidentialBool()) + .collect(Collectors.toList()); + } + /** * Returns TrackedEntityAttributes from ProgramTrackedEntityAttributes which have a legend set and * is of numeric value type. */ - public List getTrackedEntityAttributesWithLegendSet() { + public List getNonConfidentialTrackedEntityAttributesWithLegendSet() { return getTrackedEntityAttributes().stream() - .filter(a -> a.hasLegendSet() && a.isNumericType()) + .filter(a -> !a.isConfidentialBool() && a.hasLegendSet() && a.isNumericType()) .collect(Collectors.toList()); } diff --git a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/program/ProgramTest.java b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/program/ProgramTest.java index 7f7cbff8216b..3555686ada7e 100644 --- a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/program/ProgramTest.java +++ b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/program/ProgramTest.java @@ -85,6 +85,7 @@ void testGetAnalyticsDataElements() { assertEquals(2, prA.getDataElements().size()); assertTrue(prA.getDataElements().contains(deA)); assertTrue(prA.getDataElements().contains(deB)); + assertEquals(1, prA.getAnalyticsDataElements().size()); assertTrue(prA.getDataElements().contains(deA)); } @@ -105,6 +106,7 @@ void testCopyOfWithPropertyValuesSet() { // check equal assertEquals(original.getAccess(), copy.getAccess()); assertEquals(original.getAccessLevel(), copy.getAccessLevel()); + assertEquals(original.getAnalyticsDataElements(), copy.getAnalyticsDataElements()); assertEquals(original.getCategoryCombo(), copy.getCategoryCombo()); assertEquals(original.getCompleteEventsExpiryDays(), copy.getCompleteEventsExpiryDays()); assertEquals(original.getDataElements(), copy.getDataElements()); @@ -178,7 +180,10 @@ void testCopyOfWithNulls() { assertEquals("copynull", copy.getName()); assertEquals(original.getAccessLevel(), copy.getAccessLevel()); assertEquals(original.getDescription(), copy.getDescription()); + assertTrue(copy.getAnalyticsDataElements().isEmpty()); assertTrue(copy.getDataElements().isEmpty()); + assertTrue(copy.getNonConfidentialTrackedEntityAttributes().isEmpty()); + assertTrue(copy.getNonConfidentialTrackedEntityAttributesWithLegendSet().isEmpty()); assertTrue(copy.getNotificationTemplates().isEmpty()); assertTrue(copy.getOrganisationUnits().isEmpty()); assertTrue(copy.getProgramAttributes().isEmpty()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java index 7848156e434d..4d6f0683f691 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java @@ -37,7 +37,6 @@ import static org.hisp.dhis.common.DimensionalObject.PERIOD_DIM_ID; import static org.hisp.dhis.common.DimensionalObjectUtils.asList; import static org.hisp.dhis.common.DimensionalObjectUtils.asTypedList; -import static org.hisp.dhis.common.RequestTypeAware.EndpointAction.AGGREGATE; import static org.hisp.dhis.common.RequestTypeAware.EndpointAction.QUERY; import com.google.common.base.MoreObjects; @@ -999,10 +998,6 @@ public boolean isRowContext() { return rowContext; } - public boolean includeConfidentialOrSkipAnalyticsItems() { - return endpointAction == AGGREGATE; - } - // ------------------------------------------------------------------------- // Builder of immutable instances // ------------------------------------------------------------------------- diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEnrollmentAnalyticsDimensionsService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEnrollmentAnalyticsDimensionsService.java index c13b40d3d4df..2894d3771eb6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEnrollmentAnalyticsDimensionsService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEnrollmentAnalyticsDimensionsService.java @@ -31,26 +31,22 @@ import static org.hisp.dhis.analytics.common.DimensionsServiceCommon.OperationType.QUERY; import static org.hisp.dhis.analytics.common.DimensionsServiceCommon.collectDimensions; import static org.hisp.dhis.analytics.common.DimensionsServiceCommon.filterByValueType; -import static org.hisp.dhis.analytics.event.data.DefaultEventAnalyticsDimensionsService.getTeasIfRegistration; import static org.hisp.dhis.common.PrefixedDimensions.ofItemsWithProgram; import static org.hisp.dhis.common.PrefixedDimensions.ofProgramStageDataElements; import java.util.Collection; import java.util.List; import java.util.Optional; -import java.util.Set; -import java.util.function.Predicate; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import org.hisp.dhis.analytics.common.DimensionsServiceCommon; -import org.hisp.dhis.analytics.common.DimensionsServiceCommon.OperationType; import org.hisp.dhis.analytics.event.EnrollmentAnalyticsDimensionsService; import org.hisp.dhis.common.PrefixedDimension; import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramService; import org.hisp.dhis.program.ProgramStage; -import org.hisp.dhis.program.ProgramStageDataElement; import org.hisp.dhis.security.acl.AclService; +import org.hisp.dhis.trackedentity.TrackedEntityAttribute; import org.hisp.dhis.user.CurrentUserUtil; import org.hisp.dhis.user.UserDetails; import org.springframework.stereotype.Service; @@ -81,7 +77,9 @@ public List getQueryDimensionsByProgramId(String programId) { .collect(Collectors.toSet())), getProgramStageDataElements(QUERY, program), filterByValueType( - QUERY, ofItemsWithProgram(program, getTeasIfRegistration(program)))))) + QUERY, + ofItemsWithProgram( + program, getTeasIfRegistrationAndNotConfidential(program)))))) .orElse(List.of()); } @@ -89,22 +87,14 @@ private Collection getProgramStageDataElements( DimensionsServiceCommon.OperationType operationType, Program program) { return program.getProgramStages().stream() .map(ProgramStage::getProgramStageDataElements) - .map(psdes -> excludeIfSkipAnalytics(operationType, psdes)) - .map(psdes -> filterByValueType(operationType, ofProgramStageDataElements(psdes))) + .map( + programStageDataElements -> + filterByValueType( + operationType, ofProgramStageDataElements(programStageDataElements))) .flatMap(Collection::stream) .collect(Collectors.toList()); } - private Set excludeIfSkipAnalytics( - OperationType operationType, Set programStageDataElements) { - if (operationType == QUERY) { - return programStageDataElements.stream() - .filter(Predicate.not(ProgramStageDataElement::getSkipAnalytics)) - .collect(Collectors.toSet()); - } - return programStageDataElements; - } - @Override public List getAggregateDimensionsByProgramStageId(String programId) { return Optional.of(programId) @@ -119,4 +109,19 @@ public List getAggregateDimensionsByProgramStageId(String pro ofItemsWithProgram(program, program.getTrackedEntityAttributes()))))) .orElse(List.of()); } + + private Collection getTeasIfRegistrationAndNotConfidential( + Program program) { + return Optional.of(program) + .filter(Program::isRegistration) + .map(Program::getTrackedEntityAttributes) + .orElse(List.of()) + .stream() + .filter(this::isNotConfidential) + .collect(Collectors.toList()); + } + + private boolean isNotConfidential(TrackedEntityAttribute trackedEntityAttribute) { + return !trackedEntityAttribute.isConfidentialBool(); + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsDimensionsService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsDimensionsService.java index a67b341b61df..76086a2c4b27 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsDimensionsService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsDimensionsService.java @@ -135,7 +135,9 @@ private List dimensions(ProgramStage programStage) { .filter(pi -> aclService.canRead(currentUserDetails, pi)) .collect(Collectors.toSet())), filterByValueType(QUERY, ofDataElements(programStage)), - filterByValueType(QUERY, ofItemsWithProgram(p, getTeasIfRegistration(p))), + filterByValueType( + QUERY, + ofItemsWithProgram(p, getTeasIfRegistrationAndNotConfidential(p))), ofItemsWithProgram(p, getCategories(p)), ofItemsWithProgram(p, getAttributeCategoryOptionGroupSetsIfNeeded(p))))) .orElse(List.of()); @@ -185,10 +187,17 @@ private List getCategories(Program program) { .orElse(List.of()); } - static List getTeasIfRegistration(Program program) { + private List getTeasIfRegistrationAndNotConfidential(Program program) { return Optional.of(program) .filter(Program::isRegistration) .map(Program::getTrackedEntityAttributes) - .orElse(List.of()); + .orElse(List.of()) + .stream() + .filter(this::isNotConfidential) + .collect(Collectors.toList()); + } + + private boolean isNotConfidential(TrackedEntityAttribute trackedEntityAttribute) { + return !trackedEntityAttribute.isConfidentialBool(); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryService.java index 3d20c79d5a61..b08af886985a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryService.java @@ -36,8 +36,6 @@ import static org.hisp.dhis.analytics.event.data.DefaultEventDataQueryService.SortableItems.translateItemIfNecessary; import static org.hisp.dhis.analytics.util.AnalyticsUtils.illegalQueryExSupplier; import static org.hisp.dhis.analytics.util.AnalyticsUtils.throwIllegalQueryEx; -import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; -import static org.hisp.dhis.common.DimensionItemType.PROGRAM_ATTRIBUTE; import static org.hisp.dhis.common.DimensionalObject.DIMENSION_NAME_SEP; import static org.hisp.dhis.common.DimensionalObject.PERIOD_DIM_ID; import static org.hisp.dhis.common.DimensionalObjectUtils.getDimensionFromParam; @@ -54,7 +52,6 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; -import java.util.stream.Stream; import lombok.Getter; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.StringUtils; @@ -70,7 +67,6 @@ import org.hisp.dhis.analytics.table.EnrollmentAnalyticsColumnName; import org.hisp.dhis.analytics.table.EventAnalyticsColumnName; import org.hisp.dhis.common.BaseDimensionalItemObject; -import org.hisp.dhis.common.BaseIdentifiableObject; import org.hisp.dhis.common.DimensionalItemObject; import org.hisp.dhis.common.DimensionalObject; import org.hisp.dhis.common.EventAnalyticalObject; @@ -94,7 +90,6 @@ import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramService; import org.hisp.dhis.program.ProgramStage; -import org.hisp.dhis.program.ProgramStageDataElement; import org.hisp.dhis.program.ProgramStageService; import org.hisp.dhis.setting.UserSettings; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; @@ -228,73 +223,9 @@ public EventQueryParams getFromRequest(EventDataQueryRequest request, boolean an eventQueryParams = builder.build(); } - validateQueryParamsForConfidentialAndSkipAnalytics(eventQueryParams); - return eventQueryParams; } - static void validateQueryParamsForConfidentialAndSkipAnalytics( - EventQueryParams eventQueryParams) { - if (eventQueryParams.includeConfidentialOrSkipAnalyticsItems()) { - return; - } - Set confidentialAttributes = - Stream.concat( - eventQueryParams.getItems().stream(), eventQueryParams.getItemFilters().stream()) - .map(QueryItem::getItem) - .filter(Objects::nonNull) - .filter(dimObj -> dimObj.isOfType(PROGRAM_ATTRIBUTE)) - .map(TrackedEntityAttribute.class::cast) - .filter(TrackedEntityAttribute::isConfidentialBool) - .collect(Collectors.toSet()); - - if (!confidentialAttributes.isEmpty()) { - throw new IllegalQueryException( - new ErrorMessage( - ErrorCode.E7239, - confidentialAttributes.stream() - .map(TrackedEntityAttribute::getUid) - .collect(Collectors.joining(", ")))); - } - - Set skipAnalyticsDataElements = - Stream.concat( - eventQueryParams.getItems().stream(), eventQueryParams.getItemFilters().stream()) - .filter(item -> item.getItem().isOfType(DATA_ELEMENT)) - .filter(DefaultEventDataQueryService::isSkipAnalytics) - .map(item -> (DataElement) item.getItem()) - .collect(Collectors.toSet()); - - if (!skipAnalyticsDataElements.isEmpty()) { - throw new IllegalQueryException( - new ErrorMessage( - ErrorCode.E7240, - skipAnalyticsDataElements.stream() - .map(DataElement::getUid) - .collect(Collectors.joining(", ")))); - } - } - - /** - * Checks if the data element is marked as skip analytics by looking at the program stage data - * elements associated with the program stage. If any of the program stage data elements has the - * skip analytics flag set to true, the data element in it is considered to be skipAnalytics. - * - * @param item the query item - * @return true if the data element is marked as skip analytics, false otherwise - */ - static boolean isSkipAnalytics(QueryItem item) { - return Optional.of(item) - .map(QueryItem::getProgramStage) - .map(ProgramStage::getProgramStageDataElements) - .orElse(Set.of()) - .stream() - .filter(ProgramStageDataElement::getSkipAnalytics) - .map(ProgramStageDataElement::getDataElement) - .map(BaseIdentifiableObject::getUid) - .anyMatch(uid -> uid.equals(item.getItem().getUid())); - } - private boolean hasPeriodDimension(EventQueryParams eventQueryParams) { return Objects.nonNull(getPeriodDimension(eventQueryParams)); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 5179b999407f..c164dc6d6989 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -214,7 +214,7 @@ protected void populateTableInternal(AnalyticsTablePartition partition, String f * Returns a list of columns based on the given attribute. * * @param attribute the {@link TrackedEntityAttribute}. - * @return a list of {@link AnalyticsTableColumn}. + * @return a list of {@link AnaylyticsTableColumn}. */ protected List getColumnForAttribute(TrackedEntityAttribute attribute) { List columns = new ArrayList<>(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index 59bdf500f0a5..d18fbdffd80b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -187,7 +187,7 @@ private List getColumns(Program program) { * @return a list of {@link AnalyticsTableColumn}. */ private List getTrackedEntityAttributeColumns(Program program) { - return program.getTrackedEntityAttributes().stream() + return program.getNonConfidentialTrackedEntityAttributes().stream() .map(this::getColumnForAttribute) .flatMap(Collection::stream) .toList(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 4b0b572b390c..7a4ff2e0ad6c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -461,12 +461,12 @@ private List getAttributeCategoryColumns(Program program) private List getDataElementColumns(Program program) { List columns = new ArrayList<>(); columns.addAll( - program.getDataElements().stream() + program.getAnalyticsDataElements().stream() .map(de -> getColumnForDataElement(de, false)) .flatMap(Collection::stream) .toList()); columns.addAll( - program.getDataElementsWithLegendSet().stream() + program.getAnalyticsDataElementsWithLegendSet().stream() .map(de -> getColumnForDataElement(de, true)) .flatMap(Collection::stream) .toList()); @@ -567,12 +567,12 @@ private List getColumnForOrgUnitDataElement( private List getAttributeColumns(Program program) { List columns = new ArrayList<>(); columns.addAll( - program.getTrackedEntityAttributes().stream() + program.getNonConfidentialTrackedEntityAttributes().stream() .map(this::getColumnForAttribute) .flatMap(Collection::stream) .toList()); columns.addAll( - program.getTrackedEntityAttributesWithLegendSet().stream() + program.getNonConfidentialTrackedEntityAttributesWithLegendSet().stream() .map(this::getColumnForAttributeWithLegendSet) .flatMap(Collection::stream) .toList()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index e2d0bdb9e5ac..4a483d156d37 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -88,7 +88,8 @@ public class JdbcTrackedEntityAnalyticsTableManager extends AbstractJdbcTableManager { private static final String PROGRAMS_BY_TET_KEY = "programsByTetUid"; - private static final String ALL_TET_ATTRIBUTES = "allTetAttributes"; + private static final String ALL_NON_CONFIDENTIAL_TET_ATTRIBUTES = + "allNonConfidentialTetAttributes"; private final TrackedEntityTypeService trackedEntityTypeService; @@ -199,9 +200,12 @@ private List getColumns( .build())); List trackedEntityAttributes = - getAllTrackedEntityAttributes(trackedEntityType, programsByTetUid).toList(); + getAllTrackedEntityAttributes(trackedEntityType, programsByTetUid) + .filter(tea -> !tea.isConfidentialBool()) + .toList(); - params.addExtraParam(trackedEntityType.getUid(), ALL_TET_ATTRIBUTES, trackedEntityAttributes); + params.addExtraParam( + trackedEntityType.getUid(), ALL_NON_CONFIDENTIAL_TET_ATTRIBUTES, trackedEntityAttributes); columns.addAll( trackedEntityAttributes.stream() @@ -351,7 +355,7 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti Map.of())); ((List) - params.getExtraParam(trackedEntityType.getUid(), ALL_TET_ATTRIBUTES)) + params.getExtraParam(trackedEntityType.getUid(), ALL_NON_CONFIDENTIAL_TET_ATTRIBUTES)) .forEach( tea -> sql.append( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryServiceTest.java deleted file mode 100644 index 0a54a25dc6aa..000000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryServiceTest.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (c) 2004-2022, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.analytics.event.data; - -import static org.hisp.dhis.analytics.event.data.DefaultEventDataQueryService.validateQueryParamsForConfidentialAndSkipAnalytics; -import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; -import static org.hisp.dhis.common.DimensionItemType.PROGRAM_ATTRIBUTE; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.util.List; -import java.util.Set; -import java.util.function.Consumer; -import org.hisp.dhis.analytics.event.EventQueryParams; -import org.hisp.dhis.common.DimensionItemType; -import org.hisp.dhis.common.DimensionalItemObject; -import org.hisp.dhis.common.IllegalQueryException; -import org.hisp.dhis.common.QueryItem; -import org.hisp.dhis.dataelement.DataElement; -import org.hisp.dhis.program.ProgramStage; -import org.hisp.dhis.program.ProgramStageDataElement; -import org.hisp.dhis.trackedentity.TrackedEntityAttribute; -import org.junit.jupiter.api.Test; - -class DefaultEventDataQueryServiceTest { - - private EventQueryParams mockEventQueryParams( - boolean includeConfidentialOrSkipAnalyticsItems, List queryItems) { - EventQueryParams mock = mock(EventQueryParams.class); - when(mock.includeConfidentialOrSkipAnalyticsItems()) - .thenReturn(includeConfidentialOrSkipAnalyticsItems); - when(mock.getItems()).thenReturn(queryItems); - when(mock.getItemFilters()).thenReturn(List.of()); - return mock; - } - - private QueryItem mockDimensionalItemObject( - Class clazz, DimensionItemType type, List> behavious) { - T mock = mock(clazz); - when(mock.isOfType(type)).thenReturn(true); - QueryItem queryItem = mock(QueryItem.class); - when(queryItem.getItem()).thenReturn(mock); - behavious.forEach(c -> c.accept(mock)); - return queryItem; - } - - @Test - void testAggregateDontThrowExceptionForConfidential() { - QueryItem queryItem = - mockDimensionalItemObject( - TrackedEntityAttribute.class, - PROGRAM_ATTRIBUTE, - List.of(t -> when(t.isConfidentialBool()).thenReturn(true))); - EventQueryParams eventQueryParams = mockEventQueryParams(true, List.of(queryItem)); - assertDoesNotThrow(() -> validateQueryParamsForConfidentialAndSkipAnalytics(eventQueryParams)); - } - - @Test - void testAggregateDontThrowExceptionForSkipAnalytics() { - final String dataElementUid = "dataElementUid"; - - ProgramStage programStage = mock(ProgramStage.class); - ProgramStageDataElement programStageDataElement = mock(ProgramStageDataElement.class); - - when(programStage.getProgramStageDataElements()).thenReturn(Set.of(programStageDataElement)); - when(programStageDataElement.getSkipAnalytics()).thenReturn(true); - - DataElement dataElement = mock(DataElement.class); - when(dataElement.getUid()).thenReturn(dataElementUid); - when(programStageDataElement.getDataElement()).thenReturn(dataElement); - - QueryItem queryItem = - mockDimensionalItemObject( - DataElement.class, - DATA_ELEMENT, - List.of(t -> when(t.getUid()).thenReturn(dataElementUid))); - when(queryItem.getProgramStage()).thenReturn(programStage); - - EventQueryParams eventQueryParams = mockEventQueryParams(true, List.of(queryItem)); - - assertDoesNotThrow(() -> validateQueryParamsForConfidentialAndSkipAnalytics(eventQueryParams)); - } - - @Test - void testQueryThrowsForConfidential() { - QueryItem queryItem = - mockDimensionalItemObject( - TrackedEntityAttribute.class, - PROGRAM_ATTRIBUTE, - List.of(t -> when(t.isConfidentialBool()).thenReturn(true))); - EventQueryParams eventQueryParams = mockEventQueryParams(false, List.of(queryItem)); - assertThrows( - IllegalQueryException.class, - () -> validateQueryParamsForConfidentialAndSkipAnalytics(eventQueryParams)); - } - - @Test - void testQueryThrowsForSkipAnalytics() { - final String dataElementUid = "dataElementUid"; - - ProgramStage programStage = mock(ProgramStage.class); - ProgramStageDataElement programStageDataElement = mock(ProgramStageDataElement.class); - - when(programStage.getProgramStageDataElements()).thenReturn(Set.of(programStageDataElement)); - when(programStageDataElement.getSkipAnalytics()).thenReturn(true); - - DataElement dataElement = mock(DataElement.class); - when(dataElement.getUid()).thenReturn(dataElementUid); - when(programStageDataElement.getDataElement()).thenReturn(dataElement); - - QueryItem queryItem = - mockDimensionalItemObject( - DataElement.class, - DATA_ELEMENT, - List.of(t -> when(t.getUid()).thenReturn(dataElementUid))); - when(queryItem.getProgramStage()).thenReturn(programStage); - - EventQueryParams eventQueryParams = mockEventQueryParams(false, List.of(queryItem)); - - assertThrows( - IllegalQueryException.class, - () -> validateQueryParamsForConfidentialAndSkipAnalytics(eventQueryParams)); - } -} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java index b80504d26b7e..0d537f2e63da 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManagerTest.java @@ -28,6 +28,7 @@ package org.hisp.dhis.analytics.table; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; @@ -143,13 +144,13 @@ void verifyNonConfidentialTeasAreSkipped() { AnalyticsTable analyticsTable = analyticsTables.get(0); assertContainsNonConfidentialTeaColumns(analyticsTable); - assertContainsConfidentialTeaColumns(analyticsTable); + assertDoesntContainConfidentialTeaColumns(analyticsTable); } - private void assertContainsConfidentialTeaColumns(AnalyticsTable analyticsTable) { + private void assertDoesntContainConfidentialTeaColumns(AnalyticsTable analyticsTable) { List columns = analyticsTable.getColumns(); - assertTrue(columns.stream().map(Column::getName).anyMatch("confidentialTeaUid"::equals)); + assertFalse(columns.stream().map(Column::getName).anyMatch("confidentialTeaUid"::equals)); } private void assertContainsNonConfidentialTeaColumns(AnalyticsTable analyticsTable) { diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/AnalyticsDimensionsTest.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/AnalyticsDimensionsTest.java index 821adf4c0230..08811f890abf 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/AnalyticsDimensionsTest.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/analytics/AnalyticsDimensionsTest.java @@ -183,7 +183,7 @@ public void shouldOnlyReturnConfidentialAttributeInAggregateDimensions() { .query() .getDimensionsByDimensionType(trackerProgram.getUid(), "PROGRAM_ATTRIBUTE") .validate() - .body("dimensions.uid", CoreMatchers.hasItem(confidentialAttribute)); + .body("dimensions.uid", not(CoreMatchers.hasItem(confidentialAttribute))); analyticsEnrollmentsActions .aggregate() From 16429cc2f05e408bdf3c417bd1c62109331a5c61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Tue, 10 Dec 2024 18:09:17 +0100 Subject: [PATCH 26/48] fix: Update code --- .../JdbcEnrollmentAnalyticsTableManager.java | 2 +- .../org/hisp/dhis/system/grid/ListGrid.java | 38 +++++++++---------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index 172745e793d3..a57db9c8e68c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -150,7 +150,7 @@ left join analytics_rs_dateperiodstructure dps on cast(en.enrollmentdate as date left join analytics_rs_orgunitstructure ous on en.organisationunitid=ous.organisationunitid \ left join analytics_rs_organisationunitgroupsetstructure ougs on en.organisationunitid=ougs.organisationunitid \ ${attributeJoinClause}\ - where pr.programid=${programId} \ + where pr.programid = ${programId} \ and en.organisationunitid is not null \ and (ougs.startdate is null or dps.monthstartdate=ougs.startdate) \ and en.lastupdated <= '${startTime}' \ diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/grid/ListGrid.java b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/grid/ListGrid.java index d7266b8c5368..7ca237a44a47 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/grid/ListGrid.java +++ b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/grid/ListGrid.java @@ -503,11 +503,9 @@ public Grid addColumn(List columnValues) { if (grid.size() != columnValues.size()) { throw new IllegalStateException( - "Number of column values (" - + columnValues.size() - + ") is not equal to number of rows (" - + grid.size() - + ")"); + String.format( + "Number of column values (%d) is not equal to number of rows (%d)", + columnValues.size(), grid.size())); } for (int i = 0; i < grid.size(); i++) { @@ -526,11 +524,9 @@ public Grid addColumn(int columnIndex, List columnValues) { if (grid.size() != columnValues.size()) { throw new IllegalStateException( - "Number of column values (" - + columnValues.size() - + ") is not equal to number of rows (" - + grid.size() - + ")"); + String.format( + "Number of column values (%d) is not equal to number of rows (%d)", + columnValues.size(), grid.size())); } for (int i = 0; i < grid.size(); i++) { @@ -657,7 +653,7 @@ public Grid limitGrid(int limit) { public Grid limitGrid(int startPos, int endPos) { if (startPos < 0 || endPos < startPos || endPos > getHeight()) { throw new IllegalStateException( - "Illegal start / end pos: " + startPos + ", " + endPos + ", " + getHeight()); + "Illegal start or end pos: " + startPos + ", " + endPos + ", " + getHeight()); } grid = grid.subList(startPos, endPos); @@ -813,8 +809,9 @@ public Grid substituteMetaData(Map metaDataMap) { header.setName(String.valueOf(headerMetaName)); } + // Column cells + if (header.isMeta()) { - // Column cells substituteMetaData(colIndex, colIndex, metaDataMap); } @@ -1101,7 +1098,8 @@ public void repositionColumns(List columnIndexes) { row.addAll(orderedValues); } - // reposition columns in the row context structure + // Reposition columns in the row context structure + Map> orderedRowContext = new HashMap<>(); for (Map.Entry> rowContextEntry : rowContext.entrySet()) { @@ -1113,7 +1111,8 @@ public void repositionColumns(List columnIndexes) { .forEach( key -> { if (numberRegex.matcher(key).matches()) { - // reindexing of columns + // Reindexing of columns + orderedRowContextItems.put( columnIndexes.get(Integer.parseInt(key)).toString(), ctxItem.get(key)); } @@ -1151,12 +1150,9 @@ private void verifyGridState() { for (List row : grid) { if (rowLength != null && rowLength != row.size()) { throw new IllegalStateException( - "Grid rows do not have the same number of cells, previous: " - + rowLength - + ", this: " - + row.size() - + ", at row: " - + rowPos); + String.format( + "Grid rows do not have the same number of cells, previous: %d, this: %d, at row: %d", + rowLength, row.size(), rowPos)); } rowPos++; @@ -1177,7 +1173,7 @@ private void updateColumnIndexMap() { } // ------------------------------------------------------------------------- - // toString + // ToString // ------------------------------------------------------------------------- @Override From 35a252607c29ee911053963cb456495061e91ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 14:34:59 +0100 Subject: [PATCH 27/48] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 4 ---- .../table/JdbcEventAnalyticsTableManager.java | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 8a98bd7bb0b0..31bd476ab638 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -168,13 +168,9 @@ protected void populateTableInternal(AnalyticsTablePartition partition, String f List columns = partition.getMasterTable().getAnalyticsTableColumns(); String sql = "insert into " + tableName + " ("; - sql += toCommaSeparated(columns, col -> quote(col.getName())); - sql += ") select "; - sql += toCommaSeparated(columns, AnalyticsTableColumn::getSelectExpression); - sql += " " + fromClause; invokeTimeAndLog(sql, "Populating table: '{}'", tableName); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index bde55a83d977..2d22956aaf24 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -492,8 +492,7 @@ private List getColumnForDataElement( sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); String columnExpression = getColumnExpression(dataElement.getValueType(), jsonExpression); String dataFilterClause = getDataFilterClause(dataElement); - String selectExpression = - String.format("%s as %s", columnExpression, quote(dataElement.getUid())); + String selectExpression = getSelectExpression(dataElement, columnExpression); Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); if (withLegendSet) { @@ -516,6 +515,17 @@ private List getColumnForDataElement( return columns; } + /** + * Returns a select expression. + * + * @param dataElement the {@link DataElement}. + * @param columnExpression the column expression. + * @return a select expression. + */ + private String getSelectExpression(DataElement dataElement, String columnExpression) { + return String.format("%s as %s", columnExpression, quote(dataElement.getUid())); + } + /** * Returns a list of columns. * From 82deae7896e3c9b3663f532dba33073aa3294f19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 17:34:24 +0100 Subject: [PATCH 28/48] fix: Update code --- ...dbcTrackedEntityAnalyticsTableManager.java | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index d302e6ee4a0c..7c226c6ccf2a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -30,6 +30,7 @@ import static java.lang.String.join; import static java.util.stream.Collectors.groupingBy; import static org.apache.commons.collections4.CollectionUtils.emptyIfNull; +import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; import static org.hisp.dhis.analytics.AnalyticsTableType.TRACKED_ENTITY_INSTANCE; import static org.hisp.dhis.analytics.table.JdbcEventAnalyticsTableManager.EXPORTABLE_EVENT_STATUSES; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; @@ -176,25 +177,27 @@ private List getColumns( (Map>) params.getExtraParam("", PROGRAMS_BY_TET_KEY); List columns = new ArrayList<>(getFixedColumns()); - - String enrolledInProgramExpression = - """ - \s exists(select 1 from ${enrollment} en_0 \ - where en_0.trackedentityid = te.trackedentityid \ - and en_0.programid = ${programId})"""; - - emptyIfNull(programsByTetUid.get(trackedEntityType.getUid())) - .forEach( - program -> - columns.add( - AnalyticsTableColumn.builder() - .name(program.getUid()) - .dataType(BOOLEAN) - .selectExpression( - replaceQualify( - enrolledInProgramExpression, - Map.of("programId", String.valueOf(program.getId())))) - .build())); + List programs = programsByTetUid.get(trackedEntityType.getUid()); + + if (isNotEmpty(programs)) { + String enrolledInProgramExpression = + """ + \s exists(select 1 from ${enrollment} en_0 \ + where en_0.trackedentityid = te.trackedentityid \ + and en_0.programid = ${programId})"""; + + programs.forEach( + program -> + columns.add( + AnalyticsTableColumn.builder() + .name(program.getUid()) + .dataType(BOOLEAN) + .selectExpression( + replaceQualify( + enrolledInProgramExpression, + Map.of("programId", String.valueOf(program.getId())))) + .build())); + } List trackedEntityAttributes = getAllTrackedEntityAttributes(trackedEntityType, programsByTetUid) From 284e86a4a415d1f46416c7bcfb9e270a571ea16f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 17:35:34 +0100 Subject: [PATCH 29/48] fix: Update code --- ...dbcTrackedEntityAnalyticsTableManager.java | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index 7c226c6ccf2a..f3e7d05ad2f6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -311,18 +311,20 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti left join analytics_rs_organisationunitgroupsetstructure ougs on te.organisationunitid=ougs.organisationunitid""", Map.of())); - ((List) - params.getExtraParam(trackedEntityType.getUid(), ALL_NON_CONFIDENTIAL_TET_ATTRIBUTES)) - .forEach( - tea -> - sql.append( - replaceQualify( - """ + List attributes = + ((List) + params.getExtraParam(trackedEntityType.getUid(), ALL_NON_CONFIDENTIAL_TET_ATTRIBUTES)); + + attributes.forEach( + tea -> + sql.append( + replaceQualify( + """ \s left join ${trackedentityattributevalue} ${teaUid} on ${teaUid}.trackedentityid=te.trackedentityid \ and ${teaUid}.trackedentityattributeid = ${teaId}""", - Map.of( - "teaUid", quote(tea.getUid()), - "teaId", String.valueOf(tea.getId()))))); + Map.of( + "teaUid", quote(tea.getUid()), + "teaId", String.valueOf(tea.getId()))))); sql.append( replaceQualify( """ From 3608b816728684030dc5a0c6be896e85897a1dba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 17:35:50 +0100 Subject: [PATCH 30/48] fix: Update code --- .../table/JdbcTrackedEntityAnalyticsTableManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index f3e7d05ad2f6..9d34c3474a36 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -320,8 +320,8 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti sql.append( replaceQualify( """ - \s left join ${trackedentityattributevalue} ${teaUid} on ${teaUid}.trackedentityid=te.trackedentityid \ - and ${teaUid}.trackedentityattributeid = ${teaId}""", + \s left join ${trackedentityattributevalue} ${teaUid} on ${teaUid}.trackedentityid=te.trackedentityid \ + and ${teaUid}.trackedentityattributeid = ${teaId}""", Map.of( "teaUid", quote(tea.getUid()), "teaId", String.valueOf(tea.getId()))))); From e47ed4d978b85c3852691d24b10061084bed8f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 17:36:22 +0100 Subject: [PATCH 31/48] fix: Update code --- .../analytics/table/JdbcTrackedEntityAnalyticsTableManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index 9d34c3474a36..da71dd859ff7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -274,7 +274,6 @@ private List getFixedColumns() { columns.addAll(getOrganisationUnitLevelColumns()); columns.add(getOrganisationUnitNameHierarchyColumn()); columns.addAll(getFixedNonGroupByColumns()); - return columns; } From 0f7e8631eac9734f96d5b13dd438f7d819e5c7aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Wed, 11 Dec 2024 17:38:57 +0100 Subject: [PATCH 32/48] fix: Update code --- ...dbcTrackedEntityAnalyticsTableManager.java | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index da71dd859ff7..8d6fc1f302d6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -314,16 +314,19 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti ((List) params.getExtraParam(trackedEntityType.getUid(), ALL_NON_CONFIDENTIAL_TET_ATTRIBUTES)); - attributes.forEach( - tea -> - sql.append( - replaceQualify( - """ - \s left join ${trackedentityattributevalue} ${teaUid} on ${teaUid}.trackedentityid=te.trackedentityid \ - and ${teaUid}.trackedentityattributeid = ${teaId}""", - Map.of( - "teaUid", quote(tea.getUid()), - "teaId", String.valueOf(tea.getId()))))); + if (isNotEmpty(attributes)) { + attributes.forEach( + tea -> + sql.append( + replaceQualify( + """ + \s left join ${trackedentityattributevalue} ${teaUid} on ${teaUid}.trackedentityid=te.trackedentityid \ + and ${teaUid}.trackedentityattributeid = ${teaId}""", + Map.of( + "teaUid", quote(tea.getUid()), + "teaId", String.valueOf(tea.getId()))))); + } + sql.append( replaceQualify( """ From 6ec2dcfdaa3ee7a535782a539622fd3104e514a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Fri, 13 Dec 2024 18:14:45 +0100 Subject: [PATCH 33/48] fix: Update code --- .../dhis/resourcetable/ResourceTableType.java | 3 +- ...ckedEntityAttributeValueResourceTable.java | 73 +++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/TrackedEntityAttributeValueResourceTable.java diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/resourcetable/ResourceTableType.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/resourcetable/ResourceTableType.java index 1adb7e8fa0ef..bbf786ba583d 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/resourcetable/ResourceTableType.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/resourcetable/ResourceTableType.java @@ -49,5 +49,6 @@ public enum ResourceTableType { DATE_PERIOD_STRUCTURE, DATA_ELEMENT_CATEGORY_OPTION_COMBO, DATA_APPROVAL_REMAP_LEVEL, - DATA_APPROVAL_MIN_LEVEL; + DATA_APPROVAL_MIN_LEVEL, + TRACKED_ENTITTY_ATTRIBUTE_VALUE; } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/TrackedEntityAttributeValueResourceTable.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/TrackedEntityAttributeValueResourceTable.java new file mode 100644 index 000000000000..641b49adf070 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/TrackedEntityAttributeValueResourceTable.java @@ -0,0 +1,73 @@ +package org.hisp.dhis.resourcetable.table; + +import static org.hisp.dhis.db.model.Table.toStaging; +import static org.hisp.dhis.system.util.SqlUtils.appendRandom; +import java.util.List; +import java.util.Optional; +import org.hisp.dhis.db.model.Column; +import org.hisp.dhis.db.model.DataType; +import org.hisp.dhis.db.model.Index; +import org.hisp.dhis.db.model.Logged; +import org.hisp.dhis.db.model.Table; +import org.hisp.dhis.db.model.constraint.Nullable; +import org.hisp.dhis.db.model.constraint.Unique; +import org.hisp.dhis.resourcetable.ResourceTable; +import org.hisp.dhis.resourcetable.ResourceTableType; +import lombok.RequiredArgsConstructor; + +/** + * @author Lars Helge Overland + */ +@RequiredArgsConstructor +public class TrackedEntityAttributeValueResourceTable implements ResourceTable { + public static final String TABLE_NAME = "trackedentityattributevalue"; + + private final Logged logged; + + @Override + public Table getTable() { + return new Table(toStaging(TABLE_NAME), getColumns(), getPrimaryKey(), logged); + } + + @Override + public Table getMainTable() { + return new Table(TABLE_NAME, getColumns(), getPrimaryKey(), logged); + } + + private List getColumns() { + return List.of( + new Column("dataelementid", DataType.BIGINT, Nullable.NOT_NULL), + new Column("dataelementuid", DataType.CHARACTER_11, Nullable.NOT_NULL), + new Column("dataelementname", DataType.VARCHAR_255, Nullable.NOT_NULL)); + } + + private List getPrimaryKey() { + return List.of(""); + } + + @Override + public List getIndexes() { + return List.of( + Index.builder() + .name(appendRandom("in_dataelementstructure_dataelementuid")) + .tableName(toStaging(TABLE_NAME)) + .unique(Unique.UNIQUE) + .columns(List.of("dataelementuid")) + .build()); + } + + @Override + public ResourceTableType getTableType() { + return ResourceTableType.TRACKED_ENTITTY_ATTRIBUTE_VALUE; + } + + @Override + public Optional getPopulateTempTableStatement() { + return Optional.empty(); + } + + @Override + public Optional> getPopulateTempTableContent() { + return Optional.empty(); + } +} From 2c6e9c4e8b8e909374ddcb7b07d965b0a4d479e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Fri, 13 Dec 2024 22:05:27 +0100 Subject: [PATCH 34/48] fix: Update code --- ...ckedEntityAttributeValueResourceTable.java | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/TrackedEntityAttributeValueResourceTable.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/TrackedEntityAttributeValueResourceTable.java index 641b49adf070..06257c5561d1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/TrackedEntityAttributeValueResourceTable.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/TrackedEntityAttributeValueResourceTable.java @@ -1,9 +1,38 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ package org.hisp.dhis.resourcetable.table; import static org.hisp.dhis.db.model.Table.toStaging; import static org.hisp.dhis.system.util.SqlUtils.appendRandom; + import java.util.List; import java.util.Optional; +import lombok.RequiredArgsConstructor; import org.hisp.dhis.db.model.Column; import org.hisp.dhis.db.model.DataType; import org.hisp.dhis.db.model.Index; @@ -13,7 +42,6 @@ import org.hisp.dhis.db.model.constraint.Unique; import org.hisp.dhis.resourcetable.ResourceTable; import org.hisp.dhis.resourcetable.ResourceTableType; -import lombok.RequiredArgsConstructor; /** * @author Lars Helge Overland @@ -36,9 +64,9 @@ public Table getMainTable() { private List getColumns() { return List.of( - new Column("dataelementid", DataType.BIGINT, Nullable.NOT_NULL), - new Column("dataelementuid", DataType.CHARACTER_11, Nullable.NOT_NULL), - new Column("dataelementname", DataType.VARCHAR_255, Nullable.NOT_NULL)); + new Column("trackedentityid", DataType.BIGINT, Nullable.NOT_NULL), + new Column("trackedentityattributeid", DataType.BIGINT, Nullable.NOT_NULL), + new Column("value", DataType.TEXT, Nullable.NULL)); } private List getPrimaryKey() { From 0cf2a4e3a90d0540fa34033cddee19e2b14c7cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Fri, 13 Dec 2024 22:07:02 +0100 Subject: [PATCH 35/48] fix: Update code --- .../TrackedEntityAttributeValueResourceTable.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/TrackedEntityAttributeValueResourceTable.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/TrackedEntityAttributeValueResourceTable.java index 06257c5561d1..84f16f10df88 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/TrackedEntityAttributeValueResourceTable.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/TrackedEntityAttributeValueResourceTable.java @@ -28,7 +28,6 @@ package org.hisp.dhis.resourcetable.table; import static org.hisp.dhis.db.model.Table.toStaging; -import static org.hisp.dhis.system.util.SqlUtils.appendRandom; import java.util.List; import java.util.Optional; @@ -39,7 +38,6 @@ import org.hisp.dhis.db.model.Logged; import org.hisp.dhis.db.model.Table; import org.hisp.dhis.db.model.constraint.Nullable; -import org.hisp.dhis.db.model.constraint.Unique; import org.hisp.dhis.resourcetable.ResourceTable; import org.hisp.dhis.resourcetable.ResourceTableType; @@ -70,18 +68,12 @@ private List getColumns() { } private List getPrimaryKey() { - return List.of(""); + return List.of("trackedentityid", "trackedentityattributeid"); } @Override public List getIndexes() { - return List.of( - Index.builder() - .name(appendRandom("in_dataelementstructure_dataelementuid")) - .tableName(toStaging(TABLE_NAME)) - .unique(Unique.UNIQUE) - .columns(List.of("dataelementuid")) - .build()); + return List.of(); } @Override From 00cd21456176729923574d4f50e43cbb1b218906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Fri, 13 Dec 2024 22:09:17 +0100 Subject: [PATCH 36/48] fix: Update code --- .../src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java index 27da0e83f774..15f167775fc4 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java @@ -456,11 +456,12 @@ public interface SqlBuilder { */ String dropCatalogIfExists(); + /** Enumeration of time units. */ enum DateUnit { DAYS, + WEEKS, MONTHS, MINUTES, - YEARS, - WEEKS + YEARS } } From dd44d6838505387a813aafb381013e14b07e8229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 14 Dec 2024 11:29:33 +0100 Subject: [PATCH 37/48] fix: Update code --- .../DefaultResourceTableService.java | 5 +- .../resourcetable/ResourceTableStore.java | 7 -- .../jdbc/JdbcResourceTableStore.java | 55 ----------- .../JdbcTableReplicationStore.java | 99 +++++++++++++++++++ .../TableReplicationStore.java | 42 ++++++++ 5 files changed, 145 insertions(+), 63 deletions(-) create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/JdbcTableReplicationStore.java create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/TableReplicationStore.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java index 8f01d9ac7fad..04ba319c9e30 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java @@ -73,6 +73,7 @@ import org.hisp.dhis.setting.SystemSettings; import org.hisp.dhis.sqlview.SqlView; import org.hisp.dhis.sqlview.SqlViewService; +import org.hisp.dhis.tablereplication.TableReplicationStore; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -85,6 +86,8 @@ public class DefaultResourceTableService implements ResourceTableService { private final ResourceTableStore resourceTableStore; + private final TableReplicationStore tableReplicationStore; + private final IdentifiableObjectManager idObjectManager; private final OrganisationUnitService organisationUnitService; @@ -113,7 +116,7 @@ public void generateResourceTables() { @Transactional public void replicateAnalyticsResourceTables() { for (ResourceTable table : getResourceTables()) { - resourceTableStore.replicateAnalyticsResourceTable(table); + tableReplicationStore.replicateAnalyticsDatabaseTable(table.getMainTable()); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/ResourceTableStore.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/ResourceTableStore.java index a8c3d0b28b72..f7e2cf8ee3a0 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/ResourceTableStore.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/ResourceTableStore.java @@ -37,11 +37,4 @@ public interface ResourceTableStore { * @param resourceTable the {@link ResourceTable}. */ void generateResourceTable(ResourceTable resourceTable); - - /** - * Replicates analytics resource tables. - * - * @param resourceTable the {@link ResourceTable}. - */ - void replicateAnalyticsResourceTable(ResourceTable resourceTable); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/jdbc/JdbcResourceTableStore.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/jdbc/JdbcResourceTableStore.java index cf63966df74d..251175b4af66 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/jdbc/JdbcResourceTableStore.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/jdbc/JdbcResourceTableStore.java @@ -46,7 +46,6 @@ import org.hisp.dhis.resourcetable.ResourceTableStore; import org.hisp.dhis.resourcetable.ResourceTableType; import org.hisp.dhis.system.util.Clock; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; @@ -62,13 +61,8 @@ public class JdbcResourceTableStore implements ResourceTableStore { private final JdbcTemplate jdbcTemplate; - @Qualifier("analyticsJdbcTemplate") - private final JdbcTemplate analyticsJdbcTemplate; - private final SqlBuilder sqlBuilder = new PostgreSqlBuilder(); - private final SqlBuilder analyticsDatabaseSqlBuilder; - @Override public void generateResourceTable(ResourceTable resourceTable) { final Clock clock = new Clock().startClock(); @@ -98,55 +92,6 @@ public void generateResourceTable(ResourceTable resourceTable) { log.info("Resource table update done: '{}' '{}'", tableName, clock.time()); } - @Override - public void replicateAnalyticsResourceTable(ResourceTable resourceTable) { - final Clock clock = new Clock().startClock(); - final Table table = resourceTable.getMainTable(); - final String tableName = table.getName(); - - dropAnalyticsDatabaseTable(table); - - createAnalyticsDatabaseTable(table); - - replicateAnalyticsDatabaseTable(table); - - log.info("Analytics resource table replication done: '{}' '{}'", tableName, clock.time()); - } - - /** - * Drops the given analytics database table. - * - * @param table the {@link Table}. - */ - private void dropAnalyticsDatabaseTable(Table table) { - String sql = analyticsDatabaseSqlBuilder.dropTableIfExists(table); - log.info("Drop table SQL: '{}'", sql); - analyticsJdbcTemplate.execute(sql); - } - - /** - * Creates the given analytics database table. - * - * @param table the {@link Table}. - */ - private void createAnalyticsDatabaseTable(Table table) { - String sql = analyticsDatabaseSqlBuilder.createTable(table); - log.info("Create table SQL: '{}'", sql); - analyticsJdbcTemplate.execute(sql); - } - - /** - * Replicates the given table in the analytics database. - * - * @param table the {@link Table}. - */ - private void replicateAnalyticsDatabaseTable(Table table) { - String fromTable = analyticsDatabaseSqlBuilder.qualifyTable(table.getName()); - String sql = analyticsDatabaseSqlBuilder.insertIntoSelectFrom(table, fromTable); - log.info("Replicate table SQL: '{}'", sql); - analyticsJdbcTemplate.execute(sql); - } - /** * Drops the given table. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/JdbcTableReplicationStore.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/JdbcTableReplicationStore.java new file mode 100644 index 000000000000..b9f709538f08 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/JdbcTableReplicationStore.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.tablereplication; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.hisp.dhis.db.model.Table; +import org.hisp.dhis.db.sql.SqlBuilder; +import org.hisp.dhis.system.util.Clock; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Service; + +/** + * @author Lars Helge Overland + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class JdbcTableReplicationStore implements TableReplicationStore { + + @Qualifier("analyticsJdbcTemplate") + private final JdbcTemplate jdbcTemplate; + + private final SqlBuilder sqlBuilder; + + @Override + public void replicateAnalyticsDatabaseTable(Table table) { + final Clock clock = new Clock().startClock(); + final String tableName = table.getName(); + + dropTable(table); + + createTable(table); + + replicateTable(table); + + log.info("Analytics database table replicated: '{}' '{}'", tableName, clock.time()); + } + + /** + * Drops the given analytics database table. + * + * @param table the {@link Table}. + */ + private void dropTable(Table table) { + String sql = sqlBuilder.dropTableIfExists(table); + log.info("Drop table SQL: '{}'", sql); + jdbcTemplate.execute(sql); + } + + /** + * Creates the given analytics database table. + * + * @param table the {@link Table}. + */ + private void createTable(Table table) { + String sql = sqlBuilder.createTable(table); + log.info("Create table SQL: '{}'", sql); + jdbcTemplate.execute(sql); + } + + /** + * Replicates the given table in the analytics database. + * + * @param table the {@link Table}. + */ + private void replicateTable(Table table) { + String fromTable = sqlBuilder.qualifyTable(table.getName()); + String sql = sqlBuilder.insertIntoSelectFrom(table, fromTable); + log.info("Replicate table SQL: '{}'", sql); + jdbcTemplate.execute(sql); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/TableReplicationStore.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/TableReplicationStore.java new file mode 100644 index 000000000000..2f0e7681b412 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/TableReplicationStore.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.tablereplication; + +import org.hisp.dhis.db.model.Table; + +/** + * @author Lars Helge Overland + */ +public interface TableReplicationStore { + /** + * Replicates the given transactional database table in the analytics database. + * + * @param table the {@link Table} to replicate. + */ + void replicateAnalyticsDatabaseTable(Table table); +} From 68ef8e0a87c3575550b626a362df09ae31ad0d85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 14 Dec 2024 11:30:53 +0100 Subject: [PATCH 38/48] fix: Update code --- .../dhis/resourcetable/ResourceTableType.java | 3 +- ...ckedEntityAttributeValueResourceTable.java | 93 ------------------- 2 files changed, 1 insertion(+), 95 deletions(-) delete mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/TrackedEntityAttributeValueResourceTable.java diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/resourcetable/ResourceTableType.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/resourcetable/ResourceTableType.java index bbf786ba583d..1adb7e8fa0ef 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/resourcetable/ResourceTableType.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/resourcetable/ResourceTableType.java @@ -49,6 +49,5 @@ public enum ResourceTableType { DATE_PERIOD_STRUCTURE, DATA_ELEMENT_CATEGORY_OPTION_COMBO, DATA_APPROVAL_REMAP_LEVEL, - DATA_APPROVAL_MIN_LEVEL, - TRACKED_ENTITTY_ATTRIBUTE_VALUE; + DATA_APPROVAL_MIN_LEVEL; } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/TrackedEntityAttributeValueResourceTable.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/TrackedEntityAttributeValueResourceTable.java deleted file mode 100644 index 84f16f10df88..000000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/TrackedEntityAttributeValueResourceTable.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2004-2024, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.resourcetable.table; - -import static org.hisp.dhis.db.model.Table.toStaging; - -import java.util.List; -import java.util.Optional; -import lombok.RequiredArgsConstructor; -import org.hisp.dhis.db.model.Column; -import org.hisp.dhis.db.model.DataType; -import org.hisp.dhis.db.model.Index; -import org.hisp.dhis.db.model.Logged; -import org.hisp.dhis.db.model.Table; -import org.hisp.dhis.db.model.constraint.Nullable; -import org.hisp.dhis.resourcetable.ResourceTable; -import org.hisp.dhis.resourcetable.ResourceTableType; - -/** - * @author Lars Helge Overland - */ -@RequiredArgsConstructor -public class TrackedEntityAttributeValueResourceTable implements ResourceTable { - public static final String TABLE_NAME = "trackedentityattributevalue"; - - private final Logged logged; - - @Override - public Table getTable() { - return new Table(toStaging(TABLE_NAME), getColumns(), getPrimaryKey(), logged); - } - - @Override - public Table getMainTable() { - return new Table(TABLE_NAME, getColumns(), getPrimaryKey(), logged); - } - - private List getColumns() { - return List.of( - new Column("trackedentityid", DataType.BIGINT, Nullable.NOT_NULL), - new Column("trackedentityattributeid", DataType.BIGINT, Nullable.NOT_NULL), - new Column("value", DataType.TEXT, Nullable.NULL)); - } - - private List getPrimaryKey() { - return List.of("trackedentityid", "trackedentityattributeid"); - } - - @Override - public List getIndexes() { - return List.of(); - } - - @Override - public ResourceTableType getTableType() { - return ResourceTableType.TRACKED_ENTITTY_ATTRIBUTE_VALUE; - } - - @Override - public Optional getPopulateTempTableStatement() { - return Optional.empty(); - } - - @Override - public Optional> getPopulateTempTableContent() { - return Optional.empty(); - } -} From 0d1f09e351ecd78c18de3bb4814a938f5321b368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 14 Dec 2024 11:40:06 +0100 Subject: [PATCH 39/48] fix: Update code --- .../table/DefaultAnalyticsTableGenerator.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java index d3eb563d7af4..e3153ef4545d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java @@ -73,21 +73,22 @@ public class DefaultAnalyticsTableGenerator implements AnalyticsTableGenerator { @Override public void generateAnalyticsTables(AnalyticsTableUpdateParams params0, JobProgress progress) { - Clock clock = new Clock(log).startClock(); - Date lastSuccessfulUpdate = + final Clock clock = new Clock(log).startClock(); + final Date lastSuccessfulUpdate = settingsService.getCurrentSettings().getLastSuccessfulAnalyticsTablesUpdate(); - - Set availableTypes = + final Set availableTypes = analyticsTableServices.stream() .map(AnalyticsTableService::getAnalyticsTableType) .collect(Collectors.toSet()); - - AnalyticsTableUpdateParams params = + final AnalyticsTableUpdateParams params = params0.toBuilder().lastSuccessfulUpdate(lastSuccessfulUpdate).build(); + Set skipTypes = emptyIfNull(params.getSkipTableTypes()); + log.info("Found {} analytics table types: {}", availableTypes.size(), availableTypes); - log.info("Analytics table update: {}", params); + log.info("Analytics table update params: {}", params); log.info("Last successful analytics table update: {}", toLongDate(lastSuccessfulUpdate)); + log.debug("Skipping table types: {}", skipTypes); progress.startingProcess( "Analytics table update process{}", (params.isLatestUpdate() ? " (latest partition)" : "")); @@ -101,8 +102,6 @@ public void generateAnalyticsTables(AnalyticsTableUpdateParams params0, JobProgr } } - Set skipTypes = emptyIfNull(params.getSkipTableTypes()); - for (AnalyticsTableService service : analyticsTableServices) { AnalyticsTableType tableType = service.getAnalyticsTableType(); From 4756e64dd13cea37aa93ae6cdd948a3350521cc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 14 Dec 2024 11:42:22 +0100 Subject: [PATCH 40/48] fix: Update code --- .../table/DefaultAnalyticsTableGenerator.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java index e3153ef4545d..b2078185f5e2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java @@ -48,6 +48,7 @@ import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.scheduling.JobProgress; +import org.hisp.dhis.setting.SystemSettings; import org.hisp.dhis.setting.SystemSettingsService; import org.hisp.dhis.system.util.Clock; import org.springframework.stereotype.Service; @@ -74,18 +75,14 @@ public class DefaultAnalyticsTableGenerator implements AnalyticsTableGenerator { @Override public void generateAnalyticsTables(AnalyticsTableUpdateParams params0, JobProgress progress) { final Clock clock = new Clock(log).startClock(); - final Date lastSuccessfulUpdate = - settingsService.getCurrentSettings().getLastSuccessfulAnalyticsTablesUpdate(); - final Set availableTypes = - analyticsTableServices.stream() - .map(AnalyticsTableService::getAnalyticsTableType) - .collect(Collectors.toSet()); + final SystemSettings systemSettings = settingsService.getCurrentSettings(); + final Date lastSuccessfulUpdate = systemSettings.getLastSuccessfulAnalyticsTablesUpdate(); final AnalyticsTableUpdateParams params = params0.toBuilder().lastSuccessfulUpdate(lastSuccessfulUpdate).build(); Set skipTypes = emptyIfNull(params.getSkipTableTypes()); - log.info("Found {} analytics table types: {}", availableTypes.size(), availableTypes); + log.info("Found analytics table types: {}", getAvailableTableTypes()); log.info("Analytics table update params: {}", params); log.info("Last successful analytics table update: {}", toLongDate(lastSuccessfulUpdate)); log.debug("Skipping table types: {}", skipTypes); @@ -161,4 +158,10 @@ private void generateResourceTablesInternal(JobProgress progress) { settingsService.put("keyLastSuccessfulResourceTablesUpdate", new Date()); } + + private Set getAvailableTableTypes() { + return analyticsTableServices.stream() + .map(AnalyticsTableService::getAnalyticsTableType) + .collect(Collectors.toSet()); + } } From 6029a061f21708a1b306dc3623f15c8a9136ae39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 14 Dec 2024 11:43:58 +0100 Subject: [PATCH 41/48] fix: Update code --- .../hisp/dhis/analytics/table/DefaultAnalyticsTableService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java index f2d9807084c7..08e396d4ceea 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java @@ -81,7 +81,7 @@ public AnalyticsTableType getAnalyticsTableType() { @Override public void create(AnalyticsTableUpdateParams params, JobProgress progress) { - int parallelJobs = getParallelJobs(); + final int parallelJobs = getParallelJobs(); int tableUpdates = 0; log.info("Analytics table update parameters: {}", params); From 79e7a9c3772fda26fccd821f93f94808d813f1ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 14 Dec 2024 11:44:39 +0100 Subject: [PATCH 42/48] fix: Update code --- .../hisp/dhis/tablereplication/JdbcTableReplicationStore.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/JdbcTableReplicationStore.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/JdbcTableReplicationStore.java index b9f709538f08..aa5b275c879e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/JdbcTableReplicationStore.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/tablereplication/JdbcTableReplicationStore.java @@ -55,9 +55,7 @@ public void replicateAnalyticsDatabaseTable(Table table) { final String tableName = table.getName(); dropTable(table); - createTable(table); - replicateTable(table); log.info("Analytics database table replicated: '{}' '{}'", tableName, clock.time()); From 8144d52edd0c8d4c3801d6a4c63c70415ecaf1ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 14 Dec 2024 11:45:29 +0100 Subject: [PATCH 43/48] fix: Update code --- .../table/DefaultAnalyticsTableGenerator.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java index b2078185f5e2..4f1c519a367a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java @@ -146,6 +146,11 @@ public void generateResourceTables(JobProgress progress) { // Supportive methods // ------------------------------------------------------------------------- + /** + * Generates resource tables. + * + * @param progress the {@link JobProgress}. + */ private void generateResourceTablesInternal(JobProgress progress) { resourceTableService.dropAllSqlViews(progress); @@ -159,6 +164,11 @@ private void generateResourceTablesInternal(JobProgress progress) { settingsService.put("keyLastSuccessfulResourceTablesUpdate", new Date()); } + /** + * Returns the available analytics table types. + * + * @return a set of {@link AnalyticsTableType}. + */ private Set getAvailableTableTypes() { return analyticsTableServices.stream() .map(AnalyticsTableService::getAnalyticsTableType) From 62ee0e21cd72c607c8567fb84b1d44139b62e9a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 14 Dec 2024 11:45:43 +0100 Subject: [PATCH 44/48] fix: Update code --- .../dhis/analytics/table/DefaultAnalyticsTableGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java index 4f1c519a367a..690eb10ffe7a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java @@ -128,7 +128,7 @@ private void updateLastSuccessfulSystemSettings(AnalyticsTableUpdateParams param @Override public void generateResourceTables(JobProgress progress) { - Clock clock = new Clock().startClock(); + final Clock clock = new Clock().startClock(); progress.startingProcess("Generating resource tables"); From 1795f7e0051d920f2135b94bd85706dc2d8e3f6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sat, 14 Dec 2024 11:47:02 +0100 Subject: [PATCH 45/48] fix: Update code --- .../dhis/analytics/table/DefaultAnalyticsTableGenerator.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java index 690eb10ffe7a..4abc5d9b0ae5 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java @@ -79,8 +79,7 @@ public void generateAnalyticsTables(AnalyticsTableUpdateParams params0, JobProgr final Date lastSuccessfulUpdate = systemSettings.getLastSuccessfulAnalyticsTablesUpdate(); final AnalyticsTableUpdateParams params = params0.toBuilder().lastSuccessfulUpdate(lastSuccessfulUpdate).build(); - - Set skipTypes = emptyIfNull(params.getSkipTableTypes()); + final Set skipTypes = emptyIfNull(params.getSkipTableTypes()); log.info("Found analytics table types: {}", getAvailableTableTypes()); log.info("Analytics table update params: {}", params); From 02b523f517133c35897354560a65e8f2dc82ba7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 15 Dec 2024 20:33:49 +0100 Subject: [PATCH 46/48] fix: Update code --- .../table/EnrollmentAnalyticsColumnName.java | 1 - .../table/EnrollmentAnalyticsColumn.java | 25 ++++++++++--------- .../analytics/table/EventAnalyticsColumn.java | 25 ++++++++++--------- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/table/EnrollmentAnalyticsColumnName.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/table/EnrollmentAnalyticsColumnName.java index df4e708abfd2..16bce0442f5c 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/table/EnrollmentAnalyticsColumnName.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/table/EnrollmentAnalyticsColumnName.java @@ -31,7 +31,6 @@ @NoArgsConstructor(access = lombok.AccessLevel.PRIVATE) public final class EnrollmentAnalyticsColumnName { - public static final String ENROLLMENT_COLUMN_NAME = "enrollment"; public static final String TRACKED_ENTITY_COLUMN_NAME = "trackedentity"; public static final String ENROLLMENT_DATE_COLUMN_NAME = "enrollmentdate"; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EnrollmentAnalyticsColumn.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EnrollmentAnalyticsColumn.java index 1cbca99e8be3..4b6ddfe7de85 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EnrollmentAnalyticsColumn.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EnrollmentAnalyticsColumn.java @@ -49,6 +49,19 @@ @NoArgsConstructor(access = lombok.AccessLevel.PRIVATE) public final class EnrollmentAnalyticsColumn { + public static final AnalyticsTableColumn TRACKED_ENTITY = + AnalyticsTableColumn.builder() + .name(EnrollmentAnalyticsColumnName.TRACKED_ENTITY_COLUMN_NAME) + .dataType(CHARACTER_11) + .selectExpression("te.uid") + .build(); + public static final AnalyticsTableColumn TRACKED_ENTITY_GEOMETRY = + AnalyticsTableColumn.builder() + .name(EnrollmentAnalyticsColumnName.TRACKED_ENTITY_GEOMETRY_COLUMN_NAME) + .dataType(GEOMETRY) + .selectExpression("te.geometry") + .build(); + private static final AnalyticsTableColumn ENROLLMENT = AnalyticsTableColumn.builder() .name(EnrollmentAnalyticsColumnName.ENROLLMENT_COLUMN_NAME) @@ -56,12 +69,6 @@ public final class EnrollmentAnalyticsColumn { .nullable(NOT_NULL) .selectExpression("en.uid") .build(); - static final AnalyticsTableColumn TRACKED_ENTITY = - AnalyticsTableColumn.builder() - .name(EnrollmentAnalyticsColumnName.TRACKED_ENTITY_COLUMN_NAME) - .dataType(CHARACTER_11) - .selectExpression("te.uid") - .build(); private static final AnalyticsTableColumn ENROLLMENT_DATE = AnalyticsTableColumn.builder() .name(EnrollmentAnalyticsColumnName.ENROLLMENT_DATE_COLUMN_NAME) @@ -152,12 +159,6 @@ public final class EnrollmentAnalyticsColumn { .nullable(NOT_NULL) .selectExpression("coalesce(registrationou.uid,ou.uid)") .build(); - static final AnalyticsTableColumn TRACKED_ENTITY_GEOMETRY = - AnalyticsTableColumn.builder() - .name(EnrollmentAnalyticsColumnName.TRACKED_ENTITY_GEOMETRY_COLUMN_NAME) - .dataType(GEOMETRY) - .selectExpression("te.geometry") - .build(); private static final List COMMON_COLUMNS = List.of( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EventAnalyticsColumn.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EventAnalyticsColumn.java index e5ad325bdf89..2b1ecfa33a79 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EventAnalyticsColumn.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EventAnalyticsColumn.java @@ -49,6 +49,19 @@ @NoArgsConstructor(access = lombok.AccessLevel.PRIVATE) public final class EventAnalyticsColumn { + public static final AnalyticsTableColumn TRACKED_ENTITY = + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.TRACKED_ENTITY_COLUMN_NAME) + .dataType(CHARACTER_11) + .selectExpression("te.uid") + .build(); + public static final AnalyticsTableColumn TRACKED_ENTITY_GEOMETRY = + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.TRACKED_ENTITY_GEOMETRY_COLUMN_NAME) + .dataType(GEOMETRY) + .selectExpression("te.geometry") + .build(); + // Common columns that work across all databases private static final AnalyticsTableColumn EVENT = @@ -65,12 +78,6 @@ public final class EventAnalyticsColumn { .nullable(NOT_NULL) .selectExpression("en.uid") .build(); - public static final AnalyticsTableColumn TRACKED_ENTITY = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.TRACKED_ENTITY_COLUMN_NAME) - .dataType(CHARACTER_11) - .selectExpression("te.uid") - .build(); private static final AnalyticsTableColumn PS = AnalyticsTableColumn.builder() .name(EventAnalyticsColumnName.PS_COLUMN_NAME) @@ -227,12 +234,6 @@ public final class EventAnalyticsColumn { .nullable(NOT_NULL) .selectExpression("coalesce(enrollmentou.uid,ou.uid)") .build(); - public static final AnalyticsTableColumn TRACKED_ENTITY_GEOMETRY = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.TRACKED_ENTITY_GEOMETRY_COLUMN_NAME) - .dataType(GEOMETRY) - .selectExpression("te.geometry") - .build(); private static final List COMMON_COLUMNS = List.of( From 3a368d1be897c05f23fe6ba5e1c2cbefa849cbc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 15 Dec 2024 20:37:12 +0100 Subject: [PATCH 47/48] fix: Update code --- .../table/EnrollmentAnalyticsColumn.java | 7 ++++--- .../analytics/table/EventAnalyticsColumn.java | 18 +++--------------- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EnrollmentAnalyticsColumn.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EnrollmentAnalyticsColumn.java index 4b6ddfe7de85..464a77e2db47 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EnrollmentAnalyticsColumn.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EnrollmentAnalyticsColumn.java @@ -175,7 +175,6 @@ public final class EnrollmentAnalyticsColumn { OU_LEVEL, REGISTRATION_OU); - // Geometry-specific columns private static final List GEOMETRY_COLUMNS = List.of(ENROLLMENT_GEOMETRY, LONGITUDE, LATITUDE); @@ -244,12 +243,14 @@ private static List createJsonColumns(SqlBuilder sqlBuilde } public static List getColumns(SqlBuilder sqlBuilder) { - List columns = new ArrayList<>(COMMON_COLUMNS); + List columns = new ArrayList<>(); + columns.addAll(COMMON_COLUMNS); columns.addAll(createJsonColumns(sqlBuilder)); - // Add database-specific columns based on SqlBuilder capabilities + if (sqlBuilder.supportsGeospatialData()) { columns.addAll(GEOMETRY_COLUMNS); } + return columns; } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EventAnalyticsColumn.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EventAnalyticsColumn.java index 2b1ecfa33a79..bc9bb4afe316 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EventAnalyticsColumn.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EventAnalyticsColumn.java @@ -62,8 +62,6 @@ public final class EventAnalyticsColumn { .selectExpression("te.geometry") .build(); - // Common columns that work across all databases - private static final AnalyticsTableColumn EVENT = AnalyticsTableColumn.builder() .name(EventAnalyticsColumnName.EVENT_COLUMN_NAME) @@ -122,11 +120,6 @@ public final class EventAnalyticsColumn { .dataType(TIMESTAMP) .selectExpression("ev.completeddate") .build(); - - /* - * DHIS2-14981: Use the client-side timestamp if available, otherwise - * the server-side timestamp. Applies to both created and lastupdated. - */ private static final AnalyticsTableColumn CREATED = AnalyticsTableColumn.builder() .name(EventAnalyticsColumnName.CREATED_COLUMN_NAME) @@ -145,7 +138,6 @@ public final class EventAnalyticsColumn { .dataType(VARCHAR_255) .selectExpression("ev.storedby") .build(); - private static final AnalyticsTableColumn EVENT_STATUS = AnalyticsTableColumn.builder() .name(EventAnalyticsColumnName.EVENT_STATUS_COLUMN_NAME) @@ -165,7 +157,6 @@ public final class EventAnalyticsColumn { .selectExpression("ev.geometry") .indexType(IndexType.GIST) .build(); - // TODO latitude and longitude deprecated in 2.30, remove in 2.33 private static final AnalyticsTableColumn LONGITUDE = AnalyticsTableColumn.builder() .name(EventAnalyticsColumnName.LONGITUDE_COLUMN_NAME) @@ -258,11 +249,9 @@ public final class EventAnalyticsColumn { REGISTRATION_OU, ENROLLMENT_OU); - // Geometry-specific columns private static final List GEOMETRY_COLUMNS = List.of(EVENT_GEOMETRY, OU_GEOMETRY, ENROLLMENT_GEOMETRY, LONGITUDE, LATITUDE); - // JSON-specific columns (might vary by database) private static List createJsonColumns(SqlBuilder sqlBuilder) { return List.of( AnalyticsTableColumn.builder() @@ -328,15 +317,14 @@ private static List createJsonColumns(SqlBuilder sqlBuilde /** Returns the appropriate set of columns based on the SqlBuilder type */ public static List getColumns(SqlBuilder sqlBuilder) { - List columns = new ArrayList<>(COMMON_COLUMNS); + List columns = new ArrayList<>(); + columns.addAll(COMMON_COLUMNS); + columns.addAll(createJsonColumns(sqlBuilder)); - // Add database-specific columns based on SqlBuilder capabilities if (sqlBuilder.supportsGeospatialData()) { columns.addAll(GEOMETRY_COLUMNS); } - columns.addAll(createJsonColumns(sqlBuilder)); - return columns; } From 2a7b18f34a7fb1f8be2fbcdd6133ec67c4280965 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 15 Dec 2024 21:02:52 +0100 Subject: [PATCH 48/48] fix: Update code --- .../table/EnrollmentAnalyticsColumn.java | 249 ++++++------ .../analytics/table/EventAnalyticsColumn.java | 381 +++++++++--------- 2 files changed, 303 insertions(+), 327 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EnrollmentAnalyticsColumn.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EnrollmentAnalyticsColumn.java index 464a77e2db47..ff2f4dc1270c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EnrollmentAnalyticsColumn.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EnrollmentAnalyticsColumn.java @@ -62,124 +62,133 @@ public final class EnrollmentAnalyticsColumn { .selectExpression("te.geometry") .build(); - private static final AnalyticsTableColumn ENROLLMENT = - AnalyticsTableColumn.builder() - .name(EnrollmentAnalyticsColumnName.ENROLLMENT_COLUMN_NAME) - .dataType(CHARACTER_11) - .nullable(NOT_NULL) - .selectExpression("en.uid") - .build(); - private static final AnalyticsTableColumn ENROLLMENT_DATE = - AnalyticsTableColumn.builder() - .name(EnrollmentAnalyticsColumnName.ENROLLMENT_DATE_COLUMN_NAME) - .dataType(TIMESTAMP) - .selectExpression("en.enrollmentdate") - .build(); - private static final AnalyticsTableColumn OCCURRED_DATE = - AnalyticsTableColumn.builder() - .name(EnrollmentAnalyticsColumnName.OCCURRED_DATE_COLUMN_NAME) - .dataType(TIMESTAMP) - .selectExpression("en.occurreddate") - .build(); - private static final AnalyticsTableColumn COMPLETED_DATE = - AnalyticsTableColumn.builder() - .name(EnrollmentAnalyticsColumnName.COMPLETED_DATE_COLUMN_NAME) - .dataType(TIMESTAMP) - .selectExpression("case en.status when 'COMPLETED' then en.completeddate end") - .build(); - private static final AnalyticsTableColumn LAST_UPDATED = - AnalyticsTableColumn.builder() - .name(EnrollmentAnalyticsColumnName.LAST_UPDATED_COLUMN_NAME) - .dataType(TIMESTAMP) - .selectExpression("en.lastupdated") - .build(); - private static final AnalyticsTableColumn STORED_BY = - AnalyticsTableColumn.builder() - .name(EnrollmentAnalyticsColumnName.STORED_BY_COLUMN_NAME) - .dataType(VARCHAR_255) - .selectExpression("en.storedby") - .build(); - private static final AnalyticsTableColumn ENROLLMENT_STATUS = - AnalyticsTableColumn.builder() - .name(EnrollmentAnalyticsColumnName.ENROLLMENT_STATUS_COLUMN_NAME) - .dataType(VARCHAR_50) - .selectExpression("en.status") - .build(); - private static final AnalyticsTableColumn LONGITUDE = - AnalyticsTableColumn.builder() - .name(EnrollmentAnalyticsColumnName.LONGITUDE_COLUMN_NAME) - .dataType(DOUBLE) - .selectExpression( - "CASE WHEN 'POINT' = GeometryType(en.geometry) THEN ST_X(en.geometry) ELSE null END") - .build(); - private static final AnalyticsTableColumn LATITUDE = - AnalyticsTableColumn.builder() - .name(EnrollmentAnalyticsColumnName.LATITUDE_COLUMN_NAME) - .dataType(DOUBLE) - .selectExpression( - "CASE WHEN 'POINT' = GeometryType(en.geometry) THEN ST_Y(en.geometry) ELSE null END") - .build(); - private static final AnalyticsTableColumn OU = - AnalyticsTableColumn.builder() - .name(EnrollmentAnalyticsColumnName.OU_COLUMN_NAME) - .dataType(CHARACTER_11) - .nullable(NOT_NULL) - .selectExpression("ou.uid") - .build(); - private static final AnalyticsTableColumn OU_NAME = - AnalyticsTableColumn.builder() - .name(EnrollmentAnalyticsColumnName.OU_NAME_COLUMN_NAME) - .dataType(TEXT) - .nullable(NOT_NULL) - .selectExpression("ou.name") - .build(); - private static final AnalyticsTableColumn OU_CODE = - AnalyticsTableColumn.builder() - .name(EnrollmentAnalyticsColumnName.OU_CODE_COLUMN_NAME) - .dataType(TEXT) - .selectExpression("ou.code") - .build(); - private static final AnalyticsTableColumn OU_LEVEL = - AnalyticsTableColumn.builder() - .name(EnrollmentAnalyticsColumnName.OU_LEVEL_COLUMN_NAME) - .dataType(INTEGER) - .selectExpression("ous.level") - .build(); - private static final AnalyticsTableColumn ENROLLMENT_GEOMETRY = - AnalyticsTableColumn.builder() - .name(EnrollmentAnalyticsColumnName.ENROLLMENT_GEOMETRY_COLUMN_NAME) - .dataType(GEOMETRY) - .selectExpression("en.geometry") - .indexType(IndexType.GIST) - .build(); - private static final AnalyticsTableColumn REGISTRATION_OU = - AnalyticsTableColumn.builder() - .name(EnrollmentAnalyticsColumnName.REGISTRATION_OU_COLUMN_NAME) - .dataType(CHARACTER_11) - .nullable(NOT_NULL) - .selectExpression("coalesce(registrationou.uid,ou.uid)") - .build(); + /** + * Returns a list of {@link AnalyticsTableColumn}. + * + * @param sqlBuilder the {@link SqlBuilder}. + * @return a list of {@link AnalyticsTableColumn}. + */ + public static List getColumns(SqlBuilder sqlBuilder) { + List columns = new ArrayList<>(); + columns.addAll(getCommonColumns(sqlBuilder)); + columns.addAll(getJsonColumns(sqlBuilder)); - private static final List COMMON_COLUMNS = - List.of( - ENROLLMENT, - ENROLLMENT_DATE, - OCCURRED_DATE, - COMPLETED_DATE, - LAST_UPDATED, - STORED_BY, - ENROLLMENT_STATUS, - OU, - OU_NAME, - OU_CODE, - OU_LEVEL, - REGISTRATION_OU); + if (sqlBuilder.supportsGeospatialData()) { + columns.addAll(getGeometryColumns(sqlBuilder)); + } - private static final List GEOMETRY_COLUMNS = - List.of(ENROLLMENT_GEOMETRY, LONGITUDE, LATITUDE); + return columns; + } - private static List createJsonColumns(SqlBuilder sqlBuilder) { + /** + * Returns a list of {@link AnalyticsTableColumn}. + * + * @param sqlBuilder the {@link SqlBuilder}. + * @return a list of {@link AnalyticsTableColumn}. + */ + private static List getCommonColumns(SqlBuilder sqlBuilder) { + return List.of( + AnalyticsTableColumn.builder() + .name(EnrollmentAnalyticsColumnName.ENROLLMENT_COLUMN_NAME) + .dataType(CHARACTER_11) + .nullable(NOT_NULL) + .selectExpression("en.uid") + .build(), + AnalyticsTableColumn.builder() + .name(EnrollmentAnalyticsColumnName.ENROLLMENT_DATE_COLUMN_NAME) + .dataType(TIMESTAMP) + .selectExpression("en.enrollmentdate") + .build(), + AnalyticsTableColumn.builder() + .name(EnrollmentAnalyticsColumnName.OCCURRED_DATE_COLUMN_NAME) + .dataType(TIMESTAMP) + .selectExpression("en.occurreddate") + .build(), + AnalyticsTableColumn.builder() + .name(EnrollmentAnalyticsColumnName.COMPLETED_DATE_COLUMN_NAME) + .dataType(TIMESTAMP) + .selectExpression("case en.status when 'COMPLETED' then en.completeddate end") + .build(), + AnalyticsTableColumn.builder() + .name(EnrollmentAnalyticsColumnName.LAST_UPDATED_COLUMN_NAME) + .dataType(TIMESTAMP) + .selectExpression("en.lastupdated") + .build(), + AnalyticsTableColumn.builder() + .name(EnrollmentAnalyticsColumnName.STORED_BY_COLUMN_NAME) + .dataType(VARCHAR_255) + .selectExpression("en.storedby") + .build(), + AnalyticsTableColumn.builder() + .name(EnrollmentAnalyticsColumnName.ENROLLMENT_STATUS_COLUMN_NAME) + .dataType(VARCHAR_50) + .selectExpression("en.status") + .build(), + AnalyticsTableColumn.builder() + .name(EnrollmentAnalyticsColumnName.OU_COLUMN_NAME) + .dataType(CHARACTER_11) + .nullable(NOT_NULL) + .selectExpression("ou.uid") + .build(), + AnalyticsTableColumn.builder() + .name(EnrollmentAnalyticsColumnName.OU_NAME_COLUMN_NAME) + .dataType(TEXT) + .nullable(NOT_NULL) + .selectExpression("ou.name") + .build(), + AnalyticsTableColumn.builder() + .name(EnrollmentAnalyticsColumnName.OU_CODE_COLUMN_NAME) + .dataType(TEXT) + .selectExpression("ou.code") + .build(), + AnalyticsTableColumn.builder() + .name(EnrollmentAnalyticsColumnName.OU_LEVEL_COLUMN_NAME) + .dataType(INTEGER) + .selectExpression("ous.level") + .build(), + AnalyticsTableColumn.builder() + .name(EnrollmentAnalyticsColumnName.REGISTRATION_OU_COLUMN_NAME) + .dataType(CHARACTER_11) + .nullable(NOT_NULL) + .selectExpression("coalesce(registrationou.uid,ou.uid)") + .build()); + } + /** + * Returns a list of geometry {@link AnalyticsTableColumn}. + * + * @param sqlBuilder the {@link SqlBuilder}. + * @return a list of {@link AnalyticsTableColumn}. + */ + private static List getGeometryColumns(SqlBuilder sqlBuilder) { + return List.of( + AnalyticsTableColumn.builder() + .name(EnrollmentAnalyticsColumnName.ENROLLMENT_GEOMETRY_COLUMN_NAME) + .dataType(GEOMETRY) + .selectExpression("en.geometry") + .indexType(IndexType.GIST) + .build(), + AnalyticsTableColumn.builder() + .name(EnrollmentAnalyticsColumnName.LONGITUDE_COLUMN_NAME) + .dataType(DOUBLE) + .selectExpression( + "CASE WHEN 'POINT' = GeometryType(en.geometry) THEN ST_X(en.geometry) ELSE null END") + .build(), + AnalyticsTableColumn.builder() + .name(EnrollmentAnalyticsColumnName.LATITUDE_COLUMN_NAME) + .dataType(DOUBLE) + .selectExpression( + "CASE WHEN 'POINT' = GeometryType(en.geometry) THEN ST_Y(en.geometry) ELSE null END") + .build()); + } + + /** + * Returns a list of {@link AnalyticsTableColumn}. + * + * @param sqlBuilder the {@link SqlBuilder}. + * @return a list of {@link AnalyticsTableColumn}. + */ + private static List getJsonColumns(SqlBuilder sqlBuilder) { return List.of( AnalyticsTableColumn.builder() .name(EnrollmentAnalyticsColumnName.CREATED_BY_USERNAME_COLUMN_NAME) @@ -241,16 +250,4 @@ private static List createJsonColumns(SqlBuilder sqlBuilde .skipIndex(Skip.SKIP) .build()); } - - public static List getColumns(SqlBuilder sqlBuilder) { - List columns = new ArrayList<>(); - columns.addAll(COMMON_COLUMNS); - columns.addAll(createJsonColumns(sqlBuilder)); - - if (sqlBuilder.supportsGeospatialData()) { - columns.addAll(GEOMETRY_COLUMNS); - } - - return columns; - } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EventAnalyticsColumn.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EventAnalyticsColumn.java index bc9bb4afe316..cd65f8081200 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EventAnalyticsColumn.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EventAnalyticsColumn.java @@ -62,197 +62,189 @@ public final class EventAnalyticsColumn { .selectExpression("te.geometry") .build(); - private static final AnalyticsTableColumn EVENT = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.EVENT_COLUMN_NAME) - .dataType(CHARACTER_11) - .nullable(NOT_NULL) - .selectExpression("ev.uid") - .build(); - private static final AnalyticsTableColumn ENROLLMENT = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.ENROLLMENT_COLUMN_NAME) - .dataType(CHARACTER_11) - .nullable(NOT_NULL) - .selectExpression("en.uid") - .build(); - private static final AnalyticsTableColumn PS = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.PS_COLUMN_NAME) - .dataType(CHARACTER_11) - .nullable(NOT_NULL) - .selectExpression("ps.uid") - .build(); - private static final AnalyticsTableColumn AO = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.AO_COLUMN_NAME) - .dataType(CHARACTER_11) - .nullable(NOT_NULL) - .selectExpression("acs.categoryoptioncombouid") - .build(); - private static final AnalyticsTableColumn ENROLLMENT_DATE = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.ENROLLMENT_DATE_COLUMN_NAME) - .dataType(TIMESTAMP) - .selectExpression("en.enrollmentdate") - .build(); - private static final AnalyticsTableColumn ENROLLMENT_OCCURRED_DATE = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.ENROLLMENT_OCCURRED_DATE_COLUMN_NAME) - .dataType(TIMESTAMP) - .selectExpression("en.occurreddate") - .build(); - private static final AnalyticsTableColumn OCCURRED_DATE = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.OCCURRED_DATE_COLUMN_NAME) - .dataType(TIMESTAMP) - .selectExpression("ev.occurreddate") - .build(); - private static final AnalyticsTableColumn SCHEDULED_DATE = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.SCHEDULED_DATE_COLUMN_NAME) - .dataType(TIMESTAMP) - .selectExpression("ev.scheduleddate") - .build(); - private static final AnalyticsTableColumn COMPLETED_DATE = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.COMPLETED_DATE_COLUMN_NAME) - .dataType(TIMESTAMP) - .selectExpression("ev.completeddate") - .build(); - private static final AnalyticsTableColumn CREATED = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.CREATED_COLUMN_NAME) - .dataType(TIMESTAMP) - .selectExpression(firstIfNotNullOrElse("ev.createdatclient", "ev.created")) - .build(); - private static final AnalyticsTableColumn LAST_UPDATED = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.LAST_UPDATED_COLUMN_NAME) - .dataType(TIMESTAMP) - .selectExpression(firstIfNotNullOrElse("ev.lastupdatedatclient", "ev.lastupdated")) - .build(); - private static final AnalyticsTableColumn STOREDBY = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.STORED_BY_COLUMN_NAME) - .dataType(VARCHAR_255) - .selectExpression("ev.storedby") - .build(); - private static final AnalyticsTableColumn EVENT_STATUS = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.EVENT_STATUS_COLUMN_NAME) - .dataType(VARCHAR_50) - .selectExpression("ev.status") - .build(); - private static final AnalyticsTableColumn ENROLLMENT_STATUS = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.ENROLLMENT_STATUS_COLUMN_NAME) - .dataType(VARCHAR_50) - .selectExpression("en.status") - .build(); - private static final AnalyticsTableColumn EVENT_GEOMETRY = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.EVENT_GEOMETRY_COLUMN_NAME) - .dataType(GEOMETRY) - .selectExpression("ev.geometry") - .indexType(IndexType.GIST) - .build(); - private static final AnalyticsTableColumn LONGITUDE = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.LONGITUDE_COLUMN_NAME) - .dataType(DOUBLE) - .selectExpression( - "CASE WHEN 'POINT' = GeometryType(ev.geometry) THEN ST_X(ev.geometry) ELSE null END") - .build(); - private static final AnalyticsTableColumn LATITUDE = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.LATITUDE_COLUMN_NAME) - .dataType(DOUBLE) - .selectExpression( - "CASE WHEN 'POINT' = GeometryType(ev.geometry) THEN ST_Y(ev.geometry) ELSE null END") - .build(); - private static final AnalyticsTableColumn OU = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.OU_COLUMN_NAME) - .dataType(CHARACTER_11) - .nullable(NOT_NULL) - .selectExpression("ou.uid") - .build(); - private static final AnalyticsTableColumn OU_NAME = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.OU_NAME_COLUMN_NAME) - .dataType(TEXT) - .nullable(NOT_NULL) - .selectExpression("ou.name") - .build(); - private static final AnalyticsTableColumn OU_CODE = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.OU_CODE_COLUMN_NAME) - .dataType(TEXT) - .selectExpression("ou.code") - .build(); - private static final AnalyticsTableColumn OU_LEVEL = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.OU_LEVEL_COLUMN_NAME) - .dataType(INTEGER) - .selectExpression("ous.level") - .build(); - private static final AnalyticsTableColumn OU_GEOMETRY = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.OU_GEOMETRY_COLUMN_NAME) - .dataType(GEOMETRY) - .selectExpression("ou.geometry") - .indexType(IndexType.GIST) - .build(); - private static final AnalyticsTableColumn ENROLLMENT_GEOMETRY = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.ENROLLMENT_GEOMETRY_COLUMN_NAME) - .dataType(GEOMETRY) - .selectExpression("en.geometry") - .indexType(IndexType.GIST) - .build(); - private static final AnalyticsTableColumn REGISTRATION_OU = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.REGISTRATION_OU_COLUMN_NAME) - .dataType(CHARACTER_11) - .nullable(NOT_NULL) - .selectExpression("coalesce(registrationou.uid,ou.uid)") - .build(); - private static final AnalyticsTableColumn ENROLLMENT_OU = - AnalyticsTableColumn.builder() - .name(EventAnalyticsColumnName.ENROLLMENT_OU_COLUMN_NAME) - .dataType(CHARACTER_11) - .nullable(NOT_NULL) - .selectExpression("coalesce(enrollmentou.uid,ou.uid)") - .build(); + /** + * Returns a list of {@link AnalyticsTableColumn}. + * + * @param sqlBuilder the {@link SqlBuilder}. + * @return a list of {@link AnalyticsTableColumn}. + */ + public static List getColumns(SqlBuilder sqlBuilder) { + List columns = new ArrayList<>(); + columns.addAll(getCommonColumns(sqlBuilder)); + columns.addAll(getJsonColumns(sqlBuilder)); - private static final List COMMON_COLUMNS = - List.of( - EVENT, - ENROLLMENT, - PS, - AO, - ENROLLMENT_DATE, - ENROLLMENT_OCCURRED_DATE, - OCCURRED_DATE, - SCHEDULED_DATE, - COMPLETED_DATE, - CREATED, - LAST_UPDATED, - STOREDBY, - EVENT_STATUS, - ENROLLMENT_STATUS, - OU, - OU_NAME, - OU_CODE, - OU_LEVEL, - REGISTRATION_OU, - ENROLLMENT_OU); + if (sqlBuilder.supportsGeospatialData()) { + columns.addAll(getGeometryColumns(sqlBuilder)); + } + + return columns; + } + + /** + * Returns a list of {@link AnalyticsTableColumn}. + * + * @param sqlBuilder the {@link SqlBuilder}. + * @return a list of {@link AnalyticsTableColumn}. + */ + private static List getCommonColumns(SqlBuilder sqlBuilder) { + return List.of( + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.EVENT_COLUMN_NAME) + .dataType(CHARACTER_11) + .nullable(NOT_NULL) + .selectExpression("ev.uid") + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.ENROLLMENT_COLUMN_NAME) + .dataType(CHARACTER_11) + .nullable(NOT_NULL) + .selectExpression("en.uid") + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.PS_COLUMN_NAME) + .dataType(CHARACTER_11) + .nullable(NOT_NULL) + .selectExpression("ps.uid") + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.AO_COLUMN_NAME) + .dataType(CHARACTER_11) + .nullable(NOT_NULL) + .selectExpression("acs.categoryoptioncombouid") + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.ENROLLMENT_DATE_COLUMN_NAME) + .dataType(TIMESTAMP) + .selectExpression("en.enrollmentdate") + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.ENROLLMENT_OCCURRED_DATE_COLUMN_NAME) + .dataType(TIMESTAMP) + .selectExpression("en.occurreddate") + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.OCCURRED_DATE_COLUMN_NAME) + .dataType(TIMESTAMP) + .selectExpression("ev.occurreddate") + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.SCHEDULED_DATE_COLUMN_NAME) + .dataType(TIMESTAMP) + .selectExpression("ev.scheduleddate") + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.COMPLETED_DATE_COLUMN_NAME) + .dataType(TIMESTAMP) + .selectExpression("ev.completeddate") + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.CREATED_COLUMN_NAME) + .dataType(TIMESTAMP) + .selectExpression(firstIfNotNullOrElse("ev.createdatclient", "ev.created")) + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.LAST_UPDATED_COLUMN_NAME) + .dataType(TIMESTAMP) + .selectExpression(firstIfNotNullOrElse("ev.lastupdatedatclient", "ev.lastupdated")) + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.STORED_BY_COLUMN_NAME) + .dataType(VARCHAR_255) + .selectExpression("ev.storedby") + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.EVENT_STATUS_COLUMN_NAME) + .dataType(VARCHAR_50) + .selectExpression("ev.status") + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.ENROLLMENT_STATUS_COLUMN_NAME) + .dataType(VARCHAR_50) + .selectExpression("en.status") + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.OU_COLUMN_NAME) + .dataType(CHARACTER_11) + .nullable(NOT_NULL) + .selectExpression("ou.uid") + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.OU_NAME_COLUMN_NAME) + .dataType(TEXT) + .nullable(NOT_NULL) + .selectExpression("ou.name") + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.OU_CODE_COLUMN_NAME) + .dataType(TEXT) + .selectExpression("ou.code") + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.OU_LEVEL_COLUMN_NAME) + .dataType(INTEGER) + .selectExpression("ous.level") + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.REGISTRATION_OU_COLUMN_NAME) + .dataType(CHARACTER_11) + .nullable(NOT_NULL) + .selectExpression("coalesce(registrationou.uid,ou.uid)") + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.ENROLLMENT_OU_COLUMN_NAME) + .dataType(CHARACTER_11) + .nullable(NOT_NULL) + .selectExpression("coalesce(enrollmentou.uid,ou.uid)") + .build()); + } - private static final List GEOMETRY_COLUMNS = - List.of(EVENT_GEOMETRY, OU_GEOMETRY, ENROLLMENT_GEOMETRY, LONGITUDE, LATITUDE); + /** + * Returns a list of geometry {@link AnalyticsTableColumn}. + * + * @param sqlBuilder the {@link SqlBuilder}. + * @return a list of {@link AnalyticsTableColumn}. + */ + private static List getGeometryColumns(SqlBuilder sqlBuilder) { + return List.of( + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.EVENT_GEOMETRY_COLUMN_NAME) + .dataType(GEOMETRY) + .selectExpression("ev.geometry") + .indexType(IndexType.GIST) + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.OU_GEOMETRY_COLUMN_NAME) + .dataType(GEOMETRY) + .selectExpression("ou.geometry") + .indexType(IndexType.GIST) + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.ENROLLMENT_GEOMETRY_COLUMN_NAME) + .dataType(GEOMETRY) + .selectExpression("en.geometry") + .indexType(IndexType.GIST) + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.LONGITUDE_COLUMN_NAME) + .dataType(DOUBLE) + .selectExpression( + "CASE WHEN 'POINT' = GeometryType(ev.geometry) THEN ST_X(ev.geometry) ELSE null END") + .build(), + AnalyticsTableColumn.builder() + .name(EventAnalyticsColumnName.LATITUDE_COLUMN_NAME) + .dataType(DOUBLE) + .selectExpression( + "CASE WHEN 'POINT' = GeometryType(ev.geometry) THEN ST_Y(ev.geometry) ELSE null END") + .build()); + } - private static List createJsonColumns(SqlBuilder sqlBuilder) { + /** + * Returns a list of {@link AnalyticsTableColumn}. + * + * @param sqlBuilder the {@link SqlBuilder}. + * @return a list of {@link AnalyticsTableColumn}. + */ + private static List getJsonColumns(SqlBuilder sqlBuilder) { return List.of( AnalyticsTableColumn.builder() .name(EventAnalyticsColumnName.CREATED_BY_USERNAME_COLUMN_NAME) @@ -315,19 +307,6 @@ private static List createJsonColumns(SqlBuilder sqlBuilde .build()); } - /** Returns the appropriate set of columns based on the SqlBuilder type */ - public static List getColumns(SqlBuilder sqlBuilder) { - List columns = new ArrayList<>(); - columns.addAll(COMMON_COLUMNS); - columns.addAll(createJsonColumns(sqlBuilder)); - - if (sqlBuilder.supportsGeospatialData()) { - columns.addAll(GEOMETRY_COLUMNS); - } - - return columns; - } - /** * Returns a SQL expression that returns the first argument if it is not null, otherwise the * second argument.