Skip to content

Commit

Permalink
fix: Use dynamic org unit level analytics table columns [DHIS2-18417] (…
Browse files Browse the repository at this point in the history 
…#19439)
  • Loading branch information
@larshelge
larshelge authored Dec 11, 2024
1 parent d3f466e commit f4f2aca
Show file tree
Hide file tree
Showing 10 changed files with 26 additions and 127 deletions.
41 changes: 2 additions & 39 deletions ...s/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType;
import static org.hisp.dhis.analytics.util.DisplayNameUtils.getDisplayName;
import static org.hisp.dhis.commons.util.TextUtils.removeLastComma;
import static org.hisp.dhis.commons.util.TextUtils.replace;
import static org.hisp.dhis.db.model.DataType.BOOLEAN;
import static org.hisp.dhis.db.model.DataType.CHARACTER_11;
import static org.hisp.dhis.db.model.DataType.DOUBLE;
Expand Down Expand Up @@ -65,7 +64,6 @@
import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings;
import org.hisp.dhis.category.CategoryService;
import org.hisp.dhis.common.IdentifiableObjectManager;
import org.hisp.dhis.common.ValueType;
import org.hisp.dhis.dataapproval.DataApprovalLevelService;
import org.hisp.dhis.db.model.IndexType;
import org.hisp.dhis.db.model.Logged;
Expand All @@ -85,7 +83,7 @@
import org.springframework.transaction.annotation.Transactional;

@Component("org.hisp.dhis.analytics.TrackedEntityAnalyticsTableManager")
public class JdbcTrackedEntityAnalyticsTableManager extends AbstractJdbcTableManager {
public class JdbcTrackedEntityAnalyticsTableManager extends AbstractEventJdbcTableManager {
private static final String PROGRAMS_BY_TET_KEY = "programsByTetUid";

private static final String ALL_NON_CONFIDENTIAL_TET_ATTRIBUTES =
Expand Down Expand Up @@ -215,7 +213,7 @@ private List<AnalyticsTableColumn> getColumns(
.name(tea.getUid())
.dataType(getColumnType(tea.getValueType(), isSpatialSupport()))
.selectExpression(
castBasedOnType(tea.getValueType(), quote(tea.getUid()) + ".value"))
getColumnExpression(tea.getValueType(), quote(tea.getUid()) + ".value"))
.build())
.toList());

Expand All @@ -242,41 +240,6 @@ private Stream<TrackedEntityAttribute> getAllTrackedEntityAttributes(
return getAllTrackedEntityAttributesByEntityType(trackedEntityType);
}

/**
* Returns the select clause, potentially with a cast statement, based on the given value type.
*
* @param valueType the value type to represent as database column type.
*/
private String castBasedOnType(ValueType valueType, String columnName) {
if (valueType.isDecimal()) {

return replace(
" cast(${columnName} as ${type})",
Map.of("columnName", columnName, "type", sqlBuilder.dataTypeDouble()));
}
if (valueType.isInteger()) {
return replace(" cast(${columnName} as bigint)", Map.of("columnName", columnName));
}
if (valueType.isBoolean()) {
return replace(
" case when ${columnName} = 'true' then 1 when ${columnName} = 'false' then 0 end ",
Map.of("columnName", columnName));
}
if (valueType.isDate()) {
return replace(
" cast(${columnName} as ${type})",
Map.of("columnName", columnName, "type", sqlBuilder.dataTypeTimestamp()));
}
if (valueType.isGeo() && isSpatialSupport() && sqlBuilder.supportsGeospatialData()) {
return replace(
"""
\s ST_GeomFromGeoJSON('{"type":"Point", "coordinates":' || (${columnName}) || ',
"crs":{"type":"name", "properties":{"name":"EPSG:4326"}}}')""",
Map.of("columnName", columnName));
}
return columnName;
}

/**
* Returns all {@link TrackedEntityAttribute} for the given {@link TrackedEntityType} and
* programs.
Expand Down
25 changes: 1 addition & 24 deletions ...java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,30 +115,6 @@ public class JdbcTrackedEntityEnrollmentsAnalyticsTableManager extends AbstractJ
.dataType(VARCHAR_50)
.selectExpression("en.status")
.build(),
AnalyticsTableColumn.builder()
.name("uidlevel1")
.dataType(CHARACTER_11)
.nullable(NULL)
.selectExpression("ous.uidlevel1")
.build(),
AnalyticsTableColumn.builder()
.name("uidlevel2")
.dataType(CHARACTER_11)
.nullable(NULL)
.selectExpression("ous.uidlevel2")
.build(),
AnalyticsTableColumn.builder()
.name("uidlevel3")
.dataType(CHARACTER_11)
.nullable(NULL)
.selectExpression("ous.uidlevel3")
.build(),
AnalyticsTableColumn.builder()
.name("uidlevel4")
.dataType(CHARACTER_11)
.nullable(NULL)
.selectExpression("ous.uidlevel4")
.build(),
AnalyticsTableColumn.builder()
.name("ou")
.dataType(CHARACTER_11)
Expand Down Expand Up @@ -225,6 +201,7 @@ private List<AnalyticsTableColumn> getColumns() {
List<AnalyticsTableColumn> columns = new ArrayList<>();
columns.addAll(getFixedCols());
columns.add(getOrganisationUnitNameHierarchyColumn());
columns.addAll(getOrganisationUnitLevelColumns());
if (sqlBuilder.supportsDeclarativePartitioning()) {
columns.add(getPartitionColumn());
}
Expand Down
25 changes: 1 addition & 24 deletions ...main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,30 +143,6 @@ public class JdbcTrackedEntityEventsAnalyticsTableManager extends AbstractJdbcTa
.dataType(VARCHAR_50)
.selectExpression("ev.status")
.build(),
AnalyticsTableColumn.builder()
.name("uidlevel1")
.dataType(CHARACTER_11)
.nullable(NULL)
.selectExpression("ous.uidlevel1")
.build(),
AnalyticsTableColumn.builder()
.name("uidlevel2")
.dataType(CHARACTER_11)
.nullable(NULL)
.selectExpression("ous.uidlevel2")
.build(),
AnalyticsTableColumn.builder()
.name("uidlevel3")
.dataType(CHARACTER_11)
.nullable(NULL)
.selectExpression("ous.uidlevel3")
.build(),
AnalyticsTableColumn.builder()
.name("uidlevel4")
.dataType(CHARACTER_11)
.nullable(NULL)
.selectExpression("ous.uidlevel4")
.build(),
AnalyticsTableColumn.builder()
.name("ou")
.dataType(CHARACTER_11)
Expand Down Expand Up @@ -367,6 +343,7 @@ private List<AnalyticsTableColumn> getColumns() {
}

columns.add(getOrganisationUnitNameHierarchyColumn());
columns.addAll(getOrganisationUnitLevelColumns());

return columns;
}
Expand Down
30 changes: 0 additions & 30 deletions ...ces/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
*/
package org.hisp.dhis.analytics.util;

import static org.apache.commons.lang3.StringUtils.EMPTY;
import static org.hisp.dhis.common.DataDimensionItem.DATA_DIM_TYPE_CLASS_MAP;
import static org.hisp.dhis.common.DimensionalObject.ATTRIBUTEOPTIONCOMBO_DIM_ID;
import static org.hisp.dhis.common.DimensionalObject.CATEGORYOPTIONCOMBO_DIM_ID;
Expand Down Expand Up @@ -1202,33 +1201,4 @@ public static String replaceStringBetween(
Matcher matcher = pattern.matcher(original);
return matcher.replaceAll(startToken + replacement + endToken);
}

/**
* Returns a string containing closing parenthesis. The number of parenthesis is based on the
* number of missing closing parenthesis in the argument string.
*
* <p>Example:
*
* <p>{@code} input: "((( ))" -> output: ")" {@code}
*
* @param str a string.
* @return a String containing 0 or more "closing" parenthesis
*/
public static String getClosingParentheses(String str) {
if (StringUtils.isEmpty(str)) {
return EMPTY;
}

int open = 0;

for (int i = 0; i < str.length(); i++) {
if (str.charAt(i) == '(') {
open++;
} else if ((str.charAt(i) == ')') && open >= 1) {
open--;
}
}

return StringUtils.repeat(")", open);
}
}
4 changes: 2 additions & 2 deletions .../java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManagerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,9 +160,9 @@ void testPopulateTable() {
String expectedSql =
"""
insert into analytics_te_event_tetuid_temp ("trackedentity","program","enrollment","programstage","event","occurreddate","lastupdated","created",
"scheduleddate","status","uidlevel1","uidlevel2","uidlevel3","uidlevel4","ou","ouname","oucode","oulevel","eventdatavalues","eventgeometry",
"scheduleddate","status","ou","ouname","oucode","oulevel","eventdatavalues","eventgeometry",
"evlongitude","evlatitude","ounamehierarchy") select distinct te.uid,p.uid,en.uid,ps.uid,ev.uid,ev.occurreddate,ev.lastupdated,
ev.created,ev.scheduleddate,ev.status,ous.uidlevel1,ous.uidlevel2,ous.uidlevel3,ous.uidlevel4,ous.organisationunituid,ous.name,ous.code,ous.level,
ev.created,ev.scheduleddate,ev.status,ous.organisationunituid,ous.name,ous.code,ous.level,
%s,
ev.geometry,case when 'POINT' = GeometryType(ev.geometry) then ST_X(ev.geometry) end,case when 'POINT' = GeometryType(ev.geometry) then ST_Y(ev.geometry) end,concat_ws(' / ',) as ounamehierarchy
from "event" ev inner join "enrollment" en on en.enrollmentid=ev.enrollmentid
Expand Down
8 changes: 0 additions & 8 deletions ...dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/util/AnalyticsUtilsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -739,14 +739,6 @@ void testGetEnumCaseSensitivity() {
assertNull(EnumUtils.getEnum(Database.class, "postgresql"));
}

@Test
void testGetClosingParentheses() {
assertEquals("", AnalyticsUtils.getClosingParentheses(null));
assertEquals("", AnalyticsUtils.getClosingParentheses(""));
assertEquals(")", AnalyticsUtils.getClosingParentheses("from(select(select (*))"));
assertEquals("))", AnalyticsUtils.getClosingParentheses("(("));
}

@Test
void whenUncategorizedSQLException_withTableNotExisting_thenThrowException() {
SQLException sqlException = new SQLException("relation does not exist", "42P01");
Expand Down
5 changes: 5 additions & 0 deletions ...his-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,11 @@ public boolean supportsVacuum() {
return false;
}

@Override
public boolean supportsCorrelatedSubquery() {
return false;
}

@Override
public boolean requiresIndexesForAnalytics() {
return false;
Expand Down
5 changes: 5 additions & 0 deletions dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,11 @@ public boolean supportsVacuum() {
return false;
}

@Override
public boolean supportsCorrelatedSubquery() {
return true;
}

@Override
public boolean requiresIndexesForAnalytics() {
return false;
Expand Down
5 changes: 5 additions & 0 deletions ...2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,11 @@ public boolean supportsVacuum() {
return true;
}

@Override
public boolean supportsCorrelatedSubquery() {
return true;
}

@Override
public boolean requiresIndexesForAnalytics() {
return true;
Expand Down
5 changes: 5 additions & 0 deletions dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,11 @@ public interface SqlBuilder {
*/
boolean supportsVacuum();

/**
* @return true if the DBMS supports corrected subqueries.
*/
boolean supportsCorrelatedSubquery();

/**
* @return true if the DBMS requires indexes for analytics tables for performance.
*/
Expand Down

0 comments on commit f4f2aca

Please sign in to comment.