From 4bdeb79b717b23fd90a472375bb38ed54ea8b06f Mon Sep 17 00:00:00 2001 From: jason-p-pickering Date: Thu, 30 Nov 2023 11:18:16 +0100 Subject: [PATCH 1/9] chore:Replace organisation units with no groups integrity check with SQL --- .../DefaultDataIntegrityService.java | 9 -- .../main/resources/data-integrity-checks.yaml | 1 + .../orgunits/orgunits_no_groups.yaml | 54 ++++++++ .../DataIntegrityServiceTest.java | 7 -- .../DataIntegrityYamlReaderTest.java | 2 +- ...ganisationUnitsNoGroupsControllerTest.java | 118 ++++++++++++++++++ 6 files changed, 174 insertions(+), 17 deletions(-) create mode 100644 dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_no_groups.yaml create mode 100644 dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java index bd25dca42688..90731b5d7d4f 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java @@ -444,10 +444,6 @@ List getOrphanedOrganisationUnits() { return toSimpleIssueList(organisationUnitService.getOrphanedOrganisationUnits().stream()); } - List getOrganisationUnitsWithoutGroups() { - return toSimpleIssueList(organisationUnitService.getOrganisationUnitsWithoutGroups().stream()); - } - List getOrganisationUnitsViolatingExclusiveGroupSets() { return toIssueList( organisationUnitService.getOrganisationUnitsViolatingExclusiveGroupSets().stream(), @@ -609,10 +605,6 @@ public void initIntegrityChecks() { DataIntegrityCheckType.ORG_UNITS_BEING_ORPHANED, OrganisationUnit.class, this::getOrphanedOrganisationUnits); - registerNonDatabaseIntegrityCheck( - DataIntegrityCheckType.ORG_UNITS_WITHOUT_GROUPS, - OrganisationUnit.class, - this::getOrganisationUnitsWithoutGroups); registerNonDatabaseIntegrityCheck( DataIntegrityCheckType.ORG_UNITS_VIOLATING_EXCLUSIVE_GROUP_SETS, OrganisationUnit.class, @@ -621,7 +613,6 @@ public void initIntegrityChecks() { DataIntegrityCheckType.ORG_UNIT_GROUPS_WITHOUT_GROUP_SETS, OrganisationUnitGroup.class, this::getOrganisationUnitGroupsWithoutGroupSets); - registerNonDatabaseIntegrityCheck( DataIntegrityCheckType.VALIDATION_RULES_WITHOUT_GROUPS, ValidationRule.class, diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks.yaml index 42908a8c8a71..a11a8189fb4e 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks.yaml @@ -27,6 +27,7 @@ checks: - orgunits/orgunits_invalid_geometry.yaml - orgunits/orgunits_no_geometry.yaml - orgunits/orgunits_same_name_and_parent.yaml + - orgunits/orgunits_no_groups.yaml - option_sets/option_sets_empty.yaml - option_sets/unused_option_sets.yaml - option_sets/option_sets_wrong_sort_order.yaml diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_no_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_no_groups.yaml new file mode 100644 index 000000000000..15c8cc522cb7 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_no_groups.yaml @@ -0,0 +1,54 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: orgunits_no_groups +description: Organisation units with no groups. +section: Organisation units +section_order: 13 +summary_sql: >- + WITH orgunits_no_groups as ( + SELECT uid,name from organisationunit + where organisationunitid NOT IN ( + SELECT organisationunitid from orgunitgroupmembers)) + SELECT + COUNT(*)as value, + 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM organisationunit), 0) as percent + FROM orgunits_no_groups; +details_sql: >- + SELECT uid,name from organisationunit + where organisationunitid NOT IN ( + SELECT organisationunitid from orgunitgroupmembers) + ORDER BY name; +details_id_type: organisationUnits +severity: WARNING +introduction: | + All organisation units should usually belong to at least one organisation unit group. + When organisation units do not belong to any groups, they become more difficult to identify + in analysis apps like the data visualizer. +recommendation: | + Create useful organisation unit groups to help users filter certain classes of organisation + units. These groups may or may not be used in organisation unit group sets \ No newline at end of file diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityServiceTest.java b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityServiceTest.java index bf5a54541aee..1dd6b28dd1bf 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityServiceTest.java @@ -428,13 +428,6 @@ void testGetOrphanedOrganisationUnits() { verifyNoMoreInteractions(organisationUnitService); } - @Test - void testGetOrganisationUnitsWithoutGroups() { - subject.getOrganisationUnitsWithoutGroups(); - verify(organisationUnitService).getOrganisationUnitsWithoutGroups(); - verifyNoMoreInteractions(organisationUnitService); - } - @Test void testGetProgramRulesWithNoExpression() { programRuleB.setCondition(null); diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java index b34c775dee60..b65ab04cb87a 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java @@ -56,7 +56,7 @@ void testReadDataIntegrityYaml() { List checks = new ArrayList<>(); readYaml(checks, "data-integrity-checks.yaml", "data-integrity-checks", CLASS_PATH); - assertEquals(63, checks.size()); + assertEquals(64, checks.size()); // Names should be unique List allNames = checks.stream().map(DataIntegrityCheck::getName).toList(); diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java new file mode 100644 index 000000000000..6a4252877c2e --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.web.WebClientUtils.assertStatus; + +import org.hisp.dhis.web.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Checks for organisation units with no groups. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_no_groups.yaml} + * + * @author Jason P. Pickering + */ +public class DataIntegrityOrganisationUnitsNoGroupsControllerTest + extends AbstractDataIntegrityIntegrationTest { + private String orgunitA; + + private String orgunitB; + + private String testOrgUnitGroupA; + + private static final String check = "orgunits_no_groups"; + + private static final String detailsIdType = "organisationUnits"; + + @Test + void testOrgunitsNoGroups() { + + orgunitA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Fish District', 'shortName': 'Fish District', 'openingDate' : '2022-01-01'}")); + + orgunitB = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Pizza District', 'shortName': 'Pizza District', 'openingDate' : '2022-01-01'}")); + + // Create an orgunit group + testOrgUnitGroupA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnitGroups", + "{'name': 'Type A', 'shortName': 'Type A', 'organisationUnits' : [{'id' : '" + + orgunitA + + "'}]}")); + assertHasDataIntegrityIssues( + detailsIdType, check, 50, orgunitB, "Pizza District", "Type", true); + } + + @Test + void testOrgunitsInGroups() { + + orgunitA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Fish District', 'shortName': 'Fish District', 'openingDate' : '2022-01-01'}")); + + orgunitB = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnits", + "{ 'name': 'Pizza District', 'shortName': 'Pizza District', 'openingDate' : '2022-01-01'}")); + + // Create an orgunit group + testOrgUnitGroupA = + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnitGroups", + "{'name': 'Type A', 'shortName': 'Type A', 'organisationUnits' : [{'id' : '" + + orgunitA + + "'}, {'id' : '" + + orgunitB + + "'}]}")); + assertHasNoDataIntegrityIssues(detailsIdType, check, true); + } + + @Test + void testOrgUnitsNoGroupsDivideByZero() { + assertHasNoDataIntegrityIssues(detailsIdType, check, false); + } +} From 1dc2170f48cefd6287fd4eac5799a35f738e2742 Mon Sep 17 00:00:00 2001 From: jason-p-pickering Date: Thu, 30 Nov 2023 11:27:02 +0100 Subject: [PATCH 2/9] Remove public class declaration --- .../DataIntegrityOrganisationUnitsNoGroupsControllerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java index 6a4252877c2e..a261b7aabe37 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java @@ -38,7 +38,7 @@ * * @author Jason P. Pickering */ -public class DataIntegrityOrganisationUnitsNoGroupsControllerTest +private class DataIntegrityOrganisationUnitsNoGroupsControllerTest extends AbstractDataIntegrityIntegrationTest { private String orgunitA; From 14735bf2f75c5e342393e4430201eaa3dc1cbdf6 Mon Sep 17 00:00:00 2001 From: jason-p-pickering Date: Thu, 30 Nov 2023 11:29:35 +0100 Subject: [PATCH 3/9] Remove public class declaration --- .../DataIntegrityOrganisationUnitsNoGroupsControllerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java index a261b7aabe37..6b23dd010a4b 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java @@ -38,7 +38,7 @@ * * @author Jason P. Pickering */ -private class DataIntegrityOrganisationUnitsNoGroupsControllerTest +class DataIntegrityOrganisationUnitsNoGroupsControllerTest extends AbstractDataIntegrityIntegrationTest { private String orgunitA; From 6ca77a4744e46ded5320f846d5c90cf2cd3cef94 Mon Sep 17 00:00:00 2001 From: jason-p-pickering Date: Thu, 30 Nov 2023 11:31:32 +0100 Subject: [PATCH 4/9] Minor simplification. --- ...ganisationUnitsNoGroupsControllerTest.java | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java index 6b23dd010a4b..4fd60f5105a2 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java @@ -44,8 +44,6 @@ class DataIntegrityOrganisationUnitsNoGroupsControllerTest private String orgunitB; - private String testOrgUnitGroupA; - private static final String check = "orgunits_no_groups"; private static final String detailsIdType = "organisationUnits"; @@ -68,14 +66,13 @@ void testOrgunitsNoGroups() { "{ 'name': 'Pizza District', 'shortName': 'Pizza District', 'openingDate' : '2022-01-01'}")); // Create an orgunit group - testOrgUnitGroupA = - assertStatus( - HttpStatus.CREATED, - POST( - "/organisationUnitGroups", - "{'name': 'Type A', 'shortName': 'Type A', 'organisationUnits' : [{'id' : '" - + orgunitA - + "'}]}")); + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnitGroups", + "{'name': 'Type A', 'shortName': 'Type A', 'organisationUnits' : [{'id' : '" + + orgunitA + + "'}]}" ) ); assertHasDataIntegrityIssues( detailsIdType, check, 50, orgunitB, "Pizza District", "Type", true); } @@ -98,16 +95,15 @@ void testOrgunitsInGroups() { "{ 'name': 'Pizza District', 'shortName': 'Pizza District', 'openingDate' : '2022-01-01'}")); // Create an orgunit group - testOrgUnitGroupA = - assertStatus( - HttpStatus.CREATED, - POST( - "/organisationUnitGroups", - "{'name': 'Type A', 'shortName': 'Type A', 'organisationUnits' : [{'id' : '" - + orgunitA - + "'}, {'id' : '" - + orgunitB - + "'}]}")); + assertStatus( + HttpStatus.CREATED, + POST( + "/organisationUnitGroups", + "{'name': 'Type A', 'shortName': 'Type A', 'organisationUnits' : [{'id' : '" + + orgunitA + + "'}, {'id' : '" + + orgunitB + + "'}]}" ) ); assertHasNoDataIntegrityIssues(detailsIdType, check, true); } From a528b871c0a049e894aedd164a75668dac399e2c Mon Sep 17 00:00:00 2001 From: jason-p-pickering Date: Thu, 30 Nov 2023 11:39:54 +0100 Subject: [PATCH 5/9] Linting --- .../DataIntegrityOrganisationUnitsNoGroupsControllerTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java index 4fd60f5105a2..717154af0371 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java @@ -72,7 +72,7 @@ void testOrgunitsNoGroups() { "/organisationUnitGroups", "{'name': 'Type A', 'shortName': 'Type A', 'organisationUnits' : [{'id' : '" + orgunitA - + "'}]}" ) ); + + "'}]}")); assertHasDataIntegrityIssues( detailsIdType, check, 50, orgunitB, "Pizza District", "Type", true); } @@ -103,7 +103,7 @@ void testOrgunitsInGroups() { + orgunitA + "'}, {'id' : '" + orgunitB - + "'}]}" ) ); + + "'}]}")); assertHasNoDataIntegrityIssues(detailsIdType, check, true); } From e8b60900a0c60f33c4c11dc3d1386835c20856a5 Mon Sep 17 00:00:00 2001 From: jason-p-pickering Date: Thu, 30 Nov 2023 12:06:36 +0100 Subject: [PATCH 6/9] Fix legacy report --- .../org/hisp/dhis/dataintegrity/DataIntegrityCheckType.java | 1 - .../hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheckType.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheckType.java index ceec5e3e8fba..f863bae673a0 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheckType.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/DataIntegrityCheckType.java @@ -64,7 +64,6 @@ public enum DataIntegrityCheckType { // OrganisationUnits ORG_UNITS_WITH_CYCLIC_REFERENCES, ORG_UNITS_BEING_ORPHANED, - ORG_UNITS_WITHOUT_GROUPS, ORG_UNITS_VIOLATING_EXCLUSIVE_GROUP_SETS, ORG_UNIT_GROUPS_WITHOUT_GROUP_SETS, diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java index 0b6fa378afff..1aa1ba35ed4a 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java @@ -143,9 +143,10 @@ public FlattenedDataIntegrityReport(Map detailsByN this.orphanedOrganisationUnits = listOfDisplayNameOrUid( detailsByName.get(DataIntegrityCheckType.ORG_UNITS_BEING_ORPHANED.getName())); + //Replaced with SQL based equivalent this.organisationUnitsWithoutGroups = listOfDisplayNameOrUid( - detailsByName.get(DataIntegrityCheckType.ORG_UNITS_WITHOUT_GROUPS.getName())); + detailsByName.get("orgunits_no_groups")); this.organisationUnitGroupsWithoutGroupSets = listOfDisplayNameOrUid( detailsByName.get(DataIntegrityCheckType.ORG_UNIT_GROUPS_WITHOUT_GROUP_SETS.getName())); From 03c8b34aa534befb6b9a38e3c53908b7a29c692a Mon Sep 17 00:00:00 2001 From: jason-p-pickering Date: Thu, 30 Nov 2023 12:11:17 +0100 Subject: [PATCH 7/9] Linting --- .../dhis/dataintegrity/FlattenedDataIntegrityReport.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java index 1aa1ba35ed4a..e088273024b2 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java @@ -143,10 +143,9 @@ public FlattenedDataIntegrityReport(Map detailsByN this.orphanedOrganisationUnits = listOfDisplayNameOrUid( detailsByName.get(DataIntegrityCheckType.ORG_UNITS_BEING_ORPHANED.getName())); - //Replaced with SQL based equivalent + // Replaced with SQL based equivalent this.organisationUnitsWithoutGroups = - listOfDisplayNameOrUid( - detailsByName.get("orgunits_no_groups")); + listOfDisplayNameOrUid(detailsByName.get("orgunits_no_groups")); this.organisationUnitGroupsWithoutGroupSets = listOfDisplayNameOrUid( detailsByName.get(DataIntegrityCheckType.ORG_UNIT_GROUPS_WITHOUT_GROUP_SETS.getName())); From d546f1ab457937bd4fa7ca6276a1e5907cb4e138 Mon Sep 17 00:00:00 2001 From: jason-p-pickering Date: Thu, 30 Nov 2023 13:12:51 +0100 Subject: [PATCH 8/9] Renaming --- .../hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java | 2 +- .../data-integrity-checks/orgunits/orgunits_no_groups.yaml | 2 +- .../hisp/dhis/webapi/json/domain/JsonDataIntegrityReport.java | 2 +- .../DataIntegrityOrganisationUnitsNoGroupsControllerTest.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java index e088273024b2..e1666433a92b 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java @@ -145,7 +145,7 @@ public FlattenedDataIntegrityReport(Map detailsByN detailsByName.get(DataIntegrityCheckType.ORG_UNITS_BEING_ORPHANED.getName())); // Replaced with SQL based equivalent this.organisationUnitsWithoutGroups = - listOfDisplayNameOrUid(detailsByName.get("orgunits_no_groups")); + listOfDisplayNameOrUid(detailsByName.get("organisation_units_without_groups")); this.organisationUnitGroupsWithoutGroupSets = listOfDisplayNameOrUid( detailsByName.get(DataIntegrityCheckType.ORG_UNIT_GROUPS_WITHOUT_GROUP_SETS.getName())); diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_no_groups.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_no_groups.yaml index 15c8cc522cb7..7e9c535090b2 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_no_groups.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/orgunits/orgunits_no_groups.yaml @@ -25,7 +25,7 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --- -name: orgunits_no_groups +name: organisation_units_without_groups description: Organisation units with no groups. section: Organisation units section_order: 13 diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityReport.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityReport.java index b07d4bcd7dde..fdb0f42b4278 100644 --- a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityReport.java +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityReport.java @@ -102,7 +102,7 @@ default JsonList getOrphanedOrganisationUnits() { } default JsonList getOrganisationUnitsWithoutGroups() { - return getList("organisationUnitsWithoutGroups", JsonString.class); + return getList("organisation_units_without_groups", JsonString.class); } default JsonMultiMap getOrganisationUnitsViolatingExclusiveGroupSets() { diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java index 717154af0371..688e168f31cb 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityOrganisationUnitsNoGroupsControllerTest.java @@ -44,7 +44,7 @@ class DataIntegrityOrganisationUnitsNoGroupsControllerTest private String orgunitB; - private static final String check = "orgunits_no_groups"; + private static final String check = "organisation_units_without_groups"; private static final String detailsIdType = "organisationUnits"; From 801e095aa2d33b1c074ff87f16e9cffe3bb752dd Mon Sep 17 00:00:00 2001 From: jason-p-pickering Date: Thu, 30 Nov 2023 14:26:34 +0100 Subject: [PATCH 9/9] Fix issues with legacy report --- .../dataintegrity/FlattenedDataIntegrityReport.java | 10 +++++++++- .../dataintegrity/DefaultDataIntegrityService.java | 4 +++- .../webapi/json/domain/JsonDataIntegrityReport.java | 2 +- .../DataIntegrityReportControllerTest.java | 6 +++--- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java index e1666433a92b..92c776b4cb5d 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataintegrity/FlattenedDataIntegrityReport.java @@ -145,7 +145,7 @@ public FlattenedDataIntegrityReport(Map detailsByN detailsByName.get(DataIntegrityCheckType.ORG_UNITS_BEING_ORPHANED.getName())); // Replaced with SQL based equivalent this.organisationUnitsWithoutGroups = - listOfDisplayNameOrUid(detailsByName.get("organisation_units_without_groups")); + listOfDisplayNameWithUid(detailsByName.get("organisation_units_without_groups")); this.organisationUnitGroupsWithoutGroupSets = listOfDisplayNameOrUid( detailsByName.get(DataIntegrityCheckType.ORG_UNIT_GROUPS_WITHOUT_GROUP_SETS.getName())); @@ -255,6 +255,14 @@ private List listOfDisplayNameOrUid(DataIntegrityDetails details) { .collect(toUnmodifiableList()); } + private List listOfDisplayNameWithUid(DataIntegrityDetails details) { + return details == null + ? null + : details.getIssues().stream() + .map(issue -> issue.getName() + ":" + issue.getId()) + .collect(toUnmodifiableList()); + } + private List> groupedListOfDisplayNameOrUid(DataIntegrityDetails details) { return details == null ? null diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java index 90731b5d7d4f..fd93a2fc7d09 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataintegrity/DefaultDataIntegrityService.java @@ -688,7 +688,9 @@ public FlattenedDataIntegrityReport getReport(Set checks, JobProgress pr checks = Arrays.stream(DataIntegrityCheckType.values()) .map(DataIntegrityCheckType::getName) - .collect(toUnmodifiableSet()); + .collect(Collectors.toSet()); + // Add additional SQL based checks here + checks.add("organisation_units_without_groups"); } runDetailsChecks(checks, progress); return new FlattenedDataIntegrityReport(getDetails(checks, -1L)); diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityReport.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityReport.java index fdb0f42b4278..b07d4bcd7dde 100644 --- a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityReport.java +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/webapi/json/domain/JsonDataIntegrityReport.java @@ -102,7 +102,7 @@ default JsonList getOrphanedOrganisationUnits() { } default JsonList getOrganisationUnitsWithoutGroups() { - return getList("organisation_units_without_groups", JsonString.class); + return getList("organisationUnitsWithoutGroups", JsonString.class); } default JsonMultiMap getOrganisationUnitsViolatingExclusiveGroupSets() { diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityReportControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityReportControllerTest.java index 173b20daee91..5da6e59a4b12 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityReportControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityReportControllerTest.java @@ -118,9 +118,9 @@ void testOrganisationUnitsWithoutGroups() { String ouId = addOrganisationUnit("noGroupSet"); // should not match: addOrganisationUnitGroup("group", addOrganisationUnit("hasGroupSet")); - assertEquals( - singletonList("noGroupSet:" + ouId), - getDataIntegrityReport().getOrganisationUnitsWithoutGroups().toList(JsonString::string)); + List results = + getDataIntegrityReport().getOrganisationUnitsWithoutGroups().toList(JsonString::string); + assertEquals(singletonList("noGroupSet:" + ouId), results); } @Test