Merge remote-tracking branch 'origin/master' into TECH-1664-paginatio…

…n-to-store
dhis2 · Oct 26, 2023 · 3d68bce · 3d68bce
2 parents 96689f1 + b510f69
commit 3d68bce
Show file tree

Hide file tree

Showing 28 changed files with 403 additions and 287 deletions.
diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfigurationService.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobConfigurationService.java
@@ -150,14 +150,11 @@ String create(JobConfiguration config, MimeType contentType, InputStream content
    * Get all job configurations that should start within the next n seconds.
    *
    * @param dueInNextSeconds number of seconds from now the job should start
-   * @param limitToNext1 true, to only return a single config per {@link JobType}, false to return
-   *     all due jobs
    * @param includeWaiting true to also list jobs that cannot run because another job of the same
    *     type is already running
    * @return only jobs that should start soon within the given number of seconds
    */
-  List<JobConfiguration> getDueJobConfigurations(
-      int dueInNextSeconds, boolean limitToNext1, boolean includeWaiting);
+  List<JobConfiguration> getDueJobConfigurations(int dueInNextSeconds, boolean includeWaiting);
 
   /**
    * Finds stale jobs.

diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java
@@ -183,6 +183,10 @@ static Defaults dailyRandomBetween3and5(String uid, String name) {
     this.defaults = defaults;
   }
 
+  /**
+   * @return true, if {@link JobProgress} events should be forwarded to the {@link
+   *     org.eclipse.emf.common.notify.Notifier} API, otherwise false
+   */
   public boolean isUsingNotifications() {
     return this == RESOURCE_TABLE
         || this == SEND_SCHEDULED_MESSAGE
@@ -204,6 +208,10 @@ public boolean isUsingNotifications() {
         || this == GEOJSON_IMPORT;
   }
 
+  /**
+   * @return true, when an error notification should be sent by email in case the job execution
+   *     fails, otherwise false
+   */
   public boolean isUsingErrorNotification() {
     return this == ANALYTICS_TABLE
         || this == VALIDATION_RESULTS_NOTIFICATION
@@ -216,6 +224,15 @@ public boolean isUsingErrorNotification() {
         || this == METADATA_IMPORT;
   }
 
+  /**
+   * @return true, if jobs of this type should try to run as soon as possible by having job
+   *     scheduler workers execute all known ready jobs of the type, when false only the oldest of
+   *     the ready jobs per type is attempted to start in a single loop cycle
+   */
+  public boolean isUsingContinuousExecution() {
+    return this == METADATA_IMPORT;
+  }
+
   public boolean hasJobParameters() {
     return jobParameters != null;
   }

diff --git a/...administration/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java b/...administration/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java
@@ -30,6 +30,8 @@
 import static java.time.temporal.ChronoUnit.YEARS;
 import static java.util.Comparator.reverseOrder;
 import static java.util.stream.Collectors.toList;
+import static org.hisp.dhis.period.PeriodDataProvider.DataSource.DATABASE;
+import static org.hisp.dhis.period.PeriodDataProvider.DataSource.SYSTEM_DEFINED;
 import static org.hisp.dhis.scheduling.JobProgress.FailurePolicy.SKIP_ITEM;
 
 import com.google.common.collect.Lists;
@@ -184,7 +186,9 @@ public void generateDataElementTable() {
   @Override
   @Transactional
   public void generateDatePeriodTable() {
-    List<Integer> availableYears = periodDataProvider.getAvailableYears();
+    List<Integer> availableYears =
+        periodDataProvider.getAvailableYears(
+            analyticsExportSettings.getMaxPeriodYearsOffset() == null ? SYSTEM_DEFINED : DATABASE);
     checkYearsOffset(availableYears);
 
     resourceTableStore.generateResourceTable(
@@ -203,30 +207,33 @@ public void generateDatePeriodTable() {
    * @param yearsToCheck the list of years to be checked.
    */
   private void checkYearsOffset(List<Integer> yearsToCheck) {
-    int maxYearsOffset = analyticsExportSettings.getMaxPeriodYearsOffset();
-    int minRangeAllowed = Year.now().minus(maxYearsOffset, YEARS).getValue();
-    int maxRangeAllowed = Year.now().plus(maxYearsOffset, YEARS).getValue();
-
-    boolean yearsOutOfRange =
-        yearsToCheck.stream().anyMatch(year -> year < minRangeAllowed || year > maxRangeAllowed);
-
-    if (yearsOutOfRange) {
-      String errorMessage = "Your database contains years out of the allowed offset.";
-      errorMessage +=
-          "\n Range of years allowed (based on your system settings and existing data): "
-              + yearsToCheck.stream()
-                  .filter(year -> year >= minRangeAllowed && year <= maxRangeAllowed)
-                  .toList()
-              + ".";
-      errorMessage +=
-          "\n Years out of range found: "
-              + yearsToCheck.stream()
-                  .filter(year -> year < minRangeAllowed || year > maxRangeAllowed)
-                  .toList()
-              + ".";
-
-      log.warn(errorMessage);
-      throw new RuntimeException(errorMessage);
+    Integer maxYearsOffset = analyticsExportSettings.getMaxPeriodYearsOffset();
+
+    if (maxYearsOffset != null) {
+      int minRangeAllowed = Year.now().minus(maxYearsOffset, YEARS).getValue();
+      int maxRangeAllowed = Year.now().plus(maxYearsOffset, YEARS).getValue();
+
+      boolean yearsOutOfRange =
+          yearsToCheck.stream().anyMatch(year -> year < minRangeAllowed || year > maxRangeAllowed);
+
+      if (yearsOutOfRange) {
+        String errorMessage = "Your database contains years out of the allowed offset.";
+        errorMessage +=
+            "\n Range of years allowed (based on your system settings and existing data): "
+                + yearsToCheck.stream()
+                    .filter(year -> year >= minRangeAllowed && year <= maxRangeAllowed)
+                    .toList()
+                + ".";
+        errorMessage +=
+            "\n Years out of range found: "
+                + yearsToCheck.stream()
+                    .filter(year -> year < minRangeAllowed || year > maxRangeAllowed)
+                    .toList()
+                + ".";
+
+        log.warn(errorMessage);
+        throw new RuntimeException(errorMessage);
+      }
     }
   }
 

diff --git a/...nistration/src/test/java/org/hisp/dhis/resourcetable/DefaultResourceTableServiceTest.java b/...nistration/src/test/java/org/hisp/dhis/resourcetable/DefaultResourceTableServiceTest.java
@@ -28,6 +28,7 @@
 package org.hisp.dhis.resourcetable;
 
 import static java.time.temporal.ChronoUnit.YEARS;
+import static org.hisp.dhis.period.PeriodDataProvider.DataSource.DATABASE;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -63,7 +64,7 @@ void generateDatePeriodTableWhenYearIsOutOfRange() {
     int defaultOffset = 22;
 
     // When
-    when(periodDataProvider.getAvailableYears()).thenReturn(yearsToCheck);
+    when(periodDataProvider.getAvailableYears(DATABASE)).thenReturn(yearsToCheck);
     when(analyticsExportSettings.getMaxPeriodYearsOffset()).thenReturn(defaultOffset);
 
     // Then
@@ -82,7 +83,7 @@ void generateDatePeriodTableWhenOffsetIsZeroWithPreviousYears() {
     int zeroOffset = 0;
 
     // When
-    when(periodDataProvider.getAvailableYears()).thenReturn(yearsToCheck);
+    when(periodDataProvider.getAvailableYears(DATABASE)).thenReturn(yearsToCheck);
     when(analyticsExportSettings.getMaxPeriodYearsOffset()).thenReturn(zeroOffset);
 
     // Then
@@ -101,7 +102,7 @@ void generateDatePeriodTableWhenOffsetIsZeroWithCurrentYear() {
     int zeroOffset = 0;
 
     // When
-    when(periodDataProvider.getAvailableYears()).thenReturn(yearsToCheck);
+    when(periodDataProvider.getAvailableYears(DATABASE)).thenReturn(yearsToCheck);
     when(analyticsExportSettings.getMaxPeriodYearsOffset()).thenReturn(zeroOffset);
     doNothing().when(resourceTableStore).generateResourceTable(any());
 
@@ -120,7 +121,7 @@ void generateDatePeriodTableWhenYearsAreInExpectedRange() {
     int defaultOffset = 2;
 
     // When
-    when(periodDataProvider.getAvailableYears()).thenReturn(yearsToCheck);
+    when(periodDataProvider.getAvailableYears(DATABASE)).thenReturn(yearsToCheck);
     when(analyticsExportSettings.getMaxPeriodYearsOffset()).thenReturn(defaultOffset);
     doNothing().when(resourceTableStore).generateResourceTable(any());
 

diff --git a/...analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/...analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java
@@ -43,6 +43,8 @@
 import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quote;
 import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType;
 import static org.hisp.dhis.analytics.util.DisplayNameUtils.getDisplayName;
+import static org.hisp.dhis.period.PeriodDataProvider.DataSource.DATABASE;
+import static org.hisp.dhis.period.PeriodDataProvider.DataSource.SYSTEM_DEFINED;
 import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP;
 import static org.hisp.dhis.util.DateUtils.getLongDateString;
 
@@ -242,7 +244,9 @@ public List<AnalyticsTable> getAnalyticsTables(AnalyticsTableUpdateParams params
             "Get tables using earliest: %s, spatial support: %b",
             params.getFromDate(), databaseInfo.isSpatialSupport()));
 
-    List<Integer> availableDataYears = periodDataProvider.getAvailableYears();
+    List<Integer> availableDataYears =
+        periodDataProvider.getAvailableYears(
+            analyticsExportSettings.getMaxPeriodYearsOffset() == null ? SYSTEM_DEFINED : DATABASE);
 
     return params.isLatestUpdate()
         ? getLatestAnalyticsTables(params)
@@ -440,7 +444,9 @@ protected List<String> getPartitionChecks(AnalyticsTablePartition partition) {
   @Override
   protected void populateTable(
       AnalyticsTableUpdateParams params, AnalyticsTablePartition partition) {
-    List<Integer> availableDataYears = periodDataProvider.getAvailableYears();
+    List<Integer> availableDataYears =
+        periodDataProvider.getAvailableYears(
+            analyticsExportSettings.getMaxPeriodYearsOffset() == null ? SYSTEM_DEFINED : DATABASE);
     Integer firstDataYear = availableDataYears.get(0);
     Integer latestDataYear = availableDataYears.get(availableDataYears.size() - 1);
 

diff --git a/...ytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEventsAnalyticsTableManager.java b/...ytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEventsAnalyticsTableManager.java
@@ -49,6 +49,8 @@
 import static org.hisp.dhis.analytics.table.PartitionUtils.getStartDate;
 import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quote;
 import static org.hisp.dhis.commons.util.TextUtils.removeLastComma;
+import static org.hisp.dhis.period.PeriodDataProvider.DataSource.DATABASE;
+import static org.hisp.dhis.period.PeriodDataProvider.DataSource.SYSTEM_DEFINED;
 import static org.hisp.dhis.util.DateUtils.getLongDateString;
 import static org.hisp.dhis.util.DateUtils.getMediumDateString;
 import static org.springframework.util.Assert.notNull;
@@ -229,7 +231,9 @@ private List<Integer> getDataYears(AnalyticsTableUpdateParams params, TrackedEnt
               + "'");
     }
 
-    List<Integer> availableDataYears = periodDataProvider.getAvailableYears();
+    List<Integer> availableDataYears =
+        periodDataProvider.getAvailableYears(
+            analyticsExportSettings.getMaxPeriodYearsOffset() == null ? SYSTEM_DEFINED : DATABASE);
     Integer firstDataYear = availableDataYears.get(0);
     Integer latestDataYear = availableDataYears.get(availableDataYears.size() - 1);
 

diff --git a/...ytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java b/...ytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java
@@ -50,6 +50,7 @@
 import static org.hisp.dhis.analytics.ColumnDataType.INTEGER;
 import static org.hisp.dhis.analytics.ColumnDataType.TEXT;
 import static org.hisp.dhis.analytics.ColumnDataType.TIMESTAMP;
+import static org.hisp.dhis.period.PeriodDataProvider.DataSource.DATABASE;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
@@ -136,7 +137,7 @@ class JdbcEventAnalyticsTableManagerTest {
 
   @Mock private PeriodDataProvider periodDataProvider;
 
-  private AnalyticsExportSettings analyticsExportSettings;
+  @Mock private AnalyticsExportSettings analyticsExportSettings;
 
   private JdbcEventAnalyticsTableManager subject;
 
@@ -259,9 +260,10 @@ void verifyGetTableWithCategoryCombo() {
     addCategoryCombo(program, categoryCombo);
 
     when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(program));
-    when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear()));
+    when(periodDataProvider.getAvailableYears(DATABASE))
+        .thenReturn(List.of(2018, 2019, now().getYear()));
 
-    List<Integer> availableDataYears = periodDataProvider.getAvailableYears();
+    List<Integer> availableDataYears = periodDataProvider.getAvailableYears(DATABASE);
 
     when(jdbcTemplate.queryForList(
             getYearQueryForCurrentYear(program, true, availableDataYears), Integer.class))
@@ -294,9 +296,10 @@ void verifyGetTableWithCategoryCombo() {
   void verifyClientSideTimestampsColumns() {
     Program program = createProgram('A');
     when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(program));
-    when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear()));
+    when(periodDataProvider.getAvailableYears(DATABASE))
+        .thenReturn(List.of(2018, 2019, now().getYear()));
 
-    List<Integer> availableDataYears = periodDataProvider.getAvailableYears();
+    List<Integer> availableDataYears = periodDataProvider.getAvailableYears(DATABASE);
 
     when(jdbcTemplate.queryForList(
             getYearQueryForCurrentYear(program, true, availableDataYears), Integer.class))
@@ -330,9 +333,10 @@ void verifyClientSideTimestampsColumns() {
   void verifyAnalyticsEventTableHasDefaultPartition() {
     Program program = createProgram('A');
     when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(program));
-    when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2021, 2022, 2023, 2024, 2025));
+    when(periodDataProvider.getAvailableYears(DATABASE))
+        .thenReturn(List.of(2021, 2022, 2023, 2024, 2025));
 
-    List<Integer> availableDataYears = periodDataProvider.getAvailableYears();
+    List<Integer> availableDataYears = periodDataProvider.getAvailableYears(DATABASE);
 
     when(jdbcTemplate.queryForList(
             getYearQueryForCurrentYear(program, true, availableDataYears), Integer.class))
@@ -438,9 +442,10 @@ void verifyGetTableWithDataElements() {
             .withToday(today)
             .build();
 
-    when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear()));
+    when(periodDataProvider.getAvailableYears(DATABASE))
+        .thenReturn(List.of(2018, 2019, now().getYear()));
 
-    List<Integer> availableDataYears = periodDataProvider.getAvailableYears();
+    List<Integer> availableDataYears = periodDataProvider.getAvailableYears(DATABASE);
 
     when(jdbcTemplate.queryForList(
             getYearQueryForCurrentYear(program, true, availableDataYears), Integer.class))
@@ -506,9 +511,10 @@ void verifyGetTableWithTrackedEntityAttribute() {
             .withToday(today)
             .build();
 
-    when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear()));
+    when(periodDataProvider.getAvailableYears(DATABASE))
+        .thenReturn(List.of(2018, 2019, now().getYear()));
 
-    List<Integer> availableDataYears = periodDataProvider.getAvailableYears();
+    List<Integer> availableDataYears = periodDataProvider.getAvailableYears(DATABASE);
 
     when(jdbcTemplate.queryForList(
             getYearQueryForCurrentYear(program, true, availableDataYears), Integer.class))
@@ -562,9 +568,10 @@ void verifyDataElementTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable(
             .withToday(today)
             .build();
 
-    when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear()));
+    when(periodDataProvider.getAvailableYears(DATABASE))
+        .thenReturn(List.of(2018, 2019, now().getYear()));
 
-    List<Integer> availableDataYears = periodDataProvider.getAvailableYears();
+    List<Integer> availableDataYears = periodDataProvider.getAvailableYears(DATABASE);
 
     when(jdbcTemplate.queryForList(
             getYearQueryForCurrentYear(programA, true, availableDataYears), Integer.class))
@@ -603,9 +610,10 @@ void verifyTeiTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() {
     programA.setProgramAttributes(List.of(programTrackedEntityAttribute));
 
     when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(programA));
-    when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear()));
+    when(periodDataProvider.getAvailableYears(DATABASE))
+        .thenReturn(List.of(2018, 2019, now().getYear()));
 
-    List<Integer> availableDataYears = periodDataProvider.getAvailableYears();
+    List<Integer> availableDataYears = periodDataProvider.getAvailableYears(DATABASE);
 
     AnalyticsTableUpdateParams params =
         AnalyticsTableUpdateParams.newBuilder()
@@ -650,9 +658,10 @@ void verifyOrgUnitOwnershipJoinsWhenPopulatingEventAnalyticsTable() {
     programA.setProgramAttributes(List.of(programTrackedEntityAttribute));
 
     when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(programA));
-    when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear()));
+    when(periodDataProvider.getAvailableYears(DATABASE))
+        .thenReturn(List.of(2018, 2019, now().getYear()));
 
-    List<Integer> availableDataYears = periodDataProvider.getAvailableYears();
+    List<Integer> availableDataYears = periodDataProvider.getAvailableYears(DATABASE);
 
     AnalyticsTableUpdateParams params =
         AnalyticsTableUpdateParams.newBuilder()
@@ -688,9 +697,10 @@ void verifyGetAnalyticsTableWithOuLevels() {
     Program programA = rnd.nextObject(Program.class);
     programA.setId(0);
 
-    when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear()));
+    when(periodDataProvider.getAvailableYears(DATABASE))
+        .thenReturn(List.of(2018, 2019, now().getYear()));
 
-    List<Integer> availableDataYears = periodDataProvider.getAvailableYears();
+    List<Integer> availableDataYears = periodDataProvider.getAvailableYears(DATABASE);
     int startYear = availableDataYears.get(0);
     int latestYear = availableDataYears.get(availableDataYears.size() - 1);
 
@@ -751,9 +761,10 @@ void verifyGetAnalyticsTableWithOuGroupSet() {
     when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(programA));
     when(idObjectManager.getDataDimensionsNoAcl(OrganisationUnitGroupSet.class))
         .thenReturn(ouGroupSet);
-    when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear()));
+    when(periodDataProvider.getAvailableYears(DATABASE))
+        .thenReturn(List.of(2018, 2019, now().getYear()));
 
-    List<Integer> availableDataYears = periodDataProvider.getAvailableYears();
+    List<Integer> availableDataYears = periodDataProvider.getAvailableYears(DATABASE);
 
     AnalyticsTableUpdateParams params =
         AnalyticsTableUpdateParams.newBuilder().withStartTime(START_TIME).build();
@@ -791,9 +802,10 @@ void verifyGetAnalyticsTableWithOptionGroupSets() {
 
     when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(programA));
     when(categoryService.getAttributeCategoryOptionGroupSetsNoAcl()).thenReturn(cogs);
-    when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear()));
+    when(periodDataProvider.getAvailableYears(DATABASE))
+        .thenReturn(List.of(2018, 2019, now().getYear()));
 
-    List<Integer> availableDataYears = periodDataProvider.getAvailableYears();
+    List<Integer> availableDataYears = periodDataProvider.getAvailableYears(DATABASE);
 
     when(jdbcTemplate.queryForList(
             getYearQueryForCurrentYear(programA, false, availableDataYears), Integer.class))
@@ -870,9 +882,10 @@ void verifyTeaTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() {
 
     programA.setProgramAttributes(List.of(programTrackedEntityAttribute));
 
-    when(periodDataProvider.getAvailableYears()).thenReturn(List.of(2018, 2019, now().getYear()));
+    when(periodDataProvider.getAvailableYears(DATABASE))
+        .thenReturn(List.of(2018, 2019, now().getYear()));
 
-    List<Integer> availableDataYears = periodDataProvider.getAvailableYears();
+    List<Integer> availableDataYears = periodDataProvider.getAvailableYears(DATABASE);
     int startYear = availableDataYears.get(0);
     int latestYear = availableDataYears.get(availableDataYears.size() - 1);