Merge branch 'master' into 2.41-DHIS2-16065

# Conflicts:
#	dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/ResourceTableController.java

dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/TimeField.java

@@ -44,7 +44,7 @@ public enum TimeField {
   ENROLLMENT_DATE("enrollmentdate"),
   INCIDENT_DATE("incidentdate"),
   // Not a typo, different naming convention between FE and database
-  SCHEDULED_DATE("duedate"),
+  SCHEDULED_DATE("scheduleddate"),
   COMPLETED_DATE("completeddate"),
   CREATED("created"),
   LAST_UPDATED("lastupdated");

dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/AnalyticsPeriodBoundary.java

@@ -98,7 +98,7 @@ public class AnalyticsPeriodBoundary extends BaseIdentifiableObject implements E
 
   public static final String DB_INCIDENT_DATE = "incidentdate";
 
-  public static final String DB_SCHEDULED_DATE = "duedate";
+  public static final String DB_SCHEDULED_DATE = "scheduleddate";
 
   public static final String DB_QUOTE = "\"";
 

dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/Enrollment.java

@@ -71,7 +71,7 @@ public class Enrollment extends SoftDeletableObject {
 
   private Date enrollmentDate;
 
-  private Date endDate;
+  private Date completedDate;
 
   private UserInfoSnapshot createdByUserInfo;
 
@@ -238,12 +238,12 @@ public void setEnrollmentDate(Date enrollmentDate) {
 
   @JsonProperty
   @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
-  public Date getEndDate() {
-    return endDate;
+  public Date getCompletedDate() {
+    return completedDate;
   }
 
-  public void setEndDate(Date endDate) {
-    this.endDate = endDate;
+  public void setCompletedDate(Date completedDate) {
+    this.completedDate = completedDate;
   }
 
   @JsonProperty
@@ -434,7 +434,7 @@ private static void setShallowCopyValues(
     copy.setCompletedBy(original.getCompletedBy());
     copy.setCreatedAtClient(original.getCreatedAtClient());
     copy.setCreatedByUserInfo(original.getCreatedByUserInfo());
-    copy.setEndDate(original.getEndDate());
+    copy.setCompletedDate(original.getCompletedDate());
     copy.setEnrollmentDate(original.getEnrollmentDate());
     copy.setEvents(new HashSet<>());
     copy.setFollowup(original.getFollowup());

dhis-2/dhis-api/src/test/java/org/hisp/dhis/program/EnrollmentTest.java

@@ -68,7 +68,7 @@ void testCopyOfWithPropertyValuesSet() {
     assertEquals(original.getFollowup(), copy.getFollowup());
     assertEquals(original.getGeometry(), copy.getGeometry());
     assertEquals(original.getOrganisationUnit(), copy.getOrganisationUnit());
-    assertEquals(original.getEndDate(), copy.getEndDate());
+    assertEquals(original.getCompletedDate(), copy.getCompletedDate());
     assertEquals(original.getRelationshipItems(), copy.getRelationshipItems());
     assertEquals(original.getCreatedByUserInfo(), copy.getCreatedByUserInfo());
     assertEquals(original.getMessageConversations(), copy.getMessageConversations());
@@ -86,7 +86,7 @@ void testCopyOfWithNullPropertyValues() {
     assertNotEquals(original.getProgram(), copy.getProgram());
 
     assertEquals(original.getCreatedByUserInfo(), copy.getCreatedByUserInfo());
-    assertEquals(original.getEndDate(), copy.getEndDate());
+    assertEquals(original.getCompletedDate(), copy.getCompletedDate());
     assertEquals(original.getEnrollmentDate(), copy.getEnrollmentDate());
     assertEquals(original.getFollowup(), copy.getFollowup());
     assertEquals(original.getGeometry(), copy.getGeometry());
@@ -117,7 +117,7 @@ private Enrollment getNewEnrollmentWithNoNulls() {
     e.setAutoFields();
     e.setNotes(List.of(new Note("note", "amin")));
     e.setCompletedBy("admin");
-    e.setEndDate(new Date());
+    e.setCompletedDate(new Date());
     e.setEnrollmentDate(new Date());
     e.setEvents(Set.of());
     e.setFollowup(true);
@@ -138,7 +138,7 @@ private Enrollment getNewEnrollmentWithNulls() {
     e.setName(null);
     e.setNotes(null);
     e.setCompletedBy(null);
-    e.setEndDate(null);
+    e.setCompletedDate(null);
     e.setEnrollmentDate(null);
     e.setEvents(null);
     e.setIncidentDate(null);

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventDataQueryService.java

@@ -124,7 +124,7 @@ public class DefaultEventDataQueryService implements EventDataQueryService {
 
   private static final String COL_NAME_INCIDENTDATE = "incidentdate";
 
-  private static final String COL_NAME_DUEDATE = "duedate";
+  private static final String COL_NAME_DUEDATE = "scheduleddate";
 
   private final ProgramService programService;
 

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java

@@ -553,7 +553,7 @@ protected String getColumn(QueryItem item, String suffix) {
           && !item.getRepeatableStageParams().simpleStageValueExpected()) {
         return "(select json_agg(t1) from (select "
             + colName
-            + ", incidentdate, duedate, executiondate "
+            + ", incidentdate, scheduleddate, executiondate "
             + " from "
             + eventTableName
             + " where "

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java

@@ -324,7 +324,7 @@ protected String getSelectClause(EventQueryParams params) {
                 "createdbydisplayname",
                 "lastupdatedbydisplayname",
                 "lastupdated",
-                "duedate");
+                "scheduleddate");
 
     if (params.getProgram().isRegistration()) {
       cols.add("enrollmentdate", "incidentdate", "tei", "pi");

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java

@@ -142,7 +142,7 @@ public JdbcEventAnalyticsTableManager(
           new AnalyticsTableColumn(quote("enrollmentdate"), TIMESTAMP, "pi.enrollmentdate"),
           new AnalyticsTableColumn(quote("incidentdate"), TIMESTAMP, "pi.incidentdate"),
           new AnalyticsTableColumn(quote("executiondate"), TIMESTAMP, "psi.executiondate"),
-          new AnalyticsTableColumn(quote("duedate"), TIMESTAMP, "psi.duedate"),
+          new AnalyticsTableColumn(quote("scheduleddate"), TIMESTAMP, "psi.scheduleddate"),
           new AnalyticsTableColumn(quote("completeddate"), TIMESTAMP, "psi.completeddate"),
 
           /*
@@ -262,7 +262,7 @@ public List<AnalyticsTable> getAnalyticsTables(AnalyticsTableUpdateParams params
    *     status
    */
   static String getDateLinkedToStatus() {
-    return "CASE WHEN 'SCHEDULE' = psi.status THEN psi.duedate ELSE psi.executiondate END";
+    return "CASE WHEN 'SCHEDULE' = psi.status THEN psi.scheduleddate ELSE psi.executiondate END";
   }
 
   /**

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEventsAnalyticsTableManager.java

@@ -137,7 +137,7 @@ public JdbcTeiEventsAnalyticsTableManager(
           new AnalyticsTableColumn(quote("executiondate"), TIMESTAMP, "psi.executiondate"),
           new AnalyticsTableColumn(quote("lastupdated"), TIMESTAMP, "psi.lastupdated"),
           new AnalyticsTableColumn(quote("created"), TIMESTAMP, "psi.created"),
-          new AnalyticsTableColumn(quote("duedate"), TIMESTAMP, "psi.duedate"),
+          new AnalyticsTableColumn(quote("scheduleddate"), TIMESTAMP, "psi.scheduleddate"),
           new AnalyticsTableColumn(quote("status"), VARCHAR_50, "psi.status"),
           new AnalyticsTableColumn(quote("psigeometry"), GEOMETRY, "psi.geometry")
               .withIndexType(GIST),

dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsManagerTest.java

@@ -744,7 +744,7 @@ void verifyGetColumnOfTypeCoordinateAndWithProgramStagesAndParamsWithReferenceTy
         is(
             "(select json_agg(t1) from (select \""
                 + dataElementA.getUid()
-                + "\", incidentdate, duedate, executiondate  from analytics_event_"
+                + "\", incidentdate, scheduleddate, executiondate  from analytics_event_"
                 + programB.getUid()
                 + " where analytics_event_"
                 + programB.getUid()

dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventAnalyticsManagerTest.java

@@ -119,7 +119,7 @@ class EventAnalyticsManagerTest extends EventAnalyticsTest {
           + "createdbydisplayname"
           + ","
           + "lastupdatedbydisplayname"
-          + ",lastupdated,duedate,enrollmentdate,incidentdate,tei,pi,ST_AsGeoJSON(coalesce(ax.\"psigeometry\",ax.\"pigeometry\",ax.\"teigeometry\",ax.\"ougeometry\"), 6) as geometry,longitude,latitude,ouname,ounamehierarchy,"
+          + ",lastupdated,scheduleddate,enrollmentdate,incidentdate,tei,pi,ST_AsGeoJSON(coalesce(ax.\"psigeometry\",ax.\"pigeometry\",ax.\"teigeometry\",ax.\"ougeometry\"), 6) as geometry,longitude,latitude,ouname,ounamehierarchy,"
           + "oucode,pistatus,psistatus";
 
   @BeforeEach
@@ -157,7 +157,7 @@ void verifyGetEventSqlWithProgramWithNoRegistration() {
             + "createdbydisplayname"
             + ","
             + "lastupdatedbydisplayname"
-            + ",lastupdated,duedate,ST_AsGeoJSON(coalesce(ax.\"psigeometry\",ax.\"pigeometry\",ax.\"teigeometry\",ax.\"ougeometry\"), 6) as geometry,"
+            + ",lastupdated,scheduleddate,ST_AsGeoJSON(coalesce(ax.\"psigeometry\",ax.\"pigeometry\",ax.\"teigeometry\",ax.\"ougeometry\"), 6) as geometry,"
             + "longitude,latitude,ouname,ounamehierarchy,oucode,pistatus,psistatus,ax.\"quarterly\",ax.\"ou\"  from "
             + getTable(programA.getUid())
             + " as ax where (ax.\"quarterly\" in ('2000Q1') ) and ax.\"uidlevel1\" in ('ouabcdefghA') limit 101";
@@ -218,7 +218,7 @@ void verifyGetEventSqlWithOrgUnitTypeDataElement() {
             + "createdbydisplayname"
             + ","
             + "lastupdatedbydisplayname"
-            + ",lastupdated,duedate,enrollmentdate,"
+            + ",lastupdated,scheduleddate,enrollmentdate,"
             + "incidentdate,tei,pi,ST_AsGeoJSON(coalesce(ax.\"psigeometry\",ax.\"pigeometry\",ax.\"teigeometry\",ax.\"ougeometry\"), 6) as geometry,longitude,latitude,ouname,ounamehierarchy,oucode,pistatus,"
             + "psistatus,ax.\"quarterly\",ax.\"ou\",\""
             + dataElement.getUid()
@@ -334,7 +334,7 @@ void verifyGetEventsWithScheduledDateTimeFieldParam() {
     String expected =
         "ps.\"quarterly\",ax.\"ou\"  from "
             + getTable(programA.getUid())
-            + " as ax left join _dateperiodstructure as ps on cast(ax.\"duedate\" as date) = ps.\"dateperiod\" "
+            + " as ax left join _dateperiodstructure as ps on cast(ax.\"scheduleddate\" as date) = ps.\"dateperiod\" "
             + "where (ps.\"quarterly\" in ('2000Q1') ) and ax.\"uidlevel1\" "
             + "in ('ouabcdefghA') and pistatus in ('ACTIVE','COMPLETED') limit 101";
 

dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/TimeFieldSqlRendererTest.java

@@ -201,8 +201,8 @@ void testRenderEnrollmentTimeFieldSqlWhenNoContinuousMonthlyPeriodWithTimeFieldA
     params = new EventQueryParams.Builder(params).withStartEndDatesForPeriods().build();
 
     assertEquals(
-        "((ax.\"duedate\" >= '2022-03-01' and ax.\"duedate\" < '2022-04-01') "
-            + "or (ax.\"duedate\" >= '2022-09-01' and ax.\"duedate\" < '2022-10-01')) ",
+        "((ax.\"scheduleddate\" >= '2022-03-01' and ax.\"scheduleddate\" < '2022-04-01') "
+            + "or (ax.\"scheduleddate\" >= '2022-09-01' and ax.\"scheduleddate\" < '2022-10-01')) ",
         timeFieldSqlRenderer.renderPeriodTimeFieldSql(params));
   }
 

dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java

@@ -976,6 +976,6 @@ private String getYearQueryForCurrentYear(
   }
 
   private String getDateLinkedToStatus() {
-    return "CASE WHEN 'SCHEDULE' = psi.status THEN psi.duedate ELSE psi.executiondate END";
+    return "CASE WHEN 'SCHEDULE' = psi.status THEN psi.scheduleddate ELSE psi.executiondate END";
   }
 }

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/period/PeriodDataProvider.java

@@ -118,7 +118,7 @@ void addSafetyBuffer(List<Integer> years, int buffer) {
    */
   private List<Integer> fetchAvailableYears() {
     String dueDateOrExecutionDate =
-        "(case when 'SCHEDULE' = ev.status then ev.duedate else ev.executiondate end)";
+        "(case when 'SCHEDULE' = ev.status then ev.scheduleddate else ev.executiondate end)";
 
     String sql =
         "( select distinct (extract(year from pe.startdate)) as datayear from period pe )"

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/DefaultEnrollmentService.java

@@ -472,7 +472,7 @@ public void cancelEnrollmentStatus(Enrollment enrollment) {
         // Set status as skipped for overdue events, or delete
         // -------------------------------------------------------------
 
-        if (event.getScheduledDate().before(enrollment.getEndDate())) {
+        if (event.getScheduledDate().before(enrollment.getCompletedDate())) {
           event.setStatus(EventStatus.SKIPPED);
           eventStore.update(event);
         } else {
@@ -501,7 +501,7 @@ public void incompleteEnrollmentStatus(Enrollment enrollment) {
     // -----------------------------------------------------------------
 
     enrollment.setStatus(ProgramStatus.ACTIVE);
-    enrollment.setEndDate(null);
+    enrollment.setCompletedDate(null);
 
     updateEnrollment(enrollment);
   }

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/variable/vDueDate.java

@@ -42,14 +42,14 @@ public Object getSql(CommonExpressionVisitor visitor) {
     ProgramExpressionParams params = visitor.getProgParams();
 
     if (AnalyticsType.EVENT == params.getProgramIndicator().getAnalyticsType()) {
-      return "duedate";
+      return "scheduleddate";
     }
 
     return visitor
         .getStatementBuilder()
         .getProgramIndicatorEventColumnSql(
             null,
-            "duedate",
+            "scheduleddate",
             params.getReportingStartDate(),
             params.getReportingEndDate(),
             params.getProgramIndicator());

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/variable/vScheduledDate.java

@@ -47,15 +47,15 @@ public Object getSql(CommonExpressionVisitor visitor) {
               .getStatementBuilder()
               .getProgramIndicatorEventColumnSql(
                   null,
-                  "duedate",
+                  "scheduleddate",
                   params.getReportingStartDate(),
                   params.getReportingEndDate(),
                   params.getProgramIndicator());
 
       return maybeAppendEventStatusFilterIntoWhere(sqlStatement);
     }
 
-    return "duedate";
+    return "scheduleddate";
   }
 
   private String maybeAppendEventStatusFilterIntoWhere(String sqlStatement) {

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerService.java

@@ -84,6 +84,12 @@ public void executeNow(@Nonnull String jobId) throws NotFoundException, Conflict
       if (job == null) throw new NotFoundException(JobConfiguration.class, jobId);
       // run "execute now" request directly when scheduling is not active (tests)
       jobRunner.runDueJob(job);
+    } else {
+      JobConfiguration job = jobConfigurationStore.getByUid(jobId);
+      if (job == null) throw new NotFoundException(JobConfiguration.class, jobId);
+      if (job.getJobType().isUsingContinuousExecution()) {
+        jobRunner.runIfDue(job);
+      }
     }
   }
 

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/JobRunner.java

@@ -40,7 +40,27 @@
  */
 public interface JobRunner {
 
+  /**
+   * During testing the scheduler might not be active in which case this is false. Otherwise, this
+   * should always be true in a production environment.
+   *
+   * @return true, if the scheduler is running a scheduling loop cycle, otherwise false
+   */
   boolean isScheduling();
 
+  /**
+   * Runs a job if it should now run according to its {@link SchedulingType} and related information
+   * like the CRON expression or the delay time.
+   *
+   * @param config the job to check and potentially run
+   */
+  void runIfDue(JobConfiguration config);
+
+  /**
+   * Manually runs a job. OBS! This bypasses any actual checking if the job is due to run. When this
+   * is called the job will run.
+   *
+   * @param config The job to run.
+   */
   void runDueJob(JobConfiguration config);
 }

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/JobScheduler.java

@@ -137,20 +137,27 @@ private void createHousekeepingJob() {
     }
   }
 
+  @Override
+  public void runIfDue(JobConfiguration job) {
+    runIfDue(Instant.now().truncatedTo(ChronoUnit.SECONDS), job.getJobType(), List.of(job));
+  }
+
   private void runIfDue(Instant now, JobType type, List<JobConfiguration> jobs) {
     if (!type.isUsingContinuousExecution()) {
       runIfDue(now, jobs.get(0));
       return;
     }
-    Queue<String> jobIds =
-        continuousJobsByType.computeIfAbsent(type, key -> new ConcurrentLinkedQueue<>());
-    // add a worker either if no worker is on it (empty new queue) or if there are many jobs
-    boolean spawnWorker = jobIds.isEmpty();
+    Queue<String> jobIds = continuousJobsByType.get(type);
+    boolean spawnWorker = false;
+    if (jobIds == null) {
+      Queue<String> localQueue = new ConcurrentLinkedQueue<>();
+      Queue<String> sharedQueue = continuousJobsByType.putIfAbsent(type, localQueue);
+      spawnWorker = sharedQueue == null; // no previous queue => this thread put the queue
+      jobIds = continuousJobsByType.get(type);
+    }
     // add those IDs to the queue that are not yet in it
-    jobs.stream()
-        .map(JobConfiguration::getUid)
-        .filter(jobId -> !jobIds.contains(jobId))
-        .forEach(jobIds::add);
+    jobs.stream().map(JobConfiguration::getUid).forEach(jobIds::add);
+
     if (spawnWorker) {
       // we want to prevent starting more than one worker per job type
       // but if this does happen it is no issue as both will be pulling

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/trackedentity/hibernate/HibernateTrackedEntityStore.java

@@ -116,7 +116,7 @@ public class HibernateTrackedEntityStore extends SoftDeleteHibernateObjectStore<
 
   private static final String EV_EXECUTIONDATE = "EV.executiondate";
 
-  private static final String EV_DUEDATE = "EV.duedate";
+  private static final String EV_DUEDATE = "EV.scheduleddate";
 
   private static final String IS_NULL = "IS NULL";
 
@@ -1035,7 +1035,7 @@ private String getFromSubQueryEvent(TrackedEntityQueryParams params) {
             .append(SPACE)
             .append(IS_NULL)
             .append(whereHlp.whereAnd())
-            .append("date(now()) <= date(EV.duedate) ");
+            .append("date(now()) <= date(EV.scheduleddate) ");
       } else if (params.isEventStatus(EventStatus.OVERDUE)) {
         events
             .append(getQueryDateConditionBetween(whereHlp, EV_DUEDATE, start, end))
@@ -1048,7 +1048,7 @@ private String getFromSubQueryEvent(TrackedEntityQueryParams params) {
             .append(SPACE)
             .append(IS_NULL)
             .append(whereHlp.whereAnd())
-            .append("date(now()) > date(EV.duedate) ");
+            .append("date(now()) > date(EV.scheduleddate) ");
       } else if (params.isEventStatus(EventStatus.SKIPPED)) {
         events
             .append(getQueryDateConditionBetween(whereHlp, EV_DUEDATE, start, end))

dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/program/hibernate/Enrollment.hbm.xml

@@ -32,7 +32,7 @@
 
     <property name="enrollmentDate" column="enrollmentdate" not-null="true"/>
 
-    <property name="endDate" column="enddate"/>
+    <property name="completedDate" column="enddate"/>
 
     <property name="followup" column="followup"/>
 
@@ -59,7 +59,7 @@
     <many-to-one name="program" class="org.hisp.dhis.program.Program"
       column="programid" not-null="true" foreign-key="fk_programinstance_programid" lazy="false" />
 
-    <set name="events" order-by="executionDate,duedate" inverse="true">
+    <set name="events" order-by="executionDate,scheduleddate" inverse="true">
       <key column="enrollmentid"/>
       <one-to-many class="org.hisp.dhis.program.Event"/>
     </set>

dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/program/hibernate/Event.hbm.xml

@@ -44,7 +44,7 @@
 
     <property name="storedBy" />
 
-    <property name="scheduledDate" column="duedate" />
+    <property name="scheduledDate" column="scheduleddate" />
 
     <property name="occurredDate" column="executiondate" type="timestamp" index="programstageinstance_executiondate" />