ecflow_python: enable Repeat based on Date+Time

Includes Python documentation.

Re ECFLOW-1900

Pyext/src/ecflow/python/DefsDoc.cpp

@@ -745,6 +745,25 @@ const char* DefsDoc::add_repeat_date_doc() {
            "            RepeatDate('YMD',20100111,20100115))\n";
 }
 
+const char* DefsDoc::add_repeat_datetime_doc() {
+    return "Add a RepeatDateTime attribute. See :py:class:`ecflow.RepeatDateTime`\n\n"
+           "A node can only have one repeat.\n"
+           "When a RepeatDateTime is used in a trigger expression, the arithmetic value of the Repeat decays to second.\n"
+           "For example, the expression `/suite/family:DATETIME + 1` is evaluated as the number of seconds represented "
+           "by `/suite/family:DT` (since the reference epoch, i.e. 19700101T000000) plus 1."
+           "The result is an integer.\n\n"
+           "   trigger /suite/family:DT + 1 > 123456\n"
+           "\nException:\n\n"
+           "- Throws a RuntimeError if more than one repeat is added\n"
+           "\nUsage:\n\n"
+           ".. code-block:: python\n\n"
+           "  t1 = Task('t1')\n"
+           "  t1.add_repeat(RepeatDateTime('DT', '20100111T120000', '20100115T000000', '12:00:00'))\n\n"
+           "  # we can also create a repeat in Task constructor like any other attribute\n"
+           "  t2 = Task('t2',\n"
+           "            RepeatDateTime('DT', '20100101T000000', '20100115T000000', '1:00:00'))\n";
+}
+
 const char* DefsDoc::add_repeat_date_list_doc() {
     return "Add a RepeatDateList attribute. See :py:class:`ecflow.RepeatDateList`\n\n"
            "A node can only have one repeat\n"

Pyext/src/ecflow/python/DefsDoc.hpp

@@ -59,6 +59,7 @@ class DefsDoc {
     static const char* add_autorestore_doc();
     static const char* add_verify_doc();
     static const char* add_repeat_date_doc();
+    static const char* add_repeat_datetime_doc();
     static const char* add_repeat_date_list_doc();
     static const char* add_repeat_integer_doc();
     static const char* add_repeat_string_doc();

Pyext/src/ecflow/python/ExportNode.cpp

@@ -258,6 +258,10 @@ node_ptr add_repeat_date(node_ptr self, const RepeatDate& d) {
     self->addRepeat(d);
     return self;
 }
+node_ptr add_repeat_datetime(node_ptr self, const RepeatDateTime& d) {
+    self->addRepeat(d);
+    return self;
+}
 node_ptr add_repeat_date_list(node_ptr self, const RepeatDateList& d) {
     self->addRepeat(d);
     return self;
@@ -613,6 +617,7 @@ void export_Node() {
         .def("add_autorestore", &add_autorestore1)
         .def("add_verify", &Node::addVerify, DefsDoc::add_verify_doc())
         .def("add_repeat", &add_repeat_date, DefsDoc::add_repeat_date_doc())
+        .def("add_repeat", &add_repeat_datetime, DefsDoc::add_repeat_datetime_doc())
         .def("add_repeat", &add_repeat_date_list, DefsDoc::add_repeat_date_list_doc())
         .def("add_repeat", &add_repeat_integer, DefsDoc::add_repeat_integer_doc())
         .def("add_repeat", &add_repeat_string, DefsDoc::add_repeat_string_doc())

Pyext/src/ecflow/python/ExportNodeAttr.cpp

@@ -824,6 +824,23 @@ void export_NodeAttr() {
              &RepeatDate::step,
              "Return the step increment. This is used to update the repeat, until end date is reached");
 
+    class_<RepeatDateTime>("RepeatDateTime",
+                           NodeAttrDoc::repeat_datetime_doc(),
+                           init<std::string, std::string, std::string, bp::optional<std::string>>()) // name, start, end , delta
+        .def(self == self)                                                             // __eq__
+        .def("__str__", &RepeatDateTime::toString)                                     // __str__
+        .def("__copy__", copyObject<RepeatDateTime>)                                   // __copy__ uses copy constructor
+        .def("name",
+             &RepeatDateTime::name,
+             return_value_policy<copy_const_reference>(),
+             "Return the name of the repeat.")
+        .def(
+            "start", &RepeatDateTime::start, "Return the start date as an integer (i.e. seconds since 19700101T000000)")
+        .def("end", &RepeatDateTime::end, "Return the end date as an integer (i.e. seconds since 19700101T000000)")
+        .def("step",
+             &RepeatDateTime::step,
+             "Return the step increment (in seconds). This is used to update the repeat, until end instant is reached");
+
     class_<RepeatDateList>("RepeatDateList", NodeAttrDoc::repeat_date_list_doc())
         .def("__init__", make_constructor(&create_RepeatDateList))
         .def(self == self)                           // __eq__

Pyext/src/ecflow/python/NodeAttrDoc.cpp

@@ -520,6 +520,32 @@ const char* NodeAttrDoc::repeat_date_doc() {
            "            RepeatDate('YMD', 20050130, 20050203 ))\n";
 }
 
+const char* NodeAttrDoc::repeat_datetime_doc() {
+    return "Allows a `node`_ to be repeated based on date+time instants (using yyyymmddTHHMMSS format).\n\n"
+           "A node can only have one `repeat`_.\n"
+           "The repeat name can be referenced in `trigger`_ expressions.\n"
+           "The trigger expression referencing a RepeatDate will use date arithmetic\n"
+           "Here (/suite/family:YMD + 1) uses date arithmetic only, the result is still an integer\n\n"
+           "   trigger /suite/family:YMD + 1 > 20190101\n"
+           "\nConstructor::\n\n"
+           "   RepeatDateTime(variable, start, end[, step])\n"
+           "      string variable:     The name of the repeat. The Repeat can referenced in\n"
+           "                           in trigger expressions using the variable name\n"
+           "      int start:           Start date+time instant, formatted as yyyymmddTHHMMSS\n"
+           "      int end:             End date+time instant, formatted as yyyymmddTHHMMSS\n"
+           "      int step:            The increment duration used to update the instant when iterating\n"
+           "                           Formatted as HH:MM.SS, if not provided default delta is '24:00:00'.\n"
+           "\nException:\n\n"
+           "- Throws a RuntimeError if start/end are not valid date+time instants\n"
+           "- Throws a RuntimeError if delta is not a valid duration\n"
+           "\nUsage:\n\n"
+           ".. code-block:: python\n\n"
+           "   rep = RepeatDateTime('DATETIME', '20050130T000000', '20050203T000000')\n"
+           "   rep = RepeatDateTime('DATETIME', '20050130T000000', '20050203T000000', '48:00:00')\n"
+           "   t = Task('t1',\n"
+           "            RepeatDateTime('DATETIME', '20050130T000000', '20050203T120000', '1:00:00'))\n";
+}
+
 const char* NodeAttrDoc::repeat_date_list_doc() {
     return "Allows a `node`_ to be repeated using arbitrary list of yyyymmdd integers\n\n"
            "A node can only have one `repeat`_.\n"

Pyext/src/ecflow/python/NodeAttrDoc.hpp

@@ -46,6 +46,7 @@ class NodeAttrDoc {
     static const char* autorestore_doc();
     static const char* repeat_doc();
     static const char* repeat_date_doc();
+    static const char* repeat_datetime_doc();
     static const char* repeat_date_list_doc();
     static const char* repeat_integer_doc();
     static const char* repeat_enumerated_doc();

Pyext/src/ecflow/python/NodeUtil.cpp

@@ -127,6 +127,8 @@ object NodeUtil::do_add(node_ptr self, const bp::object& arg) {
         self->addZombie(extract<ZombieAttr>(arg));
     else if (extract<RepeatDate>(arg).check())
         self->addRepeat(Repeat(extract<RepeatDate>(arg)));
+    else if (extract<RepeatDateTime>(arg).check())
+        self->addRepeat(Repeat(extract<RepeatDateTime>(arg)));
     else if (extract<RepeatDateList>(arg).check())
         self->addRepeat(Repeat(extract<RepeatDateList>(arg)));
     else if (extract<RepeatInteger>(arg).check())

Pyext/test/py_s_TestClientApi.py

@@ -19,8 +19,8 @@
 
 # ecflow_test_util, see File ecflow_test_util.py
 import ecflow_test_util as Test
-from ecflow import Defs,Suite,Family,Task,Edit,Meter, Clock, DState,  Style, State, RepeatDate, PrintStyle, \
-                   File, Client, SState, CheckPt, Cron, Late, debug_build, Flag, FlagType
+from ecflow import Defs, Suite, Family, Task, Edit, Meter, Clock, DState,  Style, State, RepeatDate, RepeatDateTime, \
+                   PrintStyle, File, Client, SState, CheckPt, Cron, Late, debug_build, Flag, FlagType
 
 def ecf_includes() :  return os.getcwd() + "/test/data/includes"
 
@@ -1225,13 +1225,15 @@ def test_client_alter_delete(ci):
     task_t2 = ci.get_defs().find_abs_node(t2)
     repeat = task_t2.get_repeat()
     assert repeat.empty(), "Expected repeat to be deleted:\n" + str(ci.get_defs())
-
+
+
 def test_client_alter_change(ci):
     print_test(ci,"test_client_alter_change")
     ci.delete_all() 
     defs =create_defs("test_client_alter_change")   
     t1 = "/test_client_alter_change/f1/t1"
     repeat_date_path = "/test_client_alter_change/f1/repeat_date"
+    repeat_datetime_path = "/test_client_alter_change/f1/repeat_datetime"
 
     task_t1 = defs.find_abs_node(t1)
     task_t1.add_variable("var","value")
@@ -1259,8 +1261,12 @@ def test_client_alter_change(ci):
 
 
     f1 = defs.find_abs_node("/test_client_alter_change/f1")
+    # Add a new Repeat (based on date)
     repeat_date = f1.add_task("repeat_date")
     repeat_date.add_repeat( RepeatDate("date",20100111,20100115,2) )  # can't add cron and repeat at the same level
+    # Add a new Repeat (based on date+time)
+    repeat_datetime = f1.add_task("repeat_datetime")
+    repeat_datetime.add_repeat(RepeatDateTime("datetime", "20100111T000000", "20100115T000000", "48:00:00"))
 
     ci.load(defs)   
 
@@ -1374,15 +1380,23 @@ def test_client_alter_change(ci):
     task_t1 = ci.get_defs().find_abs_node(t1)
     label = task_t1.find_label("label")
     assert label.new_value() == "new-value", "Expected alter of label to be 'new-value' but found " + label.new_value()
-    
-    ci.alter(repeat_date_path,"change","repeat","20100113")   
+
+    ci.alter(repeat_date_path,"change","repeat","20100113")
     sync_local(ci)
     task = ci.get_defs().find_abs_node(repeat_date_path)
     repeat = task.get_repeat()
     assert repeat.value() == 20100113, "Expected alter of repeat to be 20100113 but found " + str(repeat.value())
     res = ci.query('variable',task.get_abs_node_path(),repeat.name())
     assert res == "20100113", "Expected alter of repeat to be 20100113 but found " + res
 
+    ci.alter(repeat_datetime_path, "change", "repeat", "20100113T000000")
+    sync_local(ci)
+    task = ci.get_defs().find_abs_node(repeat_datetime_path)
+    repeat = task.get_repeat()
+    assert repeat.value() == 1263340800, "Expected alter of repeat to be 1263340800 (i.e. seconds between 19700101T000000 and 20100113T000000) but found " + str(repeat.value())
+    res = ci.query('variable', task.get_abs_node_path(), repeat.name())
+    assert res == "20100113T000000", "Expected alter of repeat to be 20100113T000000 but found " + res
+
 def test_client_alter_flag(ci):
     print_test(ci,"test_client_alter_flag")
     ci.delete_all() 

Pyext/test/py_u_TestAddDelete.py

@@ -404,6 +404,11 @@
     task.delete_repeat()      
     repeat = task.get_repeat(); assert repeat.empty(), "Expected no repeat"
 
+    task.add_repeat(ecflow.RepeatDateTime("datetime", "20100111T000000", "20100115T000000", "48:00:00"))
+    repeat = task.get_repeat(); assert not repeat.empty(), "Expected repeat"
+    task.delete_repeat()
+    repeat = task.get_repeat(); assert repeat.empty(), "Expected no repeat"
+
     task.add_repeat(ecflow.RepeatDateList("date",[20100111, 20100115]))
     repeat = task.get_repeat(); assert not repeat.empty(), "Expected repeat"
     task.delete_repeat()      
@@ -757,4 +762,3 @@
 
 
     print("All Tests pass")
-

Pyext/test/py_u_TestAddDeleteFunc.py

@@ -13,9 +13,9 @@
 
 import os
 from ecflow import Defs, Suite, Variable, Limit, InLimit, Task, PartExpression, \
-                   Event, Meter, Label, Queue, RepeatInteger, RepeatEnumerated, RepeatDate,RepeatDateList, RepeatString, \
-                   TimeSlot, TimeSeries, Today, Time, Date, Day, Days, Cron, Autocancel, Late, \
-                   DState, Clock, ChildCmdType, ZombieType, ZombieAttr, ZombieUserActionType, Client, debug_build
+                   Event, Meter, Label, Queue, RepeatInteger, RepeatEnumerated, RepeatDate, RepeatDateTime, \
+                   RepeatDateList, RepeatString, TimeSlot, TimeSeries, Today, Time, Date, Day, Days, Cron, Autocancel, \
+                   Late, DState, Clock, ChildCmdType, ZombieType, ZombieAttr, ZombieUserActionType, Client, debug_build
 import ecflow_test_util as Test
 
 if __name__ == "__main__":
@@ -133,6 +133,11 @@
     task.delete_repeat()      
     repeat = task.get_repeat(); assert repeat.empty(), "Expected no repeat"
 
+    task.add_repeat(RepeatDateTime("datetime", "20100111T000000", "20100115T000000", "48:00:00")).add_variable("W", "j")
+    task.delete_repeat()
+    repeat = task.get_repeat();
+    assert repeat.empty(), "Expected no repeat"
+
     task.add_repeat( RepeatDateList("date",[20100111, 20100115]) ).add_variable("W","j")
     task.delete_repeat()      
     repeat = task.get_repeat(); assert repeat.empty(), "Expected no repeat"
@@ -324,4 +329,3 @@
         count += 1;
 
     print("All Tests pass")
-

Pyext/test/py_u_TestCopy.py

@@ -286,6 +286,12 @@
     task_copy.delete_repeat()      
     repeat = task_copy.get_repeat(); assert repeat.empty(), "Expected no repeat"
 
+    task = ecflow.Task("task")
+    task.add_repeat(ecflow.RepeatDateTime("datetime", "20100111T000000", "20100115T000000", "48:00:00"))
+    task_copy = copy.copy(task)
+    task_copy.delete_repeat()
+    repeat = task_copy.get_repeat(); assert repeat.empty(), "Expected no repeat"
+
     task = ecflow.Task("task")
     task.add_repeat(ecflow.RepeatDateList("date", [20100111, 20100115]))
     task_copy = copy.copy(task)
@@ -507,4 +513,3 @@
     assert len(list(s1.zombies)) == 0, "Expected zero zombie attributes but found " + str(len(list(s1.zombies)))
 
     print("All Tests pass")
-

Pyext/test/py_u_TestDefs.py

@@ -19,7 +19,7 @@
 
 from ecflow import Suite, Family, Task, Defs, Clock, DState, PartExpression, Variable, Limit, InLimit, \
                    Date, Day, Event, Meter, Label, Autocancel, Days, TimeSlot, TimeSeries, Style, State, \
-                   RepeatString, RepeatDate, RepeatInteger, RepeatDay, RepeatEnumerated, \
+                   RepeatString, RepeatDate, RepeatDateTime, RepeatInteger, RepeatDay, RepeatEnumerated, \
                    Verify, PrintStyle, Time, Today, Late, Cron, Client, debug_build,Ecf
 import ecflow_test_util as Test
 
@@ -137,10 +137,13 @@
     task3.add_verify( Verify(State.complete, 1) )
     task3.add_autocancel( Autocancel(20,10,False) )
     task3.add_label( Label("label_name","label_value") )
-    
+
     task3_1 = family.add_task("t3_1")
     task3_1.add_repeat( RepeatDate("testDate",20100111,20100115) )
-
+
+    task3_2 = family.add_task("t3_2")
+    task3_2.add_repeat(RepeatDateTime("testDateTime", "20100111T000000", "20100115T000000"))
+
 
     task4 = family.add_task("t4")
     task4.add_repeat( RepeatInteger("testInteger",0,100,2) )
@@ -235,4 +238,4 @@
         expected_exeption = True
     assert expected_exeption,"expected exception"  
 
-    print("All tests pass")
+    print("All tests pass")

Pyext/test/py_u_TestError.py

@@ -15,7 +15,7 @@
 
 import os
 from ecflow import Day, Date, Meter, Event, Queue, Clock, Variable, Label, Limit, InLimit, \
-                   RepeatDate, RepeatEnumerated, RepeatInteger, RepeatString, \
+                   RepeatDate, RepeatDateTime, RepeatEnumerated, RepeatInteger, RepeatString, \
                    Task, Family, Suite, Defs, Client, debug_build, Trigger
 import ecflow_test_util as Test
 
@@ -119,7 +119,14 @@ def check_repeat_date(name, start, end, step):
         return True
     except RuntimeError: 
         return False
-
+
+def check_repeat_datetime(name, start, end, step):
+    try:
+        RepeatDateTime(name,start,end,step)
+        return True
+    except RuntimeError:
+        return False
+
 def check_repeat_integer(name, start, end, step):
     try:    
         RepeatInteger(name,start,end,step)
@@ -243,6 +250,13 @@ def check_defs(path_to_defs):
     assert check_repeat_date("m",00000000,00000000,200)== False,  "Expected Exception since start/end are not valid dates is invalid."
     assert check_repeat_date("",20000101,20001201,200)==False,    "Expected Exception since no name specified"
     assert check_repeat_date(" ",20000101,20001201,200)==False,   "Expected Exception cannot have spaces for a name"
+    assert check_repeat_datetime("m", "20000101T000000", "20001201T000000", "4800:00:00"), "Expected valid repeat"
+    assert check_repeat_datetime("m", "20001201T000000", "20000101T000000", "4800:00:00") == False, "Expected exception since end YMD > start YMD"
+    assert check_repeat_datetime("m", "200001011T000000", "20001201T000000", "4800:00:00") == False, "Expected Exception since start is invalid."
+    assert check_repeat_datetime("m", "20000101T000000", "200012013T000000", "4800:00:00") == False, "Expected Exception since send is invalid."
+    assert check_repeat_datetime("m", "00000000T000000", "00000000T000000", "4800:00:00") == False, "Expected Exception since start/end are not valid dates is invalid."
+    assert check_repeat_datetime("", "20000101T000000", "20001201T000000", "4800:00:00") == False, "Expected Exception since no name specified"
+    assert check_repeat_datetime(" ", "20000101T000000", "20001201T000000", "4800:00:00") == False, "Expected Exception cannot have spaces for a name"
     assert check_repeat_integer("name",0, 10, 2 ),                "Expected valid repeat"
     assert check_repeat_integer("",0, 10, 2 )==False,             "Expected Exception since no name specified"
     assert check_repeat_integer(" ",0, 10, 2 )==False,            "Expected Exception cannot have spaces for a name"

Pyext/test/py_u_TestRepeatArithmetic.py

@@ -47,12 +47,45 @@ def test_repeat_arithmetic(repeat_to_add,repeat_to_add2):
     t2.change_trigger("t1:YMD + 1 eq 20090201");   
     assert t2.evaluate_trigger(), "Expected trigger to evaluate. 20090131 + 1  == 20090201"
 
+
+def test_repeat_datetime_arithmetic(repeat_to_add,repeat_to_add2):
+
+    defs = ecflow.Defs()
+    s1 = defs.add_suite("s1")
+    t1 = s1.add_task("t1").add_repeat( repeat_to_add )
+    t2 = s1.add_task("t2").add_trigger("t1:DT ge 20100601T000000")
+
+    # Check trigger expressions
+    print(f"## defs: {defs}")
+    assert len(defs.check()) == 0,  "Expected no errors in parsing expressions."
+
+    # Initial value of repeat is 20090101 hence trigger should fail to evaluate
+    assert t2.evaluate_trigger() == False, "Expected trigger to evaluate. 20090101T000000 >= 20100601T000000"
+
+    # Check end of month - 1 day
+    t2.change_trigger("t1:DT - 86400 eq 20081231T000000")
+    assert t2.evaluate_trigger(), "Expected trigger to evaluate. 20090101T000000 - 86400  == 20081231T000000"
+
+    # check addition
+    t2.change_trigger("t1:DT + 86400 eq 20090102T000000");
+    assert t2.evaluate_trigger(), "Expected trigger to evaluate. 20090101T000000 + 86400  == 20090102T000000"
+
+    # Check the end of each month + 1 day
+    t1.delete_repeat();
+    t1.add_repeat( repeat_to_add2 )
+    t2.change_trigger("t1:DT + 86400 eq 20090201T000000");
+    assert t2.evaluate_trigger(), "Expected trigger to evaluate. 20090131 + 86400  == 20090201T000000"
+
+
 if __name__ == "__main__":
     Test.print_test_start(os.path.basename(__file__))
-
-
-    test_repeat_arithmetic(ecflow.RepeatDate("YMD",20090101,20091231,1),    ecflow.RepeatDate("YMD",20090131,20101231,1))
-    test_repeat_arithmetic(ecflow.RepeatDateList("YMD",[20090101,20091231]),ecflow.RepeatDateList("YMD",[20090131,20101231]) )
+
+    test_repeat_arithmetic(ecflow.RepeatDate("YMD", 20090101, 20091231, 1),
+                           ecflow.RepeatDate("YMD", 20090131, 20101231, 1))
+    test_repeat_arithmetic(ecflow.RepeatDateList("YMD", [20090101, 20091231]),
+                           ecflow.RepeatDateList("YMD", [20090131, 20101231]))
+
+    test_repeat_datetime_arithmetic(ecflow.RepeatDateTime("DT", "20090101T000000", "20091231T000000", "24:00:00"),
+                                    ecflow.RepeatDateTime("DT", "20090131T000000", "20101231T000000", "24:00:00"))
 
     print("All Tests pass")
-