From 75462fefcefe972abf44812dc0f09a42f3a23a93 Mon Sep 17 00:00:00 2001 From: Daniel Milroy Date: Fri, 7 Feb 2025 21:37:02 -0800 Subject: [PATCH 1/6] traversers: break visitation cycle in modification traversal Problem: the traversal that performs full and partial cancellation visits vertices multiple times. This behavior has existed for at least six years but wasn't noticed due to full cancel being robust to multiple visitation. Partial cancel is not robust to multiple visitation. For partial cancel removals, if the removal request equals half the number of total allocated vertices for the corresponding jobid tracked by pruning filters, the double visitation will trigger the removal of the root's job tag for that jobid. Subsequent full cancels will fail to remove the allocation because the root job tag for the jobid doesn't exist. Add coloring to vertices based on visitation to break the cycle. --- resource/traversers/dfu_impl_update.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/resource/traversers/dfu_impl_update.cpp b/resource/traversers/dfu_impl_update.cpp index f74ea34c9..b47a88810 100644 --- a/resource/traversers/dfu_impl_update.cpp +++ b/resource/traversers/dfu_impl_update.cpp @@ -595,6 +595,7 @@ int dfu_impl_t::mod_dfv (vtx_t u, int64_t jobid, modify_data_t &mod_data) subsystem_t dom = m_match->dom_subsystem (); f_out_edg_iterator_t ei, ei_end; + (*m_graph)[u].idata.colors[dom] = m_color.gray (); if ((rc = mod_idata (u, jobid, dom, mod_data, stop)) != 0 || stop) goto done; if ((rc = mod_plan (u, jobid, mod_data)) != 0) @@ -610,6 +611,7 @@ int dfu_impl_t::mod_dfv (vtx_t u, int64_t jobid, modify_data_t &mod_data) rc += mod_upv (tgt, jobid, mod_data); } } + (*m_graph)[u].idata.colors[dom] = m_color.black (); done: return rc; } From 7b3b9b1c2b74636530444faf9ce17deddb64e122 Mon Sep 17 00:00:00 2001 From: Daniel Milroy Date: Sun, 9 Feb 2025 15:23:59 -0800 Subject: [PATCH 2/6] traversers: track pre and postorder visit counts in cancels Problem: the traverser currently doesn't track pre and postorder visit counts for full and partial cancels. Add the needed tracking capability. --- resource/traversers/dfu.cpp | 13 +++++++++++-- resource/traversers/dfu_impl_update.cpp | 7 +++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/resource/traversers/dfu.cpp b/resource/traversers/dfu.cpp index 97996c7ec..9241a3f1d 100644 --- a/resource/traversers/dfu.cpp +++ b/resource/traversers/dfu.cpp @@ -451,6 +451,7 @@ int dfu_traverser_t::find (std::shared_ptr &writers, const std: int dfu_traverser_t::remove (int64_t jobid) { + int rc = 0; // Clear the error message to disambiguate errors clear_err_message (); @@ -463,7 +464,11 @@ int dfu_traverser_t::remove (int64_t jobid) } vtx_t root = get_graph_db ()->metadata.roots.at (dom); - return detail::dfu_impl_t::remove (root, jobid); + + rc = detail::dfu_impl_t::remove (root, jobid); + m_total_preorder = detail::dfu_impl_t::get_preorder_count (); + m_total_postorder = detail::dfu_impl_t::get_postorder_count (); + return rc; } int dfu_traverser_t::remove (const std::string &R_to_cancel, @@ -471,6 +476,7 @@ int dfu_traverser_t::remove (const std::string &R_to_cancel, int64_t jobid, bool &full_cancel) { + int rc = 0; // Clear the error message to disambiguate errors clear_err_message (); @@ -483,7 +489,10 @@ int dfu_traverser_t::remove (const std::string &R_to_cancel, } vtx_t root = get_graph_db ()->metadata.roots.at (dom); - return detail::dfu_impl_t::remove (root, R_to_cancel, reader, jobid, full_cancel); + rc = detail::dfu_impl_t::remove (root, R_to_cancel, reader, jobid, full_cancel); + m_total_preorder = detail::dfu_impl_t::get_preorder_count (); + m_total_postorder = detail::dfu_impl_t::get_postorder_count (); + return rc; } int dfu_traverser_t::mark (const std::string &root_path, resource_pool_t::status_t status) diff --git a/resource/traversers/dfu_impl_update.cpp b/resource/traversers/dfu_impl_update.cpp index b47a88810..9cbf94663 100644 --- a/resource/traversers/dfu_impl_update.cpp +++ b/resource/traversers/dfu_impl_update.cpp @@ -595,6 +595,7 @@ int dfu_impl_t::mod_dfv (vtx_t u, int64_t jobid, modify_data_t &mod_data) subsystem_t dom = m_match->dom_subsystem (); f_out_edg_iterator_t ei, ei_end; + m_preorder++; (*m_graph)[u].idata.colors[dom] = m_color.gray (); if ((rc = mod_idata (u, jobid, dom, mod_data, stop)) != 0 || stop) goto done; @@ -612,6 +613,7 @@ int dfu_impl_t::mod_dfv (vtx_t u, int64_t jobid, modify_data_t &mod_data) } } (*m_graph)[u].idata.colors[dom] = m_color.black (); + m_postorder++; done: return rc; } @@ -780,6 +782,9 @@ int dfu_impl_t::update (vtx_t root, int dfu_impl_t::remove (vtx_t root, int64_t jobid) { + m_preorder = 0; + m_postorder = 0; + bool root_has_jtag = ((*m_graph)[root].idata.tags.find (jobid) != (*m_graph)[root].idata.tags.end ()); modify_data_t mod_data; @@ -798,6 +803,8 @@ int dfu_impl_t::remove (vtx_t root, modify_data_t mod_data; resource_graph_t &g = m_graph_db->resource_graph; resource_graph_metadata_t &m = m_graph_db->metadata; + m_preorder = 0; + m_postorder = 0; if (reader->partial_cancel (g, m, mod_data, R_to_cancel, jobid) != 0) { m_err_msg += __FUNCTION__; From ddc654c16ae31845993ae45333dbfe78c3a9fdc2 Mon Sep 17 00:00:00 2001 From: Daniel Milroy Date: Sun, 9 Feb 2025 14:53:34 -0800 Subject: [PATCH 3/6] resource-query: print pre and postorder visit counts in cancels Problem: after fixing the visitation cycles in cancellations, resource-query needs an option to output the visit counts for testsuite checks. Add the "stats" option to cancel and partial-cancel. --- resource/utilities/command.cpp | 51 +++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/resource/utilities/command.cpp b/resource/utilities/command.cpp index d3678e3c3..e0b653f41 100644 --- a/resource/utilities/command.cpp +++ b/resource/utilities/command.cpp @@ -73,12 +73,13 @@ command_t commands[] = "c", cmd_cancel, "Cancel an allocation or reservation: " - "resource-query> cancel jobid"}, + "resource-query> cancel jobid (optional subcmd: stats)"}, {"partial-cancel", "pc", cmd_partial_cancel, "Partially release an allocation: " - "resource-query> partial-cancel jobid (file format: jgf | rv1exec) R_to_cancel.file"}, + "resource-query> partial-cancel jobid (file format: jgf | rv1exec) R_to_cancel.file " + "(optional subcmd: stats)"}, {"set-property", "p", cmd_set_property, @@ -651,7 +652,7 @@ int cmd_find (std::shared_ptr &ctx, std::vector int cmd_cancel (std::shared_ptr &ctx, std::vector &args) { - if (args.size () != 2) { + if (args.size () < 2 || args.size () > 3) { std::cerr << "ERROR: malformed command" << std::endl; return 0; } @@ -659,6 +660,14 @@ int cmd_cancel (std::shared_ptr &ctx, std::vectorparams.r_fname != "") ? ctx->params.r_out : std::cout; + + if (args.size () == 3) { + stats = args[2]; + } if (ctx->allocations.find (jobid) != ctx->allocations.end ()) { if ((rc = do_remove (ctx, jobid)) == 0) @@ -675,6 +684,19 @@ int cmd_cancel (std::shared_ptr &ctx, std::vectortraverser->get_total_preorder_count (); + postorder_count = ctx->traverser->get_total_postorder_count (); + out << "INFO:" + << " =============================" << std::endl; + out << "INFO:" + << " CANCEL PREORDER COUNT=\"" << preorder_count << "\"" << std::endl; + out << "INFO:" + << " CANCEL POSTORDER COUNT=\"" << postorder_count << "\"" << std::endl; + out << "INFO:" + << " =============================" << std::endl; + } + done: return 0; } @@ -685,7 +707,7 @@ int cmd_partial_cancel (std::shared_ptr &ctx, std::vector rd; - if (args.size () != 4) { + if (args.size () < 4 || args.size () > 5) { std::cerr << "ERROR: malformed command" << std::endl; return 0; } @@ -695,6 +717,14 @@ int cmd_partial_cancel (std::shared_ptr &ctx, std::vectorparams.r_fname != "") ? ctx->params.r_out : std::cout; + + if (args.size () == 5) { + stats = args[4]; + } if (!(reader == "jgf" || reader == "rv1exec")) { std::cerr << "ERROR: unsupported reader " << args[2] << std::endl; @@ -737,6 +767,19 @@ int cmd_partial_cancel (std::shared_ptr &ctx, std::vectortraverser->get_total_preorder_count (); + postorder_count = ctx->traverser->get_total_postorder_count (); + out << "INFO:" + << " =============================" << std::endl; + out << "INFO:" + << " PARTIAL CANCEL PREORDER COUNT=\"" << preorder_count << "\"" << std::endl; + out << "INFO:" + << " PARTIAL CANCEL POSTORDER COUNT=\"" << postorder_count << "\"" << std::endl; + out << "INFO:" + << " =============================" << std::endl; + } + done: return 0; } From 40ccf30013b97b31621795c5431b4dff6fc2d4fb Mon Sep 17 00:00:00 2001 From: Daniel Milroy Date: Sun, 9 Feb 2025 15:03:07 -0800 Subject: [PATCH 4/6] testsuite: update expected output for cancel tests Problem: introducing pre and postorder visit output into the testsuite adds the counts output to the expected outputs. Update the expected outputs for the existing cancellation tests. --- t/data/resource/commands/cancel/cmds01.in | 6 +- t/data/resource/commands/cancel/cmds02.in | 34 ++++++------ t/data/resource/expected/cancel/001.R.out | 12 ++++ t/data/resource/expected/cancel/002.R.out | 68 +++++++++++++++++++++++ t/data/resource/expected/cancel/003.R.out | 12 ++++ t/data/resource/expected/cancel/004.R.out | 68 +++++++++++++++++++++++ t/data/resource/expected/cancel/011.R.out | 12 ++++ t/data/resource/expected/cancel/012.R.out | 68 +++++++++++++++++++++++ t/data/resource/expected/cancel/013.R.out | 12 ++++ t/data/resource/expected/cancel/014.R.out | 68 +++++++++++++++++++++++ 10 files changed, 340 insertions(+), 20 deletions(-) diff --git a/t/data/resource/commands/cancel/cmds01.in b/t/data/resource/commands/cancel/cmds01.in index 219f8bf4d..c526c166a 100644 --- a/t/data/resource/commands/cancel/cmds01.in +++ b/t/data/resource/commands/cancel/cmds01.in @@ -15,10 +15,10 @@ match allocate_orelse_reserve @TEST_SRCDIR@/data/resource/jobspecs/cancel/test01 match allocate_orelse_reserve @TEST_SRCDIR@/data/resource/jobspecs/cancel/test015.yaml match allocate_orelse_reserve @TEST_SRCDIR@/data/resource/jobspecs/cancel/test016.yaml match allocate_orelse_reserve @TEST_SRCDIR@/data/resource/jobspecs/cancel/test017.yaml -cancel 1 +cancel 1 stats match allocate_orelse_reserve @TEST_SRCDIR@/data/resource/jobspecs/cancel/test001.yaml -cancel 2 +cancel 2 stats match allocate_orelse_reserve @TEST_SRCDIR@/data/resource/jobspecs/cancel/test002.yaml -cancel 12 +cancel 12 stats match allocate_orelse_reserve @TEST_SRCDIR@/data/resource/jobspecs/cancel/test012.yaml quit diff --git a/t/data/resource/commands/cancel/cmds02.in b/t/data/resource/commands/cancel/cmds02.in index 59c1d1028..41a8065a9 100644 --- a/t/data/resource/commands/cancel/cmds02.in +++ b/t/data/resource/commands/cancel/cmds02.in @@ -15,23 +15,23 @@ match allocate_orelse_reserve @TEST_SRCDIR@/data/resource/jobspecs/cancel/test01 match allocate_orelse_reserve @TEST_SRCDIR@/data/resource/jobspecs/cancel/test015.yaml match allocate_orelse_reserve @TEST_SRCDIR@/data/resource/jobspecs/cancel/test016.yaml match allocate_orelse_reserve @TEST_SRCDIR@/data/resource/jobspecs/cancel/test017.yaml -cancel 1 -cancel 2 -cancel 3 -cancel 4 -cancel 5 -cancel 6 -cancel 7 -cancel 8 -cancel 9 -cancel 10 -cancel 11 -cancel 12 -cancel 13 -cancel 14 -cancel 15 -cancel 16 -cancel 17 +cancel 1 stats +cancel 2 stats +cancel 3 stats +cancel 4 stats +cancel 5 stats +cancel 6 stats +cancel 7 stats +cancel 8 stats +cancel 9 stats +cancel 10 stats +cancel 11 stats +cancel 12 stats +cancel 13 stats +cancel 14 stats +cancel 15 stats +cancel 16 stats +cancel 17 stats match allocate_orelse_reserve @TEST_SRCDIR@/data/resource/jobspecs/cancel/test001.yaml match allocate_orelse_reserve @TEST_SRCDIR@/data/resource/jobspecs/cancel/test002.yaml match allocate_orelse_reserve @TEST_SRCDIR@/data/resource/jobspecs/cancel/test003.yaml diff --git a/t/data/resource/expected/cancel/001.R.out b/t/data/resource/expected/cancel/001.R.out index 83ced0dbd..9e00327f9 100644 --- a/t/data/resource/expected/cancel/001.R.out +++ b/t/data/resource/expected/cancel/001.R.out @@ -19911,6 +19911,10 @@ INFO: ============================= INFO: JOBID=17 INFO: RESOURCES=RESERVED INFO: SCHEDULED AT=147601 +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="73" +INFO: CANCEL POSTORDER COUNT="41" INFO: ============================= ---------------core0[1:x] ---------------core1[1:x] @@ -19957,6 +19961,10 @@ INFO: ============================= INFO: JOBID=18 INFO: RESOURCES=ALLOCATED INFO: SCHEDULED AT=Now +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="4915" +INFO: CANCEL POSTORDER COUNT="3907" INFO: ============================= ---------------core0[1:x] ---------------core1[1:x] @@ -23869,6 +23877,10 @@ INFO: ============================= INFO: JOBID=19 INFO: RESOURCES=ALLOCATED INFO: SCHEDULED AT=Now +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="3151" +INFO: CANCEL POSTORDER COUNT="3141" INFO: ============================= ---------------core0[1:x] ---------------core1[1:x] diff --git a/t/data/resource/expected/cancel/002.R.out b/t/data/resource/expected/cancel/002.R.out index 47d70c39d..5add0bbd2 100644 --- a/t/data/resource/expected/cancel/002.R.out +++ b/t/data/resource/expected/cancel/002.R.out @@ -19911,6 +19911,74 @@ INFO: ============================= INFO: JOBID=17 INFO: RESOURCES=RESERVED INFO: SCHEDULED AT=147601 +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="73" +INFO: CANCEL POSTORDER COUNT="41" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="4915" +INFO: CANCEL POSTORDER COUNT="3907" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="2615" +INFO: CANCEL POSTORDER COUNT="2007" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="1679" +INFO: CANCEL POSTORDER COUNT="1285" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="3151" +INFO: CANCEL POSTORDER COUNT="3141" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="667" +INFO: CANCEL POSTORDER COUNT="640" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="427" +INFO: CANCEL POSTORDER COUNT="395" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="763" +INFO: CANCEL POSTORDER COUNT="738" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="400" +INFO: CANCEL POSTORDER COUNT="302" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="667" +INFO: CANCEL POSTORDER COUNT="640" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="409" +INFO: CANCEL POSTORDER COUNT="394" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="3151" +INFO: CANCEL POSTORDER COUNT="3141" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="379" +INFO: CANCEL POSTORDER COUNT="346" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="121" +INFO: CANCEL POSTORDER COUNT="100" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="313" +INFO: CANCEL POSTORDER COUNT="296" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="313" +INFO: CANCEL POSTORDER COUNT="296" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="2173" +INFO: CANCEL POSTORDER COUNT="2160" INFO: ============================= ---------------core0[1:x] ---------------core1[1:x] diff --git a/t/data/resource/expected/cancel/003.R.out b/t/data/resource/expected/cancel/003.R.out index f6b15ad59..25c4d6cb7 100644 --- a/t/data/resource/expected/cancel/003.R.out +++ b/t/data/resource/expected/cancel/003.R.out @@ -19911,6 +19911,10 @@ INFO: ============================= INFO: JOBID=17 INFO: RESOURCES=RESERVED INFO: SCHEDULED AT=147601 +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="73" +INFO: CANCEL POSTORDER COUNT="41" INFO: ============================= ---------------core0[1:x] ---------------core1[1:x] @@ -19957,6 +19961,10 @@ INFO: ============================= INFO: JOBID=18 INFO: RESOURCES=ALLOCATED INFO: SCHEDULED AT=Now +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="4915" +INFO: CANCEL POSTORDER COUNT="3907" INFO: ============================= ---------------core0[1:x] ---------------core1[1:x] @@ -23869,6 +23877,10 @@ INFO: ============================= INFO: JOBID=19 INFO: RESOURCES=ALLOCATED INFO: SCHEDULED AT=Now +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="3151" +INFO: CANCEL POSTORDER COUNT="3141" INFO: ============================= ---------------core0[1:x] ---------------core1[1:x] diff --git a/t/data/resource/expected/cancel/004.R.out b/t/data/resource/expected/cancel/004.R.out index a50195573..76e192654 100644 --- a/t/data/resource/expected/cancel/004.R.out +++ b/t/data/resource/expected/cancel/004.R.out @@ -19911,6 +19911,74 @@ INFO: ============================= INFO: JOBID=17 INFO: RESOURCES=RESERVED INFO: SCHEDULED AT=147601 +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="73" +INFO: CANCEL POSTORDER COUNT="41" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="4915" +INFO: CANCEL POSTORDER COUNT="3907" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="2615" +INFO: CANCEL POSTORDER COUNT="2007" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="1679" +INFO: CANCEL POSTORDER COUNT="1285" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="3151" +INFO: CANCEL POSTORDER COUNT="3141" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="667" +INFO: CANCEL POSTORDER COUNT="640" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="427" +INFO: CANCEL POSTORDER COUNT="395" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="763" +INFO: CANCEL POSTORDER COUNT="738" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="400" +INFO: CANCEL POSTORDER COUNT="302" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="667" +INFO: CANCEL POSTORDER COUNT="640" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="409" +INFO: CANCEL POSTORDER COUNT="394" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="3151" +INFO: CANCEL POSTORDER COUNT="3141" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="379" +INFO: CANCEL POSTORDER COUNT="346" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="121" +INFO: CANCEL POSTORDER COUNT="100" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="313" +INFO: CANCEL POSTORDER COUNT="296" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="313" +INFO: CANCEL POSTORDER COUNT="296" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="2173" +INFO: CANCEL POSTORDER COUNT="2160" INFO: ============================= ---------------core0[1:x] ---------------core1[1:x] diff --git a/t/data/resource/expected/cancel/011.R.out b/t/data/resource/expected/cancel/011.R.out index 7d71ae472..9deb94e1b 100644 --- a/t/data/resource/expected/cancel/011.R.out +++ b/t/data/resource/expected/cancel/011.R.out @@ -26112,6 +26112,10 @@ INFO: ============================= INFO: JOBID=17 INFO: RESOURCES=RESERVED INFO: SCHEDULED AT=147601 +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="73" +INFO: CANCEL POSTORDER COUNT="51" INFO: ============================= ---------------core0[1:x] ---------------core1[1:x] @@ -26168,6 +26172,10 @@ INFO: ============================= INFO: JOBID=18 INFO: RESOURCES=ALLOCATED INFO: SCHEDULED AT=Now +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="4915" +INFO: CANCEL POSTORDER COUNT="4907" INFO: ============================= ---------------core0[1:x] ---------------core1[1:x] @@ -31080,6 +31088,10 @@ INFO: ============================= INFO: JOBID=19 INFO: RESOURCES=ALLOCATED INFO: SCHEDULED AT=Now +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="3151" +INFO: CANCEL POSTORDER COUNT="3141" INFO: ============================= ---------------core0[1:x] ---------------core1[1:x] diff --git a/t/data/resource/expected/cancel/012.R.out b/t/data/resource/expected/cancel/012.R.out index 088482484..de47947a1 100644 --- a/t/data/resource/expected/cancel/012.R.out +++ b/t/data/resource/expected/cancel/012.R.out @@ -26112,6 +26112,74 @@ INFO: ============================= INFO: JOBID=17 INFO: RESOURCES=RESERVED INFO: SCHEDULED AT=147601 +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="73" +INFO: CANCEL POSTORDER COUNT="51" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="4915" +INFO: CANCEL POSTORDER COUNT="4907" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="4915" +INFO: CANCEL POSTORDER COUNT="4907" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="3151" +INFO: CANCEL POSTORDER COUNT="3141" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="3151" +INFO: CANCEL POSTORDER COUNT="3141" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="667" +INFO: CANCEL POSTORDER COUNT="640" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="427" +INFO: CANCEL POSTORDER COUNT="395" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="763" +INFO: CANCEL POSTORDER COUNT="738" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="745" +INFO: CANCEL POSTORDER COUNT="737" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="667" +INFO: CANCEL POSTORDER COUNT="640" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="409" +INFO: CANCEL POSTORDER COUNT="394" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="3151" +INFO: CANCEL POSTORDER COUNT="3141" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="379" +INFO: CANCEL POSTORDER COUNT="346" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="121" +INFO: CANCEL POSTORDER COUNT="100" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="313" +INFO: CANCEL POSTORDER COUNT="296" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="313" +INFO: CANCEL POSTORDER COUNT="296" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="2173" +INFO: CANCEL POSTORDER COUNT="2160" INFO: ============================= ---------------core0[1:x] ---------------core1[1:x] diff --git a/t/data/resource/expected/cancel/013.R.out b/t/data/resource/expected/cancel/013.R.out index a581d9a60..74d3c8ec1 100644 --- a/t/data/resource/expected/cancel/013.R.out +++ b/t/data/resource/expected/cancel/013.R.out @@ -26112,6 +26112,10 @@ INFO: ============================= INFO: JOBID=17 INFO: RESOURCES=RESERVED INFO: SCHEDULED AT=147601 +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="73" +INFO: CANCEL POSTORDER COUNT="51" INFO: ============================= ---------------core0[1:x] ---------------core1[1:x] @@ -26168,6 +26172,10 @@ INFO: ============================= INFO: JOBID=18 INFO: RESOURCES=ALLOCATED INFO: SCHEDULED AT=Now +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="4915" +INFO: CANCEL POSTORDER COUNT="4907" INFO: ============================= ---------------core0[1:x] ---------------core1[1:x] @@ -31080,6 +31088,10 @@ INFO: ============================= INFO: JOBID=19 INFO: RESOURCES=ALLOCATED INFO: SCHEDULED AT=Now +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="3151" +INFO: CANCEL POSTORDER COUNT="3141" INFO: ============================= ---------------core0[1:x] ---------------core1[1:x] diff --git a/t/data/resource/expected/cancel/014.R.out b/t/data/resource/expected/cancel/014.R.out index 18018889d..fa5bf6b3a 100644 --- a/t/data/resource/expected/cancel/014.R.out +++ b/t/data/resource/expected/cancel/014.R.out @@ -26112,6 +26112,74 @@ INFO: ============================= INFO: JOBID=17 INFO: RESOURCES=RESERVED INFO: SCHEDULED AT=147601 +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="73" +INFO: CANCEL POSTORDER COUNT="51" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="4915" +INFO: CANCEL POSTORDER COUNT="4907" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="4915" +INFO: CANCEL POSTORDER COUNT="4907" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="3151" +INFO: CANCEL POSTORDER COUNT="3141" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="3151" +INFO: CANCEL POSTORDER COUNT="3141" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="667" +INFO: CANCEL POSTORDER COUNT="640" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="427" +INFO: CANCEL POSTORDER COUNT="395" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="763" +INFO: CANCEL POSTORDER COUNT="738" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="745" +INFO: CANCEL POSTORDER COUNT="737" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="667" +INFO: CANCEL POSTORDER COUNT="640" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="409" +INFO: CANCEL POSTORDER COUNT="394" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="3151" +INFO: CANCEL POSTORDER COUNT="3141" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="379" +INFO: CANCEL POSTORDER COUNT="346" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="121" +INFO: CANCEL POSTORDER COUNT="100" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="313" +INFO: CANCEL POSTORDER COUNT="296" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="313" +INFO: CANCEL POSTORDER COUNT="296" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="2173" +INFO: CANCEL POSTORDER COUNT="2160" INFO: ============================= ---------------core0[1:x] ---------------core1[1:x] From 57580208033bbf7fd0069a903ef974b396ae3240 Mon Sep 17 00:00:00 2001 From: Daniel Milroy Date: Fri, 7 Feb 2025 21:37:44 -0800 Subject: [PATCH 5/6] testsuite: fix rank bug in tiny-partial-cancel.json Problem: tiny-partial-cancel.json assigns both node0 and node1 to rank0. The children of node1 are correctly assigned to rank1. This causes incorrect behavior in partial cancel tests. Assign node1 to rank1 as intended. --- t/data/resource/jgfs/elastic/tiny-partial-cancel.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/data/resource/jgfs/elastic/tiny-partial-cancel.json b/t/data/resource/jgfs/elastic/tiny-partial-cancel.json index b874096ea..0deba6683 100644 --- a/t/data/resource/jgfs/elastic/tiny-partial-cancel.json +++ b/t/data/resource/jgfs/elastic/tiny-partial-cancel.json @@ -60,7 +60,7 @@ "name": "node1", "id": 1, "uniq_id": 3, - "rank": 0, + "rank": 1, "exclusive": false, "unit": "", "size": 1, From 0fd29d7e3c4a3640838cec4d50b5121a0e728613 Mon Sep 17 00:00:00 2001 From: Daniel Milroy Date: Fri, 7 Feb 2025 21:38:24 -0800 Subject: [PATCH 6/6] testsuite: add test for successful full cancel after partial cancel Problem: the multiple visitation bug wasn't detected in partial cancel because existing tests don't perform a follow-up, full cancel and subsequent re-allocation. Add these tests to check for incorrect visitation behavior. --- t/data/resource/commands/cancel/cmds10.in | 9 + t/data/resource/commands/cancel/cmds11.in | 9 + t/data/resource/expected/cancel/024.R.out | 378 ++++++++++++++++++ .../rv1exec/cancel/rank0_cancel-jgfgraph.json | 1 + t/t3008-resource-cancel.t | 17 + 5 files changed, 414 insertions(+) create mode 100644 t/data/resource/commands/cancel/cmds10.in create mode 100644 t/data/resource/commands/cancel/cmds11.in create mode 100644 t/data/resource/expected/cancel/024.R.out create mode 100644 t/data/resource/rv1exec/cancel/rank0_cancel-jgfgraph.json diff --git a/t/data/resource/commands/cancel/cmds10.in b/t/data/resource/commands/cancel/cmds10.in new file mode 100644 index 000000000..7e0e65373 --- /dev/null +++ b/t/data/resource/commands/cancel/cmds10.in @@ -0,0 +1,9 @@ +match allocate @TEST_SRCDIR@/data/resource/jobspecs/cancel/test022.yaml +partial-cancel 1 jgf @TEST_SRCDIR@/data/resource/jgfs/elastic/node-1-partial-cancel.json stats +find sched-now=allocated +info 1 +cancel 1 stats +find sched-now=allocated +match allocate @TEST_SRCDIR@/data/resource/jobspecs/cancel/test022.yaml +find sched-now=allocated +quit diff --git a/t/data/resource/commands/cancel/cmds11.in b/t/data/resource/commands/cancel/cmds11.in new file mode 100644 index 000000000..a3839b243 --- /dev/null +++ b/t/data/resource/commands/cancel/cmds11.in @@ -0,0 +1,9 @@ +match allocate @TEST_SRCDIR@/data/resource/jobspecs/cancel/test022.yaml +partial-cancel 1 rv1exec @TEST_SRCDIR@/data/resource/rv1exec/cancel/rank0_cancel-jgfgraph.json stats +find sched-now=allocated +info 1 +cancel 1 stats +find sched-now=allocated +match allocate @TEST_SRCDIR@/data/resource/jobspecs/cancel/test022.yaml +find sched-now=allocated +quit diff --git a/t/data/resource/expected/cancel/024.R.out b/t/data/resource/expected/cancel/024.R.out new file mode 100644 index 000000000..ec8da53bd --- /dev/null +++ b/t/data/resource/expected/cancel/024.R.out @@ -0,0 +1,378 @@ + ---------------core0[1:x] + ---------------core1[1:x] + ---------------core2[1:x] + ---------------core3[1:x] + ---------------core4[1:x] + ---------------core5[1:x] + ---------------core6[1:x] + ---------------core7[1:x] + ---------------core8[1:x] + ---------------core9[1:x] + ---------------core10[1:x] + ---------------core11[1:x] + ---------------core12[1:x] + ---------------core13[1:x] + ---------------core14[1:x] + ---------------core15[1:x] + ---------------core16[1:x] + ---------------core17[1:x] + ---------------gpu0[1:x] + ---------------memory0[2:x] + ---------------memory1[2:x] + ---------------memory2[2:x] + ---------------memory3[2:x] + ------------socket0[1:x] + ---------------core18[1:x] + ---------------core19[1:x] + ---------------core20[1:x] + ---------------core21[1:x] + ---------------core22[1:x] + ---------------core23[1:x] + ---------------core24[1:x] + ---------------core25[1:x] + ---------------core26[1:x] + ---------------core27[1:x] + ---------------core28[1:x] + ---------------core29[1:x] + ---------------core30[1:x] + ---------------core31[1:x] + ---------------core32[1:x] + ---------------core33[1:x] + ---------------core34[1:x] + ---------------core35[1:x] + ---------------gpu1[1:x] + ---------------memory4[2:x] + ---------------memory5[2:x] + ---------------memory6[2:x] + ---------------memory7[2:x] + ------------socket1[1:x] + ---------node0[1:x] + ---------------core0[1:x] + ---------------core1[1:x] + ---------------core2[1:x] + ---------------core3[1:x] + ---------------core4[1:x] + ---------------core5[1:x] + ---------------core6[1:x] + ---------------core7[1:x] + ---------------core8[1:x] + ---------------core9[1:x] + ---------------core10[1:x] + ---------------core11[1:x] + ---------------core12[1:x] + ---------------core13[1:x] + ---------------core14[1:x] + ---------------core15[1:x] + ---------------core16[1:x] + ---------------core17[1:x] + ---------------gpu0[1:x] + ---------------memory0[2:x] + ---------------memory1[2:x] + ---------------memory2[2:x] + ---------------memory3[2:x] + ------------socket0[1:x] + ---------------core18[1:x] + ---------------core19[1:x] + ---------------core20[1:x] + ---------------core21[1:x] + ---------------core22[1:x] + ---------------core23[1:x] + ---------------core24[1:x] + ---------------core25[1:x] + ---------------core26[1:x] + ---------------core27[1:x] + ---------------core28[1:x] + ---------------core29[1:x] + ---------------core30[1:x] + ---------------core31[1:x] + ---------------core32[1:x] + ---------------core33[1:x] + ---------------core34[1:x] + ---------------core35[1:x] + ---------------gpu1[1:x] + ---------------memory4[2:x] + ---------------memory5[2:x] + ---------------memory6[2:x] + ---------------memory7[2:x] + ------------socket1[1:x] + ---------node1[1:x] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=1 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= +INFO: ============================= +INFO: PARTIAL CANCEL PREORDER COUNT="4" +INFO: PARTIAL CANCEL POSTORDER COUNT="2" +INFO: ============================= + ---------------core0[1:x] + ---------------core1[1:x] + ---------------core2[1:x] + ---------------core3[1:x] + ---------------core4[1:x] + ---------------core5[1:x] + ---------------core6[1:x] + ---------------core7[1:x] + ---------------core8[1:x] + ---------------core9[1:x] + ---------------core10[1:x] + ---------------core11[1:x] + ---------------core12[1:x] + ---------------core13[1:x] + ---------------core14[1:x] + ---------------core15[1:x] + ---------------core16[1:x] + ---------------core17[1:x] + ---------------gpu0[1:x] + ---------------memory0[2:x] + ---------------memory1[2:x] + ---------------memory2[2:x] + ---------------memory3[2:x] + ------------socket0[1:x] + ---------------core18[1:x] + ---------------core19[1:x] + ---------------core20[1:x] + ---------------core21[1:x] + ---------------core22[1:x] + ---------------core23[1:x] + ---------------core24[1:x] + ---------------core25[1:x] + ---------------core26[1:x] + ---------------core27[1:x] + ---------------core28[1:x] + ---------------core29[1:x] + ---------------core30[1:x] + ---------------core31[1:x] + ---------------core32[1:x] + ---------------core33[1:x] + ---------------core34[1:x] + ---------------core35[1:x] + ---------------gpu1[1:x] + ---------------memory4[2:x] + ---------------memory5[2:x] + ---------------memory6[2:x] + ---------------memory7[2:x] + ------------socket1[1:x] + ---------node1[1:x] + ------rack0[1:x] + ---tiny0[1:x] +INFO: ============================= +INFO: EXPRESSION="sched-now=allocated" +INFO: ============================= +INFO: ============================= +INFO: CANCEL PREORDER COUNT="52" +INFO: CANCEL POSTORDER COUNT="51" +INFO: ============================= +INFO: ============================= +INFO: EXPRESSION="sched-now=allocated" +INFO: ============================= + ---------------core0[1:x] + ---------------core1[1:x] + ---------------core2[1:x] + ---------------core3[1:x] + ---------------core4[1:x] + ---------------core5[1:x] + ---------------core6[1:x] + ---------------core7[1:x] + ---------------core8[1:x] + ---------------core9[1:x] + ---------------core10[1:x] + ---------------core11[1:x] + ---------------core12[1:x] + ---------------core13[1:x] + ---------------core14[1:x] + ---------------core15[1:x] + ---------------core16[1:x] + ---------------core17[1:x] + ---------------gpu0[1:x] + ---------------memory0[2:x] + ---------------memory1[2:x] + ---------------memory2[2:x] + ---------------memory3[2:x] + ------------socket0[1:x] + ---------------core18[1:x] + ---------------core19[1:x] + ---------------core20[1:x] + ---------------core21[1:x] + ---------------core22[1:x] + ---------------core23[1:x] + ---------------core24[1:x] + ---------------core25[1:x] + ---------------core26[1:x] + ---------------core27[1:x] + ---------------core28[1:x] + ---------------core29[1:x] + ---------------core30[1:x] + ---------------core31[1:x] + ---------------core32[1:x] + ---------------core33[1:x] + ---------------core34[1:x] + ---------------core35[1:x] + ---------------gpu1[1:x] + ---------------memory4[2:x] + ---------------memory5[2:x] + ---------------memory6[2:x] + ---------------memory7[2:x] + ------------socket1[1:x] + ---------node0[1:x] + ---------------core0[1:x] + ---------------core1[1:x] + ---------------core2[1:x] + ---------------core3[1:x] + ---------------core4[1:x] + ---------------core5[1:x] + ---------------core6[1:x] + ---------------core7[1:x] + ---------------core8[1:x] + ---------------core9[1:x] + ---------------core10[1:x] + ---------------core11[1:x] + ---------------core12[1:x] + ---------------core13[1:x] + ---------------core14[1:x] + ---------------core15[1:x] + ---------------core16[1:x] + ---------------core17[1:x] + ---------------gpu0[1:x] + ---------------memory0[2:x] + ---------------memory1[2:x] + ---------------memory2[2:x] + ---------------memory3[2:x] + ------------socket0[1:x] + ---------------core18[1:x] + ---------------core19[1:x] + ---------------core20[1:x] + ---------------core21[1:x] + ---------------core22[1:x] + ---------------core23[1:x] + ---------------core24[1:x] + ---------------core25[1:x] + ---------------core26[1:x] + ---------------core27[1:x] + ---------------core28[1:x] + ---------------core29[1:x] + ---------------core30[1:x] + ---------------core31[1:x] + ---------------core32[1:x] + ---------------core33[1:x] + ---------------core34[1:x] + ---------------core35[1:x] + ---------------gpu1[1:x] + ---------------memory4[2:x] + ---------------memory5[2:x] + ---------------memory6[2:x] + ---------------memory7[2:x] + ------------socket1[1:x] + ---------node1[1:x] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=2 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= + ---------------core0[1:x] + ---------------core1[1:x] + ---------------core2[1:x] + ---------------core3[1:x] + ---------------core4[1:x] + ---------------core5[1:x] + ---------------core6[1:x] + ---------------core7[1:x] + ---------------core8[1:x] + ---------------core9[1:x] + ---------------core10[1:x] + ---------------core11[1:x] + ---------------core12[1:x] + ---------------core13[1:x] + ---------------core14[1:x] + ---------------core15[1:x] + ---------------core16[1:x] + ---------------core17[1:x] + ---------------gpu0[1:x] + ---------------memory0[2:x] + ---------------memory1[2:x] + ---------------memory2[2:x] + ---------------memory3[2:x] + ------------socket0[1:x] + ---------------core18[1:x] + ---------------core19[1:x] + ---------------core20[1:x] + ---------------core21[1:x] + ---------------core22[1:x] + ---------------core23[1:x] + ---------------core24[1:x] + ---------------core25[1:x] + ---------------core26[1:x] + ---------------core27[1:x] + ---------------core28[1:x] + ---------------core29[1:x] + ---------------core30[1:x] + ---------------core31[1:x] + ---------------core32[1:x] + ---------------core33[1:x] + ---------------core34[1:x] + ---------------core35[1:x] + ---------------gpu1[1:x] + ---------------memory4[2:x] + ---------------memory5[2:x] + ---------------memory6[2:x] + ---------------memory7[2:x] + ------------socket1[1:x] + ---------node0[1:x] + ---------------core0[1:x] + ---------------core1[1:x] + ---------------core2[1:x] + ---------------core3[1:x] + ---------------core4[1:x] + ---------------core5[1:x] + ---------------core6[1:x] + ---------------core7[1:x] + ---------------core8[1:x] + ---------------core9[1:x] + ---------------core10[1:x] + ---------------core11[1:x] + ---------------core12[1:x] + ---------------core13[1:x] + ---------------core14[1:x] + ---------------core15[1:x] + ---------------core16[1:x] + ---------------core17[1:x] + ---------------gpu0[1:x] + ---------------memory0[2:x] + ---------------memory1[2:x] + ---------------memory2[2:x] + ---------------memory3[2:x] + ------------socket0[1:x] + ---------------core18[1:x] + ---------------core19[1:x] + ---------------core20[1:x] + ---------------core21[1:x] + ---------------core22[1:x] + ---------------core23[1:x] + ---------------core24[1:x] + ---------------core25[1:x] + ---------------core26[1:x] + ---------------core27[1:x] + ---------------core28[1:x] + ---------------core29[1:x] + ---------------core30[1:x] + ---------------core31[1:x] + ---------------core32[1:x] + ---------------core33[1:x] + ---------------core34[1:x] + ---------------core35[1:x] + ---------------gpu1[1:x] + ---------------memory4[2:x] + ---------------memory5[2:x] + ---------------memory6[2:x] + ---------------memory7[2:x] + ------------socket1[1:x] + ---------node1[1:x] + ------rack0[1:x] + ---tiny0[1:x] +INFO: ============================= +INFO: EXPRESSION="sched-now=allocated" +INFO: ============================= diff --git a/t/data/resource/rv1exec/cancel/rank0_cancel-jgfgraph.json b/t/data/resource/rv1exec/cancel/rank0_cancel-jgfgraph.json new file mode 100644 index 000000000..51a8292a7 --- /dev/null +++ b/t/data/resource/rv1exec/cancel/rank0_cancel-jgfgraph.json @@ -0,0 +1 @@ +{"version": 1, "execution": {"R_lite": [{"rank": "0", "children": {"core": "0-35", "gpu": "0-3", "memory": "0-7"}}], "nodelist": ["node0"], "starttime": 0, "expiration": 1000000}} diff --git a/t/t3008-resource-cancel.t b/t/t3008-resource-cancel.t index ef4ca6f0a..1f726235d 100755 --- a/t/t3008-resource-cancel.t +++ b/t/t3008-resource-cancel.t @@ -194,4 +194,21 @@ test_expect_success "${test023_desc}" ' test_cmp 022.R.out ${exp_dir}/021.R.out ' +cmds024="${cmd_dir}/cmds10.in" +test024_desc="test partial cancel and reallocation of two ranks; jgf" +test_expect_success "${test024_desc}" ' + sed "s~@TEST_SRCDIR@~${SHARNESS_TEST_SRCDIR}~g" ${cmds024} > cmds024 && + ${query} -f jgf -L ${jgfs} -S CA -P low -t 024.R.out < cmds024 && + test_cmp 024.R.out ${exp_dir}/024.R.out +' + +# Uses the same expected output as above +cmds025="${cmd_dir}/cmds11.in" +test025_desc="test partial cancel and reallocation of two ranks; jgf graph rv1exec partial cancel" +test_expect_success "${test025_desc}" ' + sed "s~@TEST_SRCDIR@~${SHARNESS_TEST_SRCDIR}~g" ${cmds025} > cmds025 && + ${query} -f jgf -L ${jgfs} -S CA -P low -t 025.R.out < cmds025 && + test_cmp 025.R.out ${exp_dir}/024.R.out +' + test_done