From 898ad1a8aaca4303dbcc3c9a8983a58247a5a491 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Tue, 3 Dec 2024 12:45:24 -0800
Subject: [PATCH 01/14] add module support for memLeaks testing

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 modules/packages/Python.chpl | 74 +++++++++++++++++++++++++++++++++---
 1 file changed, 68 insertions(+), 6 deletions(-)

diff --git a/modules/packages/Python.chpl b/modules/packages/Python.chpl
index 57bbec4290ac..6d163c018509 100644
--- a/modules/packages/Python.chpl
+++ b/modules/packages/Python.chpl
@@ -267,6 +267,8 @@ module Python {
   class Interpreter {
     @chpldoc.nodoc
     var converters: List.list(owned TypeConverter);
+    @chpldoc.nodoc
+    var objgraph: PyObjectPtr = nil;
 
     @chpldoc.nodoc
     proc init() throws {
@@ -339,9 +341,37 @@ module Python {
       // I think we can do this by setting sys.stdout and sys.stderr to a python
       // object that looks like a python file but forwards calls like write to
       // Chapel's write
+
+      if memLeaks {
+        // import objgraph
+        this.objgraph = this.importModule("objgraph");
+        if this.objgraph == nil {
+          writeln("objgraph not found, memory leak detection disabled. Install objgraph with 'pip install objgraph'");
+        } else {
+          // objgraph.growth()
+          var growth = PyObject_GetAttrString(this.objgraph, "growth");
+          this.checkException();
+          var res = PyObject_CallFunctionObjArgs(growth, Py_None, nil);
+          this.checkException();
+          Py_DECREF(res);
+          Py_DECREF(growth);
+        }
+      }
     }
     @chpldoc.nodoc
-    proc deinit() {
+    proc deinit()  {
+
+      if memLeaks && this.objgraph != nil {
+        // note: try! is used since we can't have a throwing deinit
+        // objgraph.show_growth()
+        var show_growth = PyObject_GetAttrString(this.objgraph, "show_growth");
+        try! this.checkException();
+        PyObject_CallFunctionObjArgs(show_growth, Py_None, nil);
+        try! this.checkException();
+        Py_DECREF(show_growth);
+        Py_DECREF(this.objgraph);
+      }
+
       Py_Finalize();
     }
     /*
@@ -407,12 +437,21 @@ module Python {
     inline proc checkException() throws {
       var exc = chpl_PyErr_GetRaisedException();
       if exc {
-        var py_str = PyObject_Str(exc);
-        var str = this.fromPython(string, py_str);
-        Py_DECREF(py_str);
-        Py_DECREF(exc);
-        throw new PythonException(str);
+        throw PythonException.build(this, exc);
+      }
+    }
+
+    @chpldoc.nodoc
+    inline proc importModule(in modName: string): PyObjectPtr throws {
+      var mod = PyImport_ImportModule(modName.c_str());
+      try {
+        this.checkException();
+      } catch e: ImportError {
+        return nil;
+      } catch e {
+        throw e;
       }
+      return mod;
     }
 
     /*
@@ -679,6 +718,27 @@ module Python {
     proc init(in message: string) {
       super.init(message);
     }
+    @chpldoc.nodoc
+    proc type build(interp: borrowed Interpreter, exc: PyObjectPtr): owned PythonException throws {
+      assert(exc != nil);
+      var py_str = PyObject_Str(exc);
+      var str = interp.fromPython(string, py_str);
+      Py_DECREF(py_str);
+      Py_DECREF(exc);
+      if PyErr_GivenExceptionMatches(exc, PyExc_ImportError) != 0 {
+        return new ImportError(str);
+      } else {
+        throw new PythonException(str);
+      }
+    }
+  }
+  /*
+    Represents an ImportError in the Python code
+  */
+  class ImportError: PythonException {
+    proc init(in message: string) {
+      super.init(message);
+    }
   }
   /*
     Represents an exception caused by code in the Chapel module, not forwarded from Python.
@@ -1420,6 +1480,8 @@ module Python {
     */
     extern proc PyErr_Occurred(): PyObjectPtr;
     extern proc chpl_PyErr_GetRaisedException(): PyObjectPtr;
+    extern proc PyErr_GivenExceptionMatches(given: PyObjectPtr, exc: PyObjectPtr): c_int;
+    extern const PyExc_ImportError: PyObjectPtr;
 
     /*
       Values

From 13703fb2468cfcc528adb87902c454b6fb952f2d Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Tue, 3 Dec 2024 12:45:42 -0800
Subject: [PATCH 02/14] copy env info into PRETEST

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 util/test/sub_test.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/util/test/sub_test.py b/util/test/sub_test.py
index fe47c0ada5cb..ba96e1bba98c 100755
--- a/util/test/sub_test.py
+++ b/util/test/sub_test.py
@@ -935,14 +935,15 @@ def run_compileline(flag, lookingfor):
 
     #
     # directory-wide PRETEST
-    # must be run first, in case is generates any of the following files
+    # must be run first, in case it generates any of the following files
     #
     if os.access('./PRETEST', os.R_OK|os.X_OK):
         sys.stdout.write('[Executing ./PRETEST %s]\n'%(compiler))
         sys.stdout.flush()
         stdout = run_process(['./PRETEST', compiler],
                              stdout=subprocess.PIPE,
-                             stderr=subprocess.STDOUT)[1]
+                             stderr=subprocess.STDOUT,
+                             env=os.environ)[1]
         sys.stdout.write(stdout)
         sys.stdout.flush()
 

From 1a5262a4e4f24842d652139dad0285c8a16a8bd3 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Tue, 3 Dec 2024 12:46:43 -0800
Subject: [PATCH 03/14] start adding support for memleaks testing

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 test/library/packages/Python/CLEANFILES       |  1 +
 test/library/packages/Python/EXECENV          | 15 ++++++++
 .../packages/Python/correctness/CLEANFILES    |  1 +
 .../packages/Python/correctness/COMPOPTS      |  1 +
 .../packages/Python/correctness/EXECENV       |  1 +
 .../packages/Python/correctness/PREDIFF       |  6 ++++
 .../packages/Python/correctness/PRETEST       |  8 +++++
 .../{ => correctness}/argPassingTest.chpl     |  0
 .../argPassingTest.comm-none.good             |  0
 .../{ => correctness}/argPassingTest.good     |  0
 .../{ => correctness}/argPassingTest.py       |  0
 .../Python/{ => correctness}/basicTypes.chpl  |  0
 .../Python/{ => correctness}/basicTypes.good  |  0
 .../{ => correctness}/basicTypes.prediff      |  0
 .../Python/{ => correctness}/basicTypes.py    |  0
 .../compareIterationPatterns.chpl             |  0
 .../compareIterationPatterns.good             |  0
 .../Python/{ => correctness}/customType.chpl  |  0
 .../customType.comm-none.good                 |  0
 .../Python/{ => correctness}/customType.good  |  0
 .../Python/{ => correctness}/customType.py    |  0
 .../{ => correctness}/iteratorTest.chpl       |  0
 .../{ => correctness}/iteratorTest.execopts   |  0
 .../{ => correctness}/iteratorTest.good       |  0
 .../{ => correctness}/iteratorTest.prediff    |  0
 .../Python/{ => correctness}/iteratorTest.py  |  0
 test/library/packages/Python/examples/EXECENV |  8 -----
 .../packages/Python/examples/bs4/.gitignore   |  1 -
 .../packages/Python/examples/bs4/CLEANFILES   |  2 +-
 .../packages/Python/examples/bs4/EXECENV      |  2 +-
 .../Python/examples/bs4/findLinks.skipif      | 16 ++-------
 .../packages/Python/examples/numba/.gitignore |  1 -
 .../packages/Python/examples/numba/CLEANFILES |  2 +-
 .../packages/Python/examples/numba/EXECENV    |  2 +-
 .../Python/examples/numba/apply.skipif        | 16 ++-------
 .../packages/Python/examples/torch/.gitignore |  1 -
 .../packages/Python/examples/torch/CLEANFILES |  2 +-
 .../packages/Python/examples/torch/EXECENV    |  2 +-
 .../Python/examples/torch/libs.notest         |  0
 .../Python/examples/torch/myModel.skipif      | 16 ++-------
 .../Python/skipIfAndInstallPackage.sh         | 36 +++++++++++++++++++
 41 files changed, 81 insertions(+), 59 deletions(-)
 create mode 100755 test/library/packages/Python/EXECENV
 create mode 120000 test/library/packages/Python/correctness/CLEANFILES
 create mode 120000 test/library/packages/Python/correctness/COMPOPTS
 create mode 120000 test/library/packages/Python/correctness/EXECENV
 create mode 100755 test/library/packages/Python/correctness/PREDIFF
 create mode 100755 test/library/packages/Python/correctness/PRETEST
 rename test/library/packages/Python/{ => correctness}/argPassingTest.chpl (100%)
 rename test/library/packages/Python/{ => correctness}/argPassingTest.comm-none.good (100%)
 rename test/library/packages/Python/{ => correctness}/argPassingTest.good (100%)
 rename test/library/packages/Python/{ => correctness}/argPassingTest.py (100%)
 rename test/library/packages/Python/{ => correctness}/basicTypes.chpl (100%)
 rename test/library/packages/Python/{ => correctness}/basicTypes.good (100%)
 rename test/library/packages/Python/{ => correctness}/basicTypes.prediff (100%)
 rename test/library/packages/Python/{ => correctness}/basicTypes.py (100%)
 rename test/library/packages/Python/{ => correctness}/compareIterationPatterns.chpl (100%)
 rename test/library/packages/Python/{ => correctness}/compareIterationPatterns.good (100%)
 rename test/library/packages/Python/{ => correctness}/customType.chpl (100%)
 rename test/library/packages/Python/{ => correctness}/customType.comm-none.good (100%)
 rename test/library/packages/Python/{ => correctness}/customType.good (100%)
 rename test/library/packages/Python/{ => correctness}/customType.py (100%)
 rename test/library/packages/Python/{ => correctness}/iteratorTest.chpl (100%)
 rename test/library/packages/Python/{ => correctness}/iteratorTest.execopts (100%)
 rename test/library/packages/Python/{ => correctness}/iteratorTest.good (100%)
 rename test/library/packages/Python/{ => correctness}/iteratorTest.prediff (100%)
 rename test/library/packages/Python/{ => correctness}/iteratorTest.py (100%)
 delete mode 100755 test/library/packages/Python/examples/EXECENV
 delete mode 100644 test/library/packages/Python/examples/bs4/.gitignore
 mode change 100644 => 120000 test/library/packages/Python/examples/bs4/CLEANFILES
 delete mode 100644 test/library/packages/Python/examples/numba/.gitignore
 mode change 100644 => 120000 test/library/packages/Python/examples/numba/CLEANFILES
 mode change 100644 => 120000 test/library/packages/Python/examples/torch/CLEANFILES
 create mode 100644 test/library/packages/Python/examples/torch/libs.notest
 create mode 100755 test/library/packages/Python/skipIfAndInstallPackage.sh

diff --git a/test/library/packages/Python/CLEANFILES b/test/library/packages/Python/CLEANFILES
index bee8a64b79a9..c50e32e60fd0 100644
--- a/test/library/packages/Python/CLEANFILES
+++ b/test/library/packages/Python/CLEANFILES
@@ -1 +1,2 @@
 __pycache__
+python_libs
diff --git a/test/library/packages/Python/EXECENV b/test/library/packages/Python/EXECENV
new file mode 100755
index 000000000000..27032ce79c05
--- /dev/null
+++ b/test/library/packages/Python/EXECENV
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# if CHPL_TEST_VENV_DIR is set and not none, make sure to set VIRTUAL_ENV to match
+if [ -n "$CHPL_TEST_VENV_DIR" ] && [ "$CHPL_TEST_VENV_DIR" != "none" ]; then
+  echo "VIRTUAL_ENV=$CHPL_TEST_VENV_DIR"
+fi
+
+# make sure to add the libs dir to the PYTHONPATH if it exists
+FILE_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
+MY_LIB_DIR=$FILE_DIR/python_libs
+if [ -d "$MY_LIB_DIR" ]; then
+  echo "PYTHONPATH=$MY_LIB_DIR:$PYTHONPATH"
+fi
+
+echo ""
diff --git a/test/library/packages/Python/correctness/CLEANFILES b/test/library/packages/Python/correctness/CLEANFILES
new file mode 120000
index 000000000000..da023cf323d6
--- /dev/null
+++ b/test/library/packages/Python/correctness/CLEANFILES
@@ -0,0 +1 @@
+../CLEANFILES
\ No newline at end of file
diff --git a/test/library/packages/Python/correctness/COMPOPTS b/test/library/packages/Python/correctness/COMPOPTS
new file mode 120000
index 000000000000..03322fa59628
--- /dev/null
+++ b/test/library/packages/Python/correctness/COMPOPTS
@@ -0,0 +1 @@
+../COMPOPTS
\ No newline at end of file
diff --git a/test/library/packages/Python/correctness/EXECENV b/test/library/packages/Python/correctness/EXECENV
new file mode 120000
index 000000000000..e8f0d4156706
--- /dev/null
+++ b/test/library/packages/Python/correctness/EXECENV
@@ -0,0 +1 @@
+../EXECENV
\ No newline at end of file
diff --git a/test/library/packages/Python/correctness/PREDIFF b/test/library/packages/Python/correctness/PREDIFF
new file mode 100755
index 000000000000..c76ce0f1626e
--- /dev/null
+++ b/test/library/packages/Python/correctness/PREDIFF
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# if objgraph is not installed, we should remove the objgraph warning message that may appear
+# note: this is in leiu of each test having a skipif that checks for objgraph, since thats a lot of duplicated files
+cat $2 | grep -v "objgraph not found, memory leak detection disabled." > $2.tmp
+mv $2.tmp $2
diff --git a/test/library/packages/Python/correctness/PRETEST b/test/library/packages/Python/correctness/PRETEST
new file mode 100755
index 000000000000..b026a87bf93f
--- /dev/null
+++ b/test/library/packages/Python/correctness/PRETEST
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+
+if [[ "$CHPL_MEM_LEAK_TESTING" == "true" ]]; then
+  # for memleaks testing, we need the objgraph package
+  FILE_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
+  $FILE_DIR/../skipIfAndInstallPackage.sh $FILE_DIR objgraph &>/dev/null
+fi
diff --git a/test/library/packages/Python/argPassingTest.chpl b/test/library/packages/Python/correctness/argPassingTest.chpl
similarity index 100%
rename from test/library/packages/Python/argPassingTest.chpl
rename to test/library/packages/Python/correctness/argPassingTest.chpl
diff --git a/test/library/packages/Python/argPassingTest.comm-none.good b/test/library/packages/Python/correctness/argPassingTest.comm-none.good
similarity index 100%
rename from test/library/packages/Python/argPassingTest.comm-none.good
rename to test/library/packages/Python/correctness/argPassingTest.comm-none.good
diff --git a/test/library/packages/Python/argPassingTest.good b/test/library/packages/Python/correctness/argPassingTest.good
similarity index 100%
rename from test/library/packages/Python/argPassingTest.good
rename to test/library/packages/Python/correctness/argPassingTest.good
diff --git a/test/library/packages/Python/argPassingTest.py b/test/library/packages/Python/correctness/argPassingTest.py
similarity index 100%
rename from test/library/packages/Python/argPassingTest.py
rename to test/library/packages/Python/correctness/argPassingTest.py
diff --git a/test/library/packages/Python/basicTypes.chpl b/test/library/packages/Python/correctness/basicTypes.chpl
similarity index 100%
rename from test/library/packages/Python/basicTypes.chpl
rename to test/library/packages/Python/correctness/basicTypes.chpl
diff --git a/test/library/packages/Python/basicTypes.good b/test/library/packages/Python/correctness/basicTypes.good
similarity index 100%
rename from test/library/packages/Python/basicTypes.good
rename to test/library/packages/Python/correctness/basicTypes.good
diff --git a/test/library/packages/Python/basicTypes.prediff b/test/library/packages/Python/correctness/basicTypes.prediff
similarity index 100%
rename from test/library/packages/Python/basicTypes.prediff
rename to test/library/packages/Python/correctness/basicTypes.prediff
diff --git a/test/library/packages/Python/basicTypes.py b/test/library/packages/Python/correctness/basicTypes.py
similarity index 100%
rename from test/library/packages/Python/basicTypes.py
rename to test/library/packages/Python/correctness/basicTypes.py
diff --git a/test/library/packages/Python/compareIterationPatterns.chpl b/test/library/packages/Python/correctness/compareIterationPatterns.chpl
similarity index 100%
rename from test/library/packages/Python/compareIterationPatterns.chpl
rename to test/library/packages/Python/correctness/compareIterationPatterns.chpl
diff --git a/test/library/packages/Python/compareIterationPatterns.good b/test/library/packages/Python/correctness/compareIterationPatterns.good
similarity index 100%
rename from test/library/packages/Python/compareIterationPatterns.good
rename to test/library/packages/Python/correctness/compareIterationPatterns.good
diff --git a/test/library/packages/Python/customType.chpl b/test/library/packages/Python/correctness/customType.chpl
similarity index 100%
rename from test/library/packages/Python/customType.chpl
rename to test/library/packages/Python/correctness/customType.chpl
diff --git a/test/library/packages/Python/customType.comm-none.good b/test/library/packages/Python/correctness/customType.comm-none.good
similarity index 100%
rename from test/library/packages/Python/customType.comm-none.good
rename to test/library/packages/Python/correctness/customType.comm-none.good
diff --git a/test/library/packages/Python/customType.good b/test/library/packages/Python/correctness/customType.good
similarity index 100%
rename from test/library/packages/Python/customType.good
rename to test/library/packages/Python/correctness/customType.good
diff --git a/test/library/packages/Python/customType.py b/test/library/packages/Python/correctness/customType.py
similarity index 100%
rename from test/library/packages/Python/customType.py
rename to test/library/packages/Python/correctness/customType.py
diff --git a/test/library/packages/Python/iteratorTest.chpl b/test/library/packages/Python/correctness/iteratorTest.chpl
similarity index 100%
rename from test/library/packages/Python/iteratorTest.chpl
rename to test/library/packages/Python/correctness/iteratorTest.chpl
diff --git a/test/library/packages/Python/iteratorTest.execopts b/test/library/packages/Python/correctness/iteratorTest.execopts
similarity index 100%
rename from test/library/packages/Python/iteratorTest.execopts
rename to test/library/packages/Python/correctness/iteratorTest.execopts
diff --git a/test/library/packages/Python/iteratorTest.good b/test/library/packages/Python/correctness/iteratorTest.good
similarity index 100%
rename from test/library/packages/Python/iteratorTest.good
rename to test/library/packages/Python/correctness/iteratorTest.good
diff --git a/test/library/packages/Python/iteratorTest.prediff b/test/library/packages/Python/correctness/iteratorTest.prediff
similarity index 100%
rename from test/library/packages/Python/iteratorTest.prediff
rename to test/library/packages/Python/correctness/iteratorTest.prediff
diff --git a/test/library/packages/Python/iteratorTest.py b/test/library/packages/Python/correctness/iteratorTest.py
similarity index 100%
rename from test/library/packages/Python/iteratorTest.py
rename to test/library/packages/Python/correctness/iteratorTest.py
diff --git a/test/library/packages/Python/examples/EXECENV b/test/library/packages/Python/examples/EXECENV
deleted file mode 100755
index daac37e1b728..000000000000
--- a/test/library/packages/Python/examples/EXECENV
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-
-# if CHPL_TEST_VENV_DIR is set and not none, make sure to set VIRTUAL_ENV to match
-if [ -n "$CHPL_TEST_VENV_DIR" ] && [ "$CHPL_TEST_VENV_DIR" != "none" ]; then
-  echo "VIRTUAL_ENV=$CHPL_TEST_VENV_DIR"
-else
-  echo ""
-fi
diff --git a/test/library/packages/Python/examples/bs4/.gitignore b/test/library/packages/Python/examples/bs4/.gitignore
deleted file mode 100644
index bee8a64b79a9..000000000000
--- a/test/library/packages/Python/examples/bs4/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-__pycache__
diff --git a/test/library/packages/Python/examples/bs4/CLEANFILES b/test/library/packages/Python/examples/bs4/CLEANFILES
deleted file mode 100644
index bee8a64b79a9..000000000000
--- a/test/library/packages/Python/examples/bs4/CLEANFILES
+++ /dev/null
@@ -1 +0,0 @@
-__pycache__
diff --git a/test/library/packages/Python/examples/bs4/CLEANFILES b/test/library/packages/Python/examples/bs4/CLEANFILES
new file mode 120000
index 000000000000..89371cd2ef5b
--- /dev/null
+++ b/test/library/packages/Python/examples/bs4/CLEANFILES
@@ -0,0 +1 @@
+../../CLEANFILES
\ No newline at end of file
diff --git a/test/library/packages/Python/examples/bs4/EXECENV b/test/library/packages/Python/examples/bs4/EXECENV
index e8f0d4156706..b6f5e6e9e2a7 120000
--- a/test/library/packages/Python/examples/bs4/EXECENV
+++ b/test/library/packages/Python/examples/bs4/EXECENV
@@ -1 +1 @@
-../EXECENV
\ No newline at end of file
+../../EXECENV
\ No newline at end of file
diff --git a/test/library/packages/Python/examples/bs4/findLinks.skipif b/test/library/packages/Python/examples/bs4/findLinks.skipif
index 76d1f6c0ad6b..fbd8f715ea99 100755
--- a/test/library/packages/Python/examples/bs4/findLinks.skipif
+++ b/test/library/packages/Python/examples/bs4/findLinks.skipif
@@ -1,16 +1,4 @@
 #!/usr/bin/env bash
 
-
-# respect CHPL_TEST_VENV_DIR if it is set and not none
-if [ -n "$CHPL_TEST_VENV_DIR" ] && [ "$CHPL_TEST_VENV_DIR" != "none" ]; then
-  chpl_python=$CHPL_TEST_VENV_DIR/bin/python3
-else
-  chpl_python=$($CHPL_HOME/util/config/find-python.sh)
-fi
-
-# try and import bs4, if it fails, skip the test
-if ! $chpl_python -c "import bs4" &>/dev/null; then
-  echo "True"
-else
-  echo "False"
-fi
+FILE_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
+$FILE_DIR/../skipIfAndInstallPackage.sh $FILE_DIR bs4
diff --git a/test/library/packages/Python/examples/numba/.gitignore b/test/library/packages/Python/examples/numba/.gitignore
deleted file mode 100644
index bee8a64b79a9..000000000000
--- a/test/library/packages/Python/examples/numba/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-__pycache__
diff --git a/test/library/packages/Python/examples/numba/CLEANFILES b/test/library/packages/Python/examples/numba/CLEANFILES
deleted file mode 100644
index bee8a64b79a9..000000000000
--- a/test/library/packages/Python/examples/numba/CLEANFILES
+++ /dev/null
@@ -1 +0,0 @@
-__pycache__
diff --git a/test/library/packages/Python/examples/numba/CLEANFILES b/test/library/packages/Python/examples/numba/CLEANFILES
new file mode 120000
index 000000000000..89371cd2ef5b
--- /dev/null
+++ b/test/library/packages/Python/examples/numba/CLEANFILES
@@ -0,0 +1 @@
+../../CLEANFILES
\ No newline at end of file
diff --git a/test/library/packages/Python/examples/numba/EXECENV b/test/library/packages/Python/examples/numba/EXECENV
index e8f0d4156706..b6f5e6e9e2a7 120000
--- a/test/library/packages/Python/examples/numba/EXECENV
+++ b/test/library/packages/Python/examples/numba/EXECENV
@@ -1 +1 @@
-../EXECENV
\ No newline at end of file
+../../EXECENV
\ No newline at end of file
diff --git a/test/library/packages/Python/examples/numba/apply.skipif b/test/library/packages/Python/examples/numba/apply.skipif
index 37bc3c2f2716..4c3f6b6cbbab 100755
--- a/test/library/packages/Python/examples/numba/apply.skipif
+++ b/test/library/packages/Python/examples/numba/apply.skipif
@@ -1,16 +1,4 @@
 #!/usr/bin/env bash
 
-
-# respect CHPL_TEST_VENV_DIR if it is set and not none
-if [ -n "$CHPL_TEST_VENV_DIR" ] && [ "$CHPL_TEST_VENV_DIR" != "none" ]; then
-  chpl_python=$CHPL_TEST_VENV_DIR/bin/python3
-else
-  chpl_python=$($CHPL_HOME/util/config/find-python.sh)
-fi
-
-# try and import numba, if it fails, skip the test
-if ! $chpl_python -c "import numba" &>/dev/null; then
-  echo "True"
-else
-  echo "False"
-fi
+FILE_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
+$FILE_DIR/../skipIfAndInstallPackage.sh $FILE_DIR numba
diff --git a/test/library/packages/Python/examples/torch/.gitignore b/test/library/packages/Python/examples/torch/.gitignore
index cb6ebb4bfdaf..3660321f3b83 100644
--- a/test/library/packages/Python/examples/torch/.gitignore
+++ b/test/library/packages/Python/examples/torch/.gitignore
@@ -1,2 +1 @@
 myModel.good
-__pycache__
diff --git a/test/library/packages/Python/examples/torch/CLEANFILES b/test/library/packages/Python/examples/torch/CLEANFILES
deleted file mode 100644
index bee8a64b79a9..000000000000
--- a/test/library/packages/Python/examples/torch/CLEANFILES
+++ /dev/null
@@ -1 +0,0 @@
-__pycache__
diff --git a/test/library/packages/Python/examples/torch/CLEANFILES b/test/library/packages/Python/examples/torch/CLEANFILES
new file mode 120000
index 000000000000..89371cd2ef5b
--- /dev/null
+++ b/test/library/packages/Python/examples/torch/CLEANFILES
@@ -0,0 +1 @@
+../../CLEANFILES
\ No newline at end of file
diff --git a/test/library/packages/Python/examples/torch/EXECENV b/test/library/packages/Python/examples/torch/EXECENV
index e8f0d4156706..b6f5e6e9e2a7 120000
--- a/test/library/packages/Python/examples/torch/EXECENV
+++ b/test/library/packages/Python/examples/torch/EXECENV
@@ -1 +1 @@
-../EXECENV
\ No newline at end of file
+../../EXECENV
\ No newline at end of file
diff --git a/test/library/packages/Python/examples/torch/libs.notest b/test/library/packages/Python/examples/torch/libs.notest
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/library/packages/Python/examples/torch/myModel.skipif b/test/library/packages/Python/examples/torch/myModel.skipif
index 77b5bdc58b8d..e44e71770058 100755
--- a/test/library/packages/Python/examples/torch/myModel.skipif
+++ b/test/library/packages/Python/examples/torch/myModel.skipif
@@ -1,16 +1,4 @@
 #!/usr/bin/env bash
 
-
-# respect CHPL_TEST_VENV_DIR if it is set and not none
-if [ -n "$CHPL_TEST_VENV_DIR" ] && [ "$CHPL_TEST_VENV_DIR" != "none" ]; then
-  chpl_python=$CHPL_TEST_VENV_DIR/bin/python3
-else
-  chpl_python=$($CHPL_HOME/util/config/find-python.sh)
-fi
-
-# try and import torch, if it fails, skip the test
-if ! $chpl_python -c "import torch" &>/dev/null; then
-  echo "True"
-else
-  echo "False"
-fi
+FILE_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
+$FILE_DIR/../skipIfAndInstallPackage.sh $FILE_DIR torch numpy
diff --git a/test/library/packages/Python/skipIfAndInstallPackage.sh b/test/library/packages/Python/skipIfAndInstallPackage.sh
new file mode 100755
index 000000000000..daf36d2a3e9e
--- /dev/null
+++ b/test/library/packages/Python/skipIfAndInstallPackage.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+INSTALL_DIR=$1
+shift
+PACKAGE=$1
+shift
+# technically this could be something like `-r requirements.txt` as well
+OTHER_PACKAGES=$@
+
+# respect CHPL_TEST_VENV_DIR if it is set and not none
+if [ -n "$CHPL_TEST_VENV_DIR" ] && [ "$CHPL_TEST_VENV_DIR" != "none" ]; then
+  chpl_python=$CHPL_TEST_VENV_DIR/bin/python3
+else
+  chpl_python=$($CHPL_HOME/util/config/find-python.sh)
+fi
+
+# try and import the first package, if it fails, try and install the packages
+if ! $chpl_python -c "import $PACKAGE" &>/dev/null; then
+
+  # if the SKIP_INSTALL env is set, then we should skip the installation
+  if [ ! -z "$SKIP_INSTALL" ]; then
+    echo "True"
+    exit 0
+  fi
+
+  MY_LIB_DIR=$INSTALL_DIR/python_libs
+  $chpl_python -m pip install $PACKAGE $OTHER_PACKAGES --target=$MY_LIB_DIR 2>&1
+  export PYTHONPATH=$MY_LIB_DIR:$PYTHONPATH
+  if ! $chpl_python -c "import $PACKAGE" &>/dev/null; then
+    echo "True"
+  else
+    echo "False"
+  fi
+else
+  echo "False"
+fi

From 1e5bda1e108ae41ad039cdb012e625a01d2a7c47 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Tue, 3 Dec 2024 14:27:17 -0800
Subject: [PATCH 04/14] suppress memory leaks for now

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 .../packages/Python/correctness/SUPPRESSIF    |  2 ++
 .../Python/skipIfAndInstallPackage.sh         |  6 ----
 util/test/sub_test.py                         | 30 +++++++++++++++++--
 3 files changed, 30 insertions(+), 8 deletions(-)
 create mode 100644 test/library/packages/Python/correctness/SUPPRESSIF

diff --git a/test/library/packages/Python/correctness/SUPPRESSIF b/test/library/packages/Python/correctness/SUPPRESSIF
new file mode 100644
index 000000000000..e081259409ec
--- /dev/null
+++ b/test/library/packages/Python/correctness/SUPPRESSIF
@@ -0,0 +1,2 @@
+# not all memory is currently cleaned up
+CHPL_MEM_LEAK_TESTING==true
diff --git a/test/library/packages/Python/skipIfAndInstallPackage.sh b/test/library/packages/Python/skipIfAndInstallPackage.sh
index daf36d2a3e9e..5911781b5941 100755
--- a/test/library/packages/Python/skipIfAndInstallPackage.sh
+++ b/test/library/packages/Python/skipIfAndInstallPackage.sh
@@ -17,12 +17,6 @@ fi
 # try and import the first package, if it fails, try and install the packages
 if ! $chpl_python -c "import $PACKAGE" &>/dev/null; then
 
-  # if the SKIP_INSTALL env is set, then we should skip the installation
-  if [ ! -z "$SKIP_INSTALL" ]; then
-    echo "True"
-    exit 0
-  fi
-
   MY_LIB_DIR=$INSTALL_DIR/python_libs
   $chpl_python -m pip install $PACKAGE $OTHER_PACKAGES --target=$MY_LIB_DIR 2>&1
   export PYTHONPATH=$MY_LIB_DIR:$PYTHONPATH
diff --git a/util/test/sub_test.py b/util/test/sub_test.py
index ba96e1bba98c..f5301ab09fb1 100755
--- a/util/test/sub_test.py
+++ b/util/test/sub_test.py
@@ -92,7 +92,7 @@
 # PRETEST: Script to execute before running the test. (arguments: <compiler
 #    executable>).
 # PERFNUMTRIALS: Number of trials to run for performance testing
-#
+# SUPPRESSIF: Suppress this test if certain environment conditions hold true
 #
 # TEST-SPECIFIC FILES:  These setting override or augment the directory-wide
 #  settings.  Unless otherwise specified, these files are named
@@ -1306,6 +1306,7 @@ def run_compileline(flag, lookingfor):
 
         # Get the list of files starting with 'test_filename.'
         test_filename_files = fnmatch.filter(dirlist, test_filename+'.*')
+        test_filename_files += fnmatch.filter(dirlist, 'SUPPRESSIF')
         # print test_filename_files, dirlist
 
         if (perftest and (test_filename_files.count(PerfTFile(test_filename,'keys'))==0) and
@@ -1353,12 +1354,13 @@ def run_compileline(flag, lookingfor):
         # Deal with these files
         do_not_test=False
         for f in test_filename_files:
+            name = os.path.basename(f)
             (root, suffix) = os.path.splitext(f)
             # sys.stdout.write("**** %s ****\n"%(f))
 
             # 'f' is of the form test_filename.SOMETHING.suffix,
             # not pertinent at the moment
-            if root != test_filename:
+            if root != test_filename and name != "SUPPRESSIF":
                 continue
 
             # Deal with these later
@@ -1421,6 +1423,30 @@ def run_compileline(flag, lookingfor):
                     do_not_test=True
                     break
 
+            elif (name == "SUPPRESSIF" and (os.access(f, os.R_OK))):
+                try:
+                    suppressme=False
+                    suppresstest=runSkipIf(f)
+                    if suppresstest.strip() != "False":
+                        suppressme = suppresstest.strip() == "True" or int(suppresstest) == 1
+                    if suppressme:
+                        suppressline = ""
+                        with open(f, 'r') as suppressfile:
+                            for line in suppressfile:
+                                line = line.strip()
+                                if (line.startswith("#") and
+                                    not line.startswith("#!")):
+                                    suppressline = line.replace('#','').strip()
+                                    break
+                        futuretest='Suppress (' + suppressline + ') '
+                except (ValueError, RuntimeError) as e:
+                    sys.stdout.write(str(e)+'\n')
+                    sys.stdout.write('[Error processing SUPPRESSIF file for %s]\n'
+                        % os.path.join(localdir, test_filename))
+                    printEndOfTestMsg(os.path.join(localdir, test_filename), 0.0)
+                    do_not_test=True
+                    break
+
             elif (suffix==timeoutsuffix and os.access(f, os.R_OK)):
                 fileTimeout = ReadIntegerValue(f, localdir)
                 if fileTimeout < defaultTimeout or fileTimeout > globalTimeout:

From 7716ad3abe2d4c53bec246c120d2be9f8e75eae8 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Tue, 3 Dec 2024 14:37:00 -0800
Subject: [PATCH 05/14] commit gitignore

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 test/library/packages/Python/.gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/library/packages/Python/.gitignore b/test/library/packages/Python/.gitignore
index bee8a64b79a9..c50e32e60fd0 100644
--- a/test/library/packages/Python/.gitignore
+++ b/test/library/packages/Python/.gitignore
@@ -1 +1,2 @@
 __pycache__
+python_libs

From 9467a1baec1995348aa918efbc857c4a08f92f0d Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Tue, 3 Dec 2024 16:47:18 -0800
Subject: [PATCH 06/14] fix skipifs

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 test/library/packages/Python/examples/bs4/findLinks.skipif | 2 +-
 test/library/packages/Python/examples/numba/apply.skipif   | 2 +-
 test/library/packages/Python/examples/torch/myModel.skipif | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/library/packages/Python/examples/bs4/findLinks.skipif b/test/library/packages/Python/examples/bs4/findLinks.skipif
index fbd8f715ea99..a44eca0fbf1a 100755
--- a/test/library/packages/Python/examples/bs4/findLinks.skipif
+++ b/test/library/packages/Python/examples/bs4/findLinks.skipif
@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 
 FILE_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
-$FILE_DIR/../skipIfAndInstallPackage.sh $FILE_DIR bs4
+$FILE_DIR/../../skipIfAndInstallPackage.sh $FILE_DIR bs4
diff --git a/test/library/packages/Python/examples/numba/apply.skipif b/test/library/packages/Python/examples/numba/apply.skipif
index 4c3f6b6cbbab..7941d63cea6b 100755
--- a/test/library/packages/Python/examples/numba/apply.skipif
+++ b/test/library/packages/Python/examples/numba/apply.skipif
@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 
 FILE_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
-$FILE_DIR/../skipIfAndInstallPackage.sh $FILE_DIR numba
+$FILE_DIR/../../skipIfAndInstallPackage.sh $FILE_DIR numba
diff --git a/test/library/packages/Python/examples/torch/myModel.skipif b/test/library/packages/Python/examples/torch/myModel.skipif
index e44e71770058..d8eb0a10bbbb 100755
--- a/test/library/packages/Python/examples/torch/myModel.skipif
+++ b/test/library/packages/Python/examples/torch/myModel.skipif
@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 
 FILE_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
-$FILE_DIR/../skipIfAndInstallPackage.sh $FILE_DIR torch numpy
+$FILE_DIR/../../skipIfAndInstallPackage.sh $FILE_DIR torch numpy

From 7819d24ec1895cc3c2be6f20d4973d512e44bbf2 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Tue, 3 Dec 2024 17:29:13 -0800
Subject: [PATCH 07/14] add more suppressif

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 test/library/packages/Python/doc-examples/SUPPRESSIF   | 2 ++
 test/library/packages/Python/examples/bs4/SUPPRESSIF   | 2 ++
 test/library/packages/Python/examples/numba/SUPPRESSIF | 2 ++
 test/library/packages/Python/examples/torch/SUPPRESSIF | 2 ++
 4 files changed, 8 insertions(+)
 create mode 100644 test/library/packages/Python/doc-examples/SUPPRESSIF
 create mode 100644 test/library/packages/Python/examples/bs4/SUPPRESSIF
 create mode 100644 test/library/packages/Python/examples/numba/SUPPRESSIF
 create mode 100644 test/library/packages/Python/examples/torch/SUPPRESSIF

diff --git a/test/library/packages/Python/doc-examples/SUPPRESSIF b/test/library/packages/Python/doc-examples/SUPPRESSIF
new file mode 100644
index 000000000000..740e3cd9216f
--- /dev/null
+++ b/test/library/packages/Python/doc-examples/SUPPRESSIF
@@ -0,0 +1,2 @@
+# this directory has not yet been enabled for memleaks testing with Python
+CHPL_MEM_LEAK_TESTING==true
diff --git a/test/library/packages/Python/examples/bs4/SUPPRESSIF b/test/library/packages/Python/examples/bs4/SUPPRESSIF
new file mode 100644
index 000000000000..740e3cd9216f
--- /dev/null
+++ b/test/library/packages/Python/examples/bs4/SUPPRESSIF
@@ -0,0 +1,2 @@
+# this directory has not yet been enabled for memleaks testing with Python
+CHPL_MEM_LEAK_TESTING==true
diff --git a/test/library/packages/Python/examples/numba/SUPPRESSIF b/test/library/packages/Python/examples/numba/SUPPRESSIF
new file mode 100644
index 000000000000..740e3cd9216f
--- /dev/null
+++ b/test/library/packages/Python/examples/numba/SUPPRESSIF
@@ -0,0 +1,2 @@
+# this directory has not yet been enabled for memleaks testing with Python
+CHPL_MEM_LEAK_TESTING==true
diff --git a/test/library/packages/Python/examples/torch/SUPPRESSIF b/test/library/packages/Python/examples/torch/SUPPRESSIF
new file mode 100644
index 000000000000..740e3cd9216f
--- /dev/null
+++ b/test/library/packages/Python/examples/torch/SUPPRESSIF
@@ -0,0 +1,2 @@
+# this directory has not yet been enabled for memleaks testing with Python
+CHPL_MEM_LEAK_TESTING==true

From c3cee0673d41433812a2dff656f047fbaf65fc1d Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Wed, 4 Dec 2024 10:12:57 -0800
Subject: [PATCH 08/14] cleanup more memory leaks

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 modules/packages/Python.chpl                  | 89 ++++++++++++++++---
 .../PythonHelper/ChapelPythonHelper.h         |  1 +
 2 files changed, 80 insertions(+), 10 deletions(-)

diff --git a/modules/packages/Python.chpl b/modules/packages/Python.chpl
index 6d163c018509..4d616ced1e37 100644
--- a/modules/packages/Python.chpl
+++ b/modules/packages/Python.chpl
@@ -227,6 +227,8 @@ module Python {
   private use CWChar;
   private use OS.POSIX only getenv;
 
+  config const pyMemLeaks = false;
+
   // TODO: this must be first to avoid use-before-def, but that makes it first in the docs
   // is there a way to avoid this?
   /* Represents the python NoneType */
@@ -268,6 +270,8 @@ module Python {
     @chpldoc.nodoc
     var converters: List.list(owned TypeConverter);
     @chpldoc.nodoc
+    var toFree: List.list(PyObjectPtr);
+    @chpldoc.nodoc
     var objgraph: PyObjectPtr = nil;
 
     @chpldoc.nodoc
@@ -342,7 +346,7 @@ module Python {
       // object that looks like a python file but forwards calls like write to
       // Chapel's write
 
-      if memLeaks {
+      if pyMemLeaks {
         // import objgraph
         this.objgraph = this.importModule("objgraph");
         if this.objgraph == nil {
@@ -361,14 +365,32 @@ module Python {
     @chpldoc.nodoc
     proc deinit()  {
 
-      if memLeaks && this.objgraph != nil {
+      for obj in this.toFree {
+        Py_CLEAR(c_ptrTo(obj));
+      }
+
+      if pyMemLeaks && this.objgraph != nil {
         // note: try! is used since we can't have a throwing deinit
+
+        // run gc.collect() before showing growth
+        var gc = PyImport_ImportModule("gc");
+        try! this.checkException();
+        var collect = PyObject_GetAttrString(gc, "collect");
+        try! this.checkException();
+        PyObject_CallNoArgs(collect);
+        try! this.checkException();
+        Py_DECREF(collect);
+        Py_DECREF(gc);
+
+
         // objgraph.show_growth()
         var show_growth = PyObject_GetAttrString(this.objgraph, "show_growth");
         try! this.checkException();
-        PyObject_CallFunctionObjArgs(show_growth, Py_None, nil);
+
+        PyObject_CallOneArg(show_growth, Py_None);
         try! this.checkException();
         Py_DECREF(show_growth);
+
         Py_DECREF(this.objgraph);
       }
 
@@ -550,7 +572,17 @@ module Python {
         this.checkException();
         return v;
       } else if isArrayType(t) {
-        return fromList(t, obj);
+        // we need to create a dummy array to determine the type
+        pragma "no init"
+        var dummy: t;
+
+        if dummy.rank == 1 && dummy.isDefaultRectangular() {
+          return fromList(t, obj);
+        } else if dummy.isAssociative() {
+          return fromDict(t, obj);
+        } else {
+          halt("Unsupported fromPython array type: '" + t:string + "'");
+        }
       } else if isSubtype(t, List.list) {
         return fromList(t, obj);
       } else if isSubtype(t, Function) {
@@ -574,6 +606,7 @@ module Python {
       where isArrayType(arr.type) && arr.rank == 1 && arr.isDefaultRectangular() {
       var pyList = PyList_New(arr.size);
       this.checkException();
+      this.toFree.pushBack(pyList);
       for i in 0..<arr.size {
         const idx = arr.domain.orderToIndex(i);
         PyList_SetItem(pyList, i, toPython(arr[idx]));
@@ -589,6 +622,7 @@ module Python {
     proc toList(l: List.list(?)): PyObjectPtr throws {
       var pyList = PyList_New(l.size);
       this.checkException();
+      this.toFree.pushBack(pyList);
       for i in 0..<l.size {
         PyList_SetItem(pyList, i, toPython(l(i)));
         this.checkException();
@@ -607,13 +641,16 @@ module Python {
         throw new ChapelException("Can only convert a sequence with a known size to an array");
 
       // if its a sequence with a known size, we can just iterate over it
-      var res: T = noinit;
+      var res: T;
       if PySequence_Size(obj) != res.size {
         throw new ChapelException("Size mismatch");
       }
       for i in 0..<res.size {
         const idx = res.domain.orderToIndex(i);
-        res[idx] = fromPython(res.eltType, PySequence_GetItem(obj, i));
+        var elm = PySequence_GetItem(obj, i);
+        this.checkException();
+        defer Py_DECREF(elm);
+        res[idx] = fromPython(res.eltType, elm);
         this.checkException();
       }
       return res;
@@ -631,6 +668,7 @@ module Python {
         for i in 0..<PySequence_Size(obj) {
           var item = PySequence_GetItem(obj, i);
           this.checkException();
+          defer Py_DECREF(item);
           res.pushBack(fromPython(T.eltType, item));
         }
         return res;
@@ -640,6 +678,7 @@ module Python {
         while true {
           var item = PyIter_Next(obj);
           this.checkException();
+          defer Py_DECREF(item);
           if item == nil {
             break;
           }
@@ -678,6 +717,7 @@ module Python {
       where isArrayType(arr.type) && arr.isAssociative() {
       var pyDict = PyDict_New();
       this.checkException();
+      this.toFree.pushBack(pyDict);
       for key in arr.domain {
         var pyKey = toPython(key);
         var pyValue = toPython(arr[key]);
@@ -692,9 +732,33 @@ module Python {
     /*
       Converts a python dictionary to an associative array
     */
-    // proc fromDict(type T, obj: PyObjectPtr): T throws {
-      // TODO
-    // }
+    proc fromDict(type T, obj: PyObjectPtr): T throws
+      where isArrayType(T) {
+
+      // no init causes segfaults :(
+      var dummy: T;
+
+      // rebuild the array with a modifiable domain
+      var dom = dummy.domain;
+      var arr: [dom] dummy.eltType;
+
+      type keyType = arr.idxType;
+      type valType = arr.eltType;
+      var keys = PyDict_Keys(obj);
+      defer Py_DECREF(keys);
+      this.checkException();
+      for i in 0..<PyList_Size(keys) {
+        var key = PyList_GetItem(keys, i);
+        this.checkException();
+        var val = PyDict_GetItem(obj, key);
+        this.checkException();
+
+        var keyVal = this.fromPython(keyType, key);
+        dom.add(keyVal);
+        arr[keyVal] = this.fromPython(valType, val);
+      }
+      return arr;
+    }
 
     // TODO: convert python dict to chapel map
 
@@ -911,6 +975,7 @@ module Python {
       else
         pyRes = PyObject_CallFunctionObjArgs(this.fn.get(), (...pyArgs), nil);
       interpreter.checkException();
+      interpreter.toFree.pushBack(pyRes);
 
       var res = interpreter.fromPython(retType, pyRes);
       return res;
@@ -918,6 +983,7 @@ module Python {
     proc this(type retType): retType throws {
       var pyRes = PyObject_CallNoArgs(this.fn.get());
       interpreter.checkException();
+      interpreter.toFree.pushBack(pyRes);
 
       var res = interpreter.fromPython(retType, pyRes);
       return res;
@@ -940,6 +1006,7 @@ module Python {
 
       var pyRes = PyObject_Call(this.fn.get(), pyArg, pyKwargs);
       interpreter.checkException();
+      interpreter.toFree.pushBack(pyRes);
 
       var res = interpreter.fromPython(retType, pyRes);
       return res;
@@ -959,12 +1026,12 @@ module Python {
 
       var pyRes = PyObject_Call(this.fn.get(), pyArg, pyKwargs);
       interpreter.checkException();
+      interpreter.toFree.pushBack(pyRes);
 
       var res = interpreter.fromPython(retType, pyRes);
       return res;
     }
 
-
     proc getAttr(type t, attr: string): t throws {
       var pyAttr = PyObject_GetAttrString(this.fn.get(), attr.c_str());
       interpreter.checkException();
@@ -1448,6 +1515,7 @@ module Python {
     extern proc Py_Finalize();
     extern proc Py_INCREF(obj: PyObjectPtr);
     extern "chpl_Py_DECREF" proc Py_DECREF(obj: PyObjectPtr);
+    extern "chpl_Py_CLEAR" proc Py_CLEAR(obj: c_ptr(PyObjectPtr));
     extern proc PyObject_Str(obj: PyObjectPtr): PyObjectPtr; // `str(obj)`
     extern proc PyImport_ImportModule(name: c_ptrConst(c_char)): PyObjectPtr;
 
@@ -1558,6 +1626,7 @@ module Python {
     extern proc PyDict_GetItemString(dict: PyObjectPtr,
                                      key: c_ptrConst(c_char)): PyObjectPtr;
     extern proc PyDict_Size(dict: PyObjectPtr): c_long;
+    extern proc PyDict_Keys(dict: PyObjectPtr): PyObjectPtr;
 
     extern proc PyObject_GetAttrString(obj: PyObjectPtr,
                                        name: c_ptrConst(c_char)): PyObjectPtr;
diff --git a/modules/packages/PythonHelper/ChapelPythonHelper.h b/modules/packages/PythonHelper/ChapelPythonHelper.h
index 1c2698f29c75..ca3d97cd99ff 100644
--- a/modules/packages/PythonHelper/ChapelPythonHelper.h
+++ b/modules/packages/PythonHelper/ChapelPythonHelper.h
@@ -53,6 +53,7 @@ static inline PyObject* chpl_PyErr_GetRaisedException(void) {
 }
 
 static inline void chpl_Py_DECREF(PyObject* o) { Py_DECREF(o); }
+static inline void chpl_Py_CLEAR(PyObject** o) { Py_CLEAR(*o); }
 
 static inline int chpl_PyList_Check(PyObject* o) { return PyList_Check(o); }
 static inline int chpl_PyGen_Check(PyObject* o) { return PyGen_Check(o); }

From d8701704fe346128fabdc6ee44f38319d6e97a0b Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Wed, 4 Dec 2024 10:13:11 -0800
Subject: [PATCH 09/14] improve memleaks testing

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 .../packages/Python/correctness/PREDIFF       |  6 --
 .../packages/Python/correctness/PRETEST       |  8 --
 .../packages/Python/correctness/SUPPRESSIF    |  2 -
 .../packages/Python/doc-examples/SUPPRESSIF   |  2 -
 .../packages/Python/examples/bs4/SUPPRESSIF   |  2 -
 .../packages/Python/examples/numba/SUPPRESSIF |  2 -
 .../packages/Python/examples/torch/SUPPRESSIF |  2 -
 .../packages/Python/memleaks/CLEANFILES       |  1 +
 .../library/packages/Python/memleaks/COMPOPTS |  1 +
 test/library/packages/Python/memleaks/EXECENV |  1 +
 .../library/packages/Python/memleaks/EXECOPTS |  1 +
 test/library/packages/Python/memleaks/PRETEST |  5 ++
 test/library/packages/Python/memleaks/README  |  8 ++
 .../packages/Python/memleaks/SUPPRESSIF       | 20 +++++
 .../packages/Python/memleaks/basicTypes.chpl  | 74 +++++++++++++++++++
 .../packages/Python/memleaks/basicTypes.good  |  0
 .../packages/Python/memleaks/lambdas.chpl     | 13 ++++
 .../packages/Python/memleaks/lambdas.good     |  3 +
 18 files changed, 127 insertions(+), 24 deletions(-)
 delete mode 100755 test/library/packages/Python/correctness/PREDIFF
 delete mode 100755 test/library/packages/Python/correctness/PRETEST
 delete mode 100644 test/library/packages/Python/correctness/SUPPRESSIF
 delete mode 100644 test/library/packages/Python/doc-examples/SUPPRESSIF
 delete mode 100644 test/library/packages/Python/examples/bs4/SUPPRESSIF
 delete mode 100644 test/library/packages/Python/examples/numba/SUPPRESSIF
 delete mode 100644 test/library/packages/Python/examples/torch/SUPPRESSIF
 create mode 120000 test/library/packages/Python/memleaks/CLEANFILES
 create mode 120000 test/library/packages/Python/memleaks/COMPOPTS
 create mode 120000 test/library/packages/Python/memleaks/EXECENV
 create mode 100644 test/library/packages/Python/memleaks/EXECOPTS
 create mode 100755 test/library/packages/Python/memleaks/PRETEST
 create mode 100644 test/library/packages/Python/memleaks/README
 create mode 100755 test/library/packages/Python/memleaks/SUPPRESSIF
 create mode 100644 test/library/packages/Python/memleaks/basicTypes.chpl
 create mode 100644 test/library/packages/Python/memleaks/basicTypes.good
 create mode 100644 test/library/packages/Python/memleaks/lambdas.chpl
 create mode 100644 test/library/packages/Python/memleaks/lambdas.good

diff --git a/test/library/packages/Python/correctness/PREDIFF b/test/library/packages/Python/correctness/PREDIFF
deleted file mode 100755
index c76ce0f1626e..000000000000
--- a/test/library/packages/Python/correctness/PREDIFF
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-# if objgraph is not installed, we should remove the objgraph warning message that may appear
-# note: this is in leiu of each test having a skipif that checks for objgraph, since thats a lot of duplicated files
-cat $2 | grep -v "objgraph not found, memory leak detection disabled." > $2.tmp
-mv $2.tmp $2
diff --git a/test/library/packages/Python/correctness/PRETEST b/test/library/packages/Python/correctness/PRETEST
deleted file mode 100755
index b026a87bf93f..000000000000
--- a/test/library/packages/Python/correctness/PRETEST
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-
-
-if [[ "$CHPL_MEM_LEAK_TESTING" == "true" ]]; then
-  # for memleaks testing, we need the objgraph package
-  FILE_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
-  $FILE_DIR/../skipIfAndInstallPackage.sh $FILE_DIR objgraph &>/dev/null
-fi
diff --git a/test/library/packages/Python/correctness/SUPPRESSIF b/test/library/packages/Python/correctness/SUPPRESSIF
deleted file mode 100644
index e081259409ec..000000000000
--- a/test/library/packages/Python/correctness/SUPPRESSIF
+++ /dev/null
@@ -1,2 +0,0 @@
-# not all memory is currently cleaned up
-CHPL_MEM_LEAK_TESTING==true
diff --git a/test/library/packages/Python/doc-examples/SUPPRESSIF b/test/library/packages/Python/doc-examples/SUPPRESSIF
deleted file mode 100644
index 740e3cd9216f..000000000000
--- a/test/library/packages/Python/doc-examples/SUPPRESSIF
+++ /dev/null
@@ -1,2 +0,0 @@
-# this directory has not yet been enabled for memleaks testing with Python
-CHPL_MEM_LEAK_TESTING==true
diff --git a/test/library/packages/Python/examples/bs4/SUPPRESSIF b/test/library/packages/Python/examples/bs4/SUPPRESSIF
deleted file mode 100644
index 740e3cd9216f..000000000000
--- a/test/library/packages/Python/examples/bs4/SUPPRESSIF
+++ /dev/null
@@ -1,2 +0,0 @@
-# this directory has not yet been enabled for memleaks testing with Python
-CHPL_MEM_LEAK_TESTING==true
diff --git a/test/library/packages/Python/examples/numba/SUPPRESSIF b/test/library/packages/Python/examples/numba/SUPPRESSIF
deleted file mode 100644
index 740e3cd9216f..000000000000
--- a/test/library/packages/Python/examples/numba/SUPPRESSIF
+++ /dev/null
@@ -1,2 +0,0 @@
-# this directory has not yet been enabled for memleaks testing with Python
-CHPL_MEM_LEAK_TESTING==true
diff --git a/test/library/packages/Python/examples/torch/SUPPRESSIF b/test/library/packages/Python/examples/torch/SUPPRESSIF
deleted file mode 100644
index 740e3cd9216f..000000000000
--- a/test/library/packages/Python/examples/torch/SUPPRESSIF
+++ /dev/null
@@ -1,2 +0,0 @@
-# this directory has not yet been enabled for memleaks testing with Python
-CHPL_MEM_LEAK_TESTING==true
diff --git a/test/library/packages/Python/memleaks/CLEANFILES b/test/library/packages/Python/memleaks/CLEANFILES
new file mode 120000
index 000000000000..da023cf323d6
--- /dev/null
+++ b/test/library/packages/Python/memleaks/CLEANFILES
@@ -0,0 +1 @@
+../CLEANFILES
\ No newline at end of file
diff --git a/test/library/packages/Python/memleaks/COMPOPTS b/test/library/packages/Python/memleaks/COMPOPTS
new file mode 120000
index 000000000000..03322fa59628
--- /dev/null
+++ b/test/library/packages/Python/memleaks/COMPOPTS
@@ -0,0 +1 @@
+../COMPOPTS
\ No newline at end of file
diff --git a/test/library/packages/Python/memleaks/EXECENV b/test/library/packages/Python/memleaks/EXECENV
new file mode 120000
index 000000000000..e8f0d4156706
--- /dev/null
+++ b/test/library/packages/Python/memleaks/EXECENV
@@ -0,0 +1 @@
+../EXECENV
\ No newline at end of file
diff --git a/test/library/packages/Python/memleaks/EXECOPTS b/test/library/packages/Python/memleaks/EXECOPTS
new file mode 100644
index 000000000000..41cebffdf75f
--- /dev/null
+++ b/test/library/packages/Python/memleaks/EXECOPTS
@@ -0,0 +1 @@
+--pyMemLeaks
diff --git a/test/library/packages/Python/memleaks/PRETEST b/test/library/packages/Python/memleaks/PRETEST
new file mode 100755
index 000000000000..0b253e64a0cd
--- /dev/null
+++ b/test/library/packages/Python/memleaks/PRETEST
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+# for memleaks testing, we need the objgraph package
+FILE_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
+$FILE_DIR/../skipIfAndInstallPackage.sh $FILE_DIR objgraph &>/dev/null
diff --git a/test/library/packages/Python/memleaks/README b/test/library/packages/Python/memleaks/README
new file mode 100644
index 000000000000..2f9e98c486b3
--- /dev/null
+++ b/test/library/packages/Python/memleaks/README
@@ -0,0 +1,8 @@
+
+Python is very leaky, the following code results in leftover references that we can't control
+```
+import foo
+del foo
+```
+
+Because of this, the testing we do here is mostly to make sure any python objects we create we cleanup
diff --git a/test/library/packages/Python/memleaks/SUPPRESSIF b/test/library/packages/Python/memleaks/SUPPRESSIF
new file mode 100755
index 000000000000..632a312dbe8a
--- /dev/null
+++ b/test/library/packages/Python/memleaks/SUPPRESSIF
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# pymemleaks testing requires objgraph
+
+# respect CHPL_TEST_VENV_DIR if it is set and not none
+if [ -n "$CHPL_TEST_VENV_DIR" ] && [ "$CHPL_TEST_VENV_DIR" != "none" ]; then
+  chpl_python=$CHPL_TEST_VENV_DIR/bin/python3
+else
+  chpl_python=$($CHPL_HOME/util/config/find-python.sh)
+fi
+
+FILE_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
+export PYTHONPATH=$FILE_DIR/python_libs:$PYTHONPATH
+
+# try and import objgraph
+if ! $chpl_python -c "import objgraph" &>/dev/null; then
+  echo "True"
+else
+  echo "False"
+fi
diff --git a/test/library/packages/Python/memleaks/basicTypes.chpl b/test/library/packages/Python/memleaks/basicTypes.chpl
new file mode 100644
index 000000000000..08f327a61771
--- /dev/null
+++ b/test/library/packages/Python/memleaks/basicTypes.chpl
@@ -0,0 +1,74 @@
+use Python, List;
+
+config const print = false;
+var interp = new Interpreter();
+
+proc isEqual(A, B) {
+  if isArray(A) && isArray(B) {
+    if A.size != B.size {
+      return false;
+    }
+    if A.isAssociative() {
+      var same = true;
+      forall (ai, bi) in zip(A.domain, B.domain) with (&& reduce same) {
+        same = same && isEqual(ai, bi) && isEqual(A[ai], B[bi]);
+      }
+      return same;
+    } else {
+      var same = true;
+      forall (a, b) in zip(A, B) with (&& reduce same) do
+        same = same && isEqual(a, b);
+      return same;
+    }
+  } else {
+    return A == B;
+  }
+}
+
+proc testType(type t, value: t) {
+  if print then writeln("  type: ", t:string, " value: ", value);
+  var res = interp.toPython(value);
+  if print then writeln("    res: ", res);
+  const from = interp.fromPython(t, res);
+  if !isEqual(value, from) {
+    writeln("mismatch: ", value, " != ", from);
+  }
+}
+
+proc main() {
+
+  testType(uint(16), 42);
+  testType(int, 100);
+  testType(real(32), 3.14);
+  testType(string, "this is a decently long string that may require extra work");
+
+  { // int arrays
+    var arr: [0..<10] int = 0..<10;
+    testType(arr.type, arr);
+
+    var l = new List.list(arr);
+    testType(l.type, l);
+
+    var l2 = new List.list(list(int));
+    l2.pushBack(l); l2.pushBack(l);
+    testType(l2.type, l2);
+  }
+
+  { // string arrays
+    var strArr = [i in 0..<100] i:string;
+    testType(strArr.type, strArr);
+
+    var l = new List.list(strArr);
+    testType(l.type, l);
+
+    var l2 = new List.list(list(string));
+    l2.pushBack(l); l2.pushBack(l);
+    testType(l2.type, l2);
+  }
+
+  { // assoc arrays
+    var assoc = ["foo" => "bar", "baz" => "qux"];
+    testType(assoc.type, assoc);
+  }
+
+}
diff --git a/test/library/packages/Python/memleaks/basicTypes.good b/test/library/packages/Python/memleaks/basicTypes.good
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/test/library/packages/Python/memleaks/lambdas.chpl b/test/library/packages/Python/memleaks/lambdas.chpl
new file mode 100644
index 000000000000..2920ea97c718
--- /dev/null
+++ b/test/library/packages/Python/memleaks/lambdas.chpl
@@ -0,0 +1,13 @@
+use Python, List;
+
+proc main() {
+
+  var interp = new Interpreter();
+
+  var l = new Function(interp, "lambda x,: [x] + [x]");
+  writeln(l(list(int), 1));
+  writeln(l(list(string), "hi"));
+
+  // this has a an extra lis
+  writeln(l(list(list(int)), [1, 2, 3]));
+}
diff --git a/test/library/packages/Python/memleaks/lambdas.good b/test/library/packages/Python/memleaks/lambdas.good
new file mode 100644
index 000000000000..cea54b84f82e
--- /dev/null
+++ b/test/library/packages/Python/memleaks/lambdas.good
@@ -0,0 +1,3 @@
+[1, 1]
+[hi, hi]
+[[1, 2, 3], [1, 2, 3]]

From 8d3b23fcd7a5edddd547e4acaf474c247b7ad732 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Wed, 4 Dec 2024 10:23:26 -0800
Subject: [PATCH 10/14] resolve todo

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 modules/packages/Python.chpl | 2 --
 1 file changed, 2 deletions(-)

diff --git a/modules/packages/Python.chpl b/modules/packages/Python.chpl
index 4d616ced1e37..644a077e0f57 100644
--- a/modules/packages/Python.chpl
+++ b/modules/packages/Python.chpl
@@ -34,8 +34,6 @@
 // for example, a List, Set, Dict, Tuple, etc.
 // these should provide native like operation, so `for i in pyList` should work
 
-// TODO: there are decrefs missing all over the place
-
 // TODO: using the Py*_Check, we may be able to avoid needing to specify the type of the return value
 
 // TODO: implement operators as dunder methods

From 5c0e566605f9b3bdd108cb83625b6cd5837196c7 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Wed, 4 Dec 2024 10:23:50 -0800
Subject: [PATCH 11/14] free pylist strong references at the end

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 modules/packages/Python.chpl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/packages/Python.chpl b/modules/packages/Python.chpl
index 644a077e0f57..f3d517ba40d3 100644
--- a/modules/packages/Python.chpl
+++ b/modules/packages/Python.chpl
@@ -647,7 +647,7 @@ module Python {
         const idx = res.domain.orderToIndex(i);
         var elm = PySequence_GetItem(obj, i);
         this.checkException();
-        defer Py_DECREF(elm);
+        this.toFree.pushBack(elm);
         res[idx] = fromPython(res.eltType, elm);
         this.checkException();
       }
@@ -666,7 +666,7 @@ module Python {
         for i in 0..<PySequence_Size(obj) {
           var item = PySequence_GetItem(obj, i);
           this.checkException();
-          defer Py_DECREF(item);
+          this.toFree.pushBack(item);
           res.pushBack(fromPython(T.eltType, item));
         }
         return res;
@@ -676,7 +676,7 @@ module Python {
         while true {
           var item = PyIter_Next(obj);
           this.checkException();
-          defer Py_DECREF(item);
+          this.toFree.pushBack(item);
           if item == nil {
             break;
           }

From 33713e2d14eb2c03822b398e318d151157b95c82 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Tue, 7 Jan 2025 14:12:38 -0800
Subject: [PATCH 12/14] simplify subtest

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 util/test/sub_test.py | 120 +++++++++++++++++-------------------------
 1 file changed, 48 insertions(+), 72 deletions(-)

diff --git a/util/test/sub_test.py b/util/test/sub_test.py
index f5301ab09fb1..7c0a14995015 100755
--- a/util/test/sub_test.py
+++ b/util/test/sub_test.py
@@ -855,7 +855,7 @@ def run_compileline(flag, lookingfor):
                                               stderr=subprocess.PIPE)
         result = result.rstrip()
         if returncode != 0:
-          Fatal('Cannot find ' + lookingfor)
+            Fatal('Cannot find ' + lookingfor)
         return result
 
     c_compiler = run_compileline('--compile', 'c compiler')
@@ -915,7 +915,7 @@ def run_compileline(flag, lookingfor):
     # CHPL_NETWORK_ATOMICS
     chplna=os.getenv('CHPL_NETWORK_ATOMICS','none').strip()
     chplnastr='.na-'+chplna
-    #sys.stdout.write('chplna=%s\n'%(chplna))
+    # sys.stdout.write('chplna=%s\n'%(chplna))
 
     # CHPL_LAUNCHER
     chpllauncher=os.getenv('CHPL_LAUNCHER','none').strip()
@@ -923,7 +923,7 @@ def run_compileline(flag, lookingfor):
     # CHPL_LOCALE_MODEL
     chpllm=os.getenv('CHPL_LOCALE_MODEL','flat').strip()
     chpllmstr='.lm-'+chpllm
-    #sys.stdout.write('lm=%s\n'%(chpllm))
+    # sys.stdout.write('lm=%s\n'%(chpllm))
 
     # CHPL_TASKS
     chpltasks=os.getenv('CHPL_TASKS', 'none').strip()
@@ -1064,7 +1064,6 @@ def run_compileline(flag, lookingfor):
         globalCatfiles=None
     # sys.stdout.write('globalCatfiles=%s\n'%(globalCatfiles))
 
-
     #
     # valgrind stuff
     #
@@ -1094,7 +1093,6 @@ def run_compileline(flag, lookingfor):
         valgrindbinopts = None
     # sys.stdout.write('valgrindbin=%s %s\n'%(valgrindbin, valgrindbinopts))
 
-
     #
     # Misc set up
     #
@@ -1169,7 +1167,7 @@ def run_compileline(flag, lookingfor):
     if envCompopts is not None:
         envCompopts = shlex.split(envCompopts)
     else:
-      envCompopts = []
+        envCompopts = []
 
     # Global CHPLDOCOPTS
     if os.access('./CHPLDOCOPTS', os.R_OK):
@@ -1399,31 +1397,7 @@ def run_compileline(flag, lookingfor):
                 if do_not_test:
                     break
 
-            elif (suffix=='.suppressif' and (os.access(f, os.R_OK))):
-                try:
-                    suppressme=False
-                    suppresstest=runSkipIf(f)
-                    if suppresstest.strip() != "False":
-                        suppressme = suppresstest.strip() == "True" or int(suppresstest) == 1
-                    if suppressme:
-                        suppressline = ""
-                        with open('./'+test_filename+'.suppressif', 'r') as suppressfile:
-                            for line in suppressfile:
-                                line = line.strip()
-                                if (line.startswith("#") and
-                                    not line.startswith("#!")):
-                                    suppressline = line.replace('#','').strip()
-                                    break
-                        futuretest='Suppress (' + suppressline + ') '
-                except (ValueError, RuntimeError) as e:
-                    sys.stdout.write(str(e)+'\n')
-                    sys.stdout.write('[Error processing .suppressif file for %s]\n'
-                        % os.path.join(localdir, test_filename))
-                    printEndOfTestMsg(os.path.join(localdir, test_filename), 0.0)
-                    do_not_test=True
-                    break
-
-            elif (name == "SUPPRESSIF" and (os.access(f, os.R_OK))):
+            elif ((suffix=='.suppressif' or name == 'SUPPRESSIF') and (os.access(f, os.R_OK))):
                 try:
                     suppressme=False
                     suppresstest=runSkipIf(f)
@@ -1431,22 +1405,32 @@ def run_compileline(flag, lookingfor):
                         suppressme = suppresstest.strip() == "True" or int(suppresstest) == 1
                     if suppressme:
                         suppressline = ""
-                        with open(f, 'r') as suppressfile:
+                        suppress_file_name = (
+                            "./" + test_filename + ".suppressif"
+                            if name != "SUPPRESSIF"
+                            else f
+                        )
+                        with open(suppress_file_name, 'r') as suppressfile:
                             for line in suppressfile:
                                 line = line.strip()
-                                if (line.startswith("#") and
-                                    not line.startswith("#!")):
+                                is_comment = (line.startswith("#") and
+                                                not line.startswith("#!"))
+                                if is_comment:
                                     suppressline = line.replace('#','').strip()
                                     break
                         futuretest='Suppress (' + suppressline + ') '
                 except (ValueError, RuntimeError) as e:
                     sys.stdout.write(str(e)+'\n')
-                    sys.stdout.write('[Error processing SUPPRESSIF file for %s]\n'
-                        % os.path.join(localdir, test_filename))
+                    suppress_name = (
+                        ".suppressif" if name != "SUPPRESSIF" else "SUPPRESSIF"
+                    )
+                    sys.stdout.write(
+                        "[Error processing %s file for %s]\n"
+                        % (suppress_name, os.path.join(localdir, test_filename))
+                    )
                     printEndOfTestMsg(os.path.join(localdir, test_filename), 0.0)
                     do_not_test=True
                     break
-
             elif (suffix==timeoutsuffix and os.access(f, os.R_OK)):
                 fileTimeout = ReadIntegerValue(f, localdir)
                 if fileTimeout < defaultTimeout or fileTimeout > globalTimeout:
@@ -1593,7 +1577,7 @@ def run_compileline(flag, lookingfor):
                 usearg = ' '.join(useopt)
                 # But change all-spaces into single space.
                 if usearg.strip() == '':
-                  usearg = ' '
+                    usearg = ' '
                 usecompoptslist += [usearg]
         compoptslist = usecompoptslist
 
@@ -1608,7 +1592,6 @@ def run_compileline(flag, lookingfor):
         for var, val in [env.split('=', 1) for env in compenv]:
             testcompenv[var.strip()] = val.strip()
 
-
         # Get list of test specific exec options
         if os.access(test_filename+execoptssuffix, os.R_OK):
             execoptsfile=True
@@ -1674,7 +1657,6 @@ def run_compileline(flag, lookingfor):
                 sys.stdout.write(stdout)
                 sys.stdout.flush()
 
-
             #
             # Build the test program
             #
@@ -1803,12 +1785,12 @@ def run_compileline(flag, lookingfor):
 
             # remove some_file: output from C compilers
             if is_c_or_cpp_test:
-              for arg in args:
-                if arg.endswith(".c") or arg.endswith(".cpp"):
-                  # remove lines like
-                  # somefile.c:
-                  # that some C compilers emit when compiling multiple files
-                  output = output.replace(arg + ":\n", "")
+                for arg in args:
+                    if arg.endswith(".c") or arg.endswith(".cpp"):
+                        # remove lines like
+                        # somefile.c:
+                        # that some C compilers emit when compiling multiple files
+                        output = output.replace(arg + ":\n", "")
 
             if (status!=0 or not executebin):
                 # Save original output
@@ -1858,7 +1840,6 @@ def run_compileline(flag, lookingfor):
                     sys.stdout.write(stdout)
                     sys.stdout.flush()
 
-
                 # find the compiler .good file to compare against. The compiler
                 # .good file can be of the form testname.<configuration>.good or
                 # explicitname.<configuration>.good or
@@ -2029,10 +2010,9 @@ def run_compileline(flag, lookingfor):
                             if os.path.isfile(datFile):
                                 os.unlink(datFile)
 
-                #delete the timing file
+                # delete the timing file
                 cleanup(printpassesfile)
 
-
             if os.getenv('CHPL_COMPONLY'):
                 sys.stdout.write('[Note: Not executing or comparing the output due to -noexec flags]\n')
                 cleanup(execname)
@@ -2191,27 +2171,27 @@ def run_compileline(flag, lookingfor):
                 # sys.stdout.write("args=%s\n"%(args))
 
                 if len(args) >= 2 and '<' in args:
-                  redirIdx = args.index('<')
-                  execOptRedirect = args[redirIdx + 1]
-                  args.pop(redirIdx+1)
-                  args.pop(redirIdx)
-                  if redirectin == None:
-                      # It is a little unfortunate that we compile the test only to skip it here.
-                      # In order to prevent this, the logic for combining all the places execpopts
-                      # come from and checking for '<' would have to be factored out or duplicated
-                      print('[Skipping test with stdin redirection ("<") in execopts since '
+                    redirIdx = args.index('<')
+                    execOptRedirect = args[redirIdx + 1]
+                    args.pop(redirIdx+1)
+                    args.pop(redirIdx)
+                    if redirectin == None:
+                        # It is a little unfortunate that we compile the test only to skip it here.
+                        # In order to prevent this, the logic for combining all the places execpopts
+                        # come from and checking for '<' would have to be factored out or duplicated
+                        print('[Skipping test with stdin redirection ("<") in execopts since '
                             '-nostdinredirect is given {0}/{1}]'.format(localdir, test_filename))
-                      break
-                  elif redirectin == "/dev/null":
-                    if os.access(execOptRedirect, os.R_OK):
-                      redirectin = execOptRedirect
-                      redirectin_set_in_loop = True
+                        break
+                    elif redirectin == "/dev/null":
+                        if os.access(execOptRedirect, os.R_OK):
+                            redirectin = execOptRedirect
+                            redirectin_set_in_loop = True
+                        else:
+                            sys.stdout.write('[Error: redirection file %s does not exist]\n'%(execOptRedirect))
+                            break
                     else:
-                      sys.stdout.write('[Error: redirection file %s does not exist]\n'%(execOptRedirect))
-                      break
-                  else:
-                    sys.stdout.write('[Error: a redirection file already exists: %s]\n'%(redirectin))
-                    break
+                        sys.stdout.write('[Error: a redirection file already exists: %s]\n'%(redirectin))
+                        break
 
                 #
                 # Run program (with timeout)
@@ -2289,7 +2269,6 @@ def run_compileline(flag, lookingfor):
                                     subprocess.call(psCom + '| head -n 1', shell=True)
                                     subprocess.call(psCom + '| tail -n +2 | sort -r -k 3 | head -n 5', shell=True)
 
-
                         else:
                             if redirectin == None:
                                 my_stdin = None
@@ -2341,7 +2320,6 @@ def run_compileline(flag, lookingfor):
                     print('[Elapsed execution time for "{0}" - {1:.3f} '
                         'seconds]'.format(test_name, elapsedExecTime))
 
-
                     if execTimeWarnLimit and elapsedExecTime > execTimeWarnLimit:
                         sys.stdout.write('[Warning: %s/%s took over %.0f seconds to '
                             'execute]\n' %(localdir, test_filename, execTimeWarnLimit))
@@ -2457,7 +2435,6 @@ def run_compileline(flag, lookingfor):
                                     printTestVariation(compoptsnum, compoptslist)
                                     sys.stdout.write(']\n')
 
-
                     if perftest:
                         if not os.path.isdir(perfdir) and not os.path.isfile(perfdir):
                             py3_compat.makedirs(perfdir, exist_ok=True)
@@ -2519,7 +2496,6 @@ def run_compileline(flag, lookingfor):
         test_name = os.path.join(localdir, test_filename)
         printEndOfTestMsg(test_name, elapsedCurFileTestTime)
 
-
     sys.exit(0)
 
 if __name__ == '__main__':

From a66c40e1386405490a8918e07c505fbf02a8fb03 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Tue, 7 Jan 2025 14:12:51 -0800
Subject: [PATCH 13/14] remove catch throw

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 modules/packages/Python.chpl | 2 --
 1 file changed, 2 deletions(-)

diff --git a/modules/packages/Python.chpl b/modules/packages/Python.chpl
index f3d517ba40d3..12d52cbb7368 100644
--- a/modules/packages/Python.chpl
+++ b/modules/packages/Python.chpl
@@ -468,8 +468,6 @@ module Python {
         this.checkException();
       } catch e: ImportError {
         return nil;
-      } catch e {
-        throw e;
       }
       return mod;
     }

From ba2f73d598e9aa62c051b84b6430ae15cdf7cea5 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Tue, 7 Jan 2025 16:07:10 -0800
Subject: [PATCH 14/14] use internal helpers

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 modules/packages/Python.chpl | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/modules/packages/Python.chpl b/modules/packages/Python.chpl
index 12d52cbb7368..4d375ed8776d 100644
--- a/modules/packages/Python.chpl
+++ b/modules/packages/Python.chpl
@@ -731,12 +731,10 @@ module Python {
     proc fromDict(type T, obj: PyObjectPtr): T throws
       where isArrayType(T) {
 
-      // no init causes segfaults :(
-      var dummy: T;
-
       // rebuild the array with a modifiable domain
-      var dom = dummy.domain;
-      var arr: [dom] dummy.eltType;
+      var dom = chpl__domainFromArrayRuntimeType(T);
+      type eltType = chpl__eltTypeFromArrayRuntimeType(T);
+      var arr: [dom] eltType;
 
       type keyType = arr.idxType;
       type valType = arr.eltType;