Work on debugging the sandbox issue (#126)

* Add unittest that *should* fail with this issue

* Add error message to hopefully exactly where the sandbox error is occuring

* Ignore untestable code

* omg i hate coverage sometimes

* omg i missed one im crying

source/autograder_cli/build_autograder.py

@@ -154,8 +154,8 @@ def createFolders(self):
         if os.path.exists(self.binRoot):
             try:
                 shutil.rmtree(self.binRoot, ignore_errors=True)
-            except OSError:
-                print("WARN: Failed to clean bin directory")
+            except OSError:  # pragma: no coverage
+                print("WARN: Failed to clean bin directory")  # pragma: no coverage
 
         # create directories
         os.makedirs(self.generationDirectory, exist_ok=True)

source/autograder_platform/Executors/Executor.py

@@ -1,5 +1,6 @@
 import shutil
 import os
+import sys
 
 from autograder_platform.Executors.Environment import ExecutionEnvironment
 
@@ -16,11 +17,13 @@ class Executor:
     def setup(cls, environment: ExecutionEnvironment, runner: TaskRunner, autograderConfig: AutograderConfiguration) -> ISubmissionProcess:
         cls.cleanup(environment)
 
+        # we are temporarily suppressing the errors with file creation should they occur.
         try:
             # create the sandbox and ensure that we have RWX permissions
             os.mkdir(environment.SANDBOX_LOCATION)
-        except OSError as ex:
-            raise EnvironmentError(f"Failed to create sandbox for test run. Error is: {ex}")
+        except OSError as ex:  # pragma: no coverage
+            # raise EnvironmentError(f"Failed to create sandbox for test run. Error is: {ex}")
+            print(f"ERROR: Failed to create sandbox folder.\n{ex}", file=sys.stderr)  # pragma: no coverage
 
         # TODO Logging
 
@@ -31,8 +34,8 @@ def setup(cls, environment: ExecutionEnvironment, runner: TaskRunner, autograder
                 try:
                     os.makedirs(os.path.dirname(dest), exist_ok=True)
                     shutil.copy(src, dest)
-                except OSError as ex:
-                    raise EnvironmentError(f"Failed to move file '{src}' to '{dest}'. Error is: {ex}")
+                except OSError as ex:  # pragma: no coverage
+                    raise EnvironmentError(f"Failed to move file '{src}' to '{dest}'. Error is: {ex}")  # pragma: no coverage
 
         return process
 
@@ -61,4 +64,7 @@ def postRun(cls, environment: ExecutionEnvironment,
     @classmethod
     def cleanup(cls, environment: ExecutionEnvironment):
         if os.path.exists(environment.SANDBOX_LOCATION):
-            shutil.rmtree(environment.SANDBOX_LOCATION)
+            try:
+                shutil.rmtree(environment.SANDBOX_LOCATION)
+            except OSError as ex:  # pragma: no coverage
+                print(f"ERROR: Failed to remove sandbox folder.\n{ex}", file=sys.stderr)  # pragma: no coverage

source/autograder_platform/StudentSubmissionImpl/Python/PythonSubmissionProcess.py

@@ -116,7 +116,13 @@ def _setup(self) -> None:
 
         This method also injects whatever import MetaPathFinders
         """
-        os.chdir(self.executionDirectory)
+        # This may error? so we are going to catch it and log the error
+        try:
+            os.chdir(self.executionDirectory)
+        except OSError as ex:  # pragma: no coverage
+            print(f"ERROR: Failed to change directory to sandbox folder.\n{ex}", file=sys.stderr)  # pragma: no coverage
+
+
         sys.path.append(os.getcwd())
 
         for importHandler in self.importHandlers:

source/autograder_platform/__init__.py

@@ -1 +1 @@
-__version__ = "5.0.5"
+__version__ = "5.0.6"

tests/platform_tests/impl/python/testFullExecutions.py

@@ -432,3 +432,29 @@ def INJECTED_testClassTest1(test1Cls, expectedValue):
         Executor.execute(environment, runner)
 
         self.assertEqual(expected, getResults(environment).return_val)
+
+    def testVerifySandboxDeletedAfterTests(self):
+        program = "print('OUTPUT stuff')"
+        self.writePythonFile("submission.py", program)
+
+        self.writePythonFile(".keep", "")
+
+        submission = PythonSubmission()\
+            .setSubmissionRoot(self.PYTHON_PROGRAM_DIRECTORY)\
+            .load()\
+            .build()\
+            .validate()
+
+        environment = ExecutionEnvironmentBuilder()\
+            .addFile(os.path.join(self.PYTHON_PROGRAM_DIRECTORY, ".keep"), ".keep")\
+            .setTimeout(5)\
+            .build()
+
+        for _ in range(10):
+            runner = PythonRunnerBuilder(submission)\
+                .setEntrypoint(module=True)\
+                .build()
+            Executor.execute(environment, runner)
+
+
+