Defer command expansion and fix used variable tracking

This commit defers command expansion until the make_experiments phase,
to allow other phases to modify the variable definitions used in
command expansion (for example, in `mpi_command`).

Additionally, this fixes an issue during the reset of used variables
that caused the dictionary to change while it was being iterated over.

lib/ramble/ramble/application.py

@@ -153,6 +153,7 @@ def __init__(self, file_path):
         self._exp_lock = None
         self._input_lock = None
         self._software_lock = None
+        self._experiment_graph = None
 
         self.hash_inventory = {
             "application_definition": None,
@@ -532,6 +533,9 @@ def build_used_variables(self, workspace):
 
         backup_variables = self.variables.copy()
 
+        self._define_commands(self._executable_graph, workspace.success_list)
+        self._define_formatted_executables()
+
         ########################
         # Define extra variables
         ########################
@@ -582,13 +586,19 @@ def build_used_variables(self, workspace):
         ############################
         # Reset variable definitions
         ############################
+        to_remove = set()
         for var in self.variables:
             if var not in backup_variables:
-                del self.variables[var]
+                to_remove.add(var)
+
+        for var in to_remove:
+            del self.variables[var]
 
         for var, val in backup_variables.items():
             self.variables[var] = val
 
+        self._command_list = []
+
         return self.expander._used_variables
 
     def print_internals(self, indent=""):
@@ -1230,11 +1240,9 @@ def add_expand_vars(self, workspace):
         """
         if not self._vars_are_expanded:
             self._validate_experiment()
-            exec_graph = self._get_executable_graph(self.expander.workload_name)
+            self._executable_graph = self._get_executable_graph(self.expander.workload_name)
             self._set_default_experiment_variables()
             self._set_input_path()
-            self._define_commands(exec_graph, workspace.success_list)
-            self._define_formatted_executables()
 
             self._derive_variables_for_template_path(workspace)
             self._vars_are_expanded = True
@@ -1420,6 +1428,9 @@ def _make_experiments(self, workspace, app_inst=None):
 
         exp_lock = self.experiment_lock()
 
+        self._define_commands(self._executable_graph, workspace.success_list)
+        self._define_formatted_executables()
+
         with lk.WriteTransaction(exp_lock):
             experiment_run_dir = self.expander.experiment_run_dir
             fs.mkdirp(experiment_run_dir)