Merge pull request #271 from rem1776/run-fremake-exec-option

add --execute option to run-fremake

fre/make/fremake.py

@@ -66,12 +66,17 @@ def make_cli():
               "--no-parallel-checkout",
               is_flag = True,
               help = no_parallel_checkout_opt_help)
+@click.option("-e",
+              "--execute",
+              is_flag = True,
+              default = False,
+              help = "Use this to run the created compilation script.")
 @click.option("-v",
               "--verbose",
               is_flag = True,
               help = verbose_opt_help)
 @click.pass_context
-def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, verbose):
+def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, execute, verbose):
     """ - Perform all fremake functions to run checkout and compile model"""
     context.forward(runFremake._fremake_run)
 

fre/make/gfdlfremake/buildBaremetal.py

@@ -10,14 +10,14 @@
 def fremake_parallel(fremakeBuildList):
     """
     Brief: Called for parallel execution purposes.  Runs the builds.
-    Param: 
+    Param:
         - fremakeBuildList : fremakeBuild object list passes by pool.map
     """
     fremakeBuildList.run()
 
 class buildBaremetal():
     """
-    Brief: Creates the build script to compile the model 
+    Brief: Creates the build script to compile the model
     Param:
         - self : The buildScript object
         - exp  : The experiment name

fre/make/gfdlfremake/buildDocker.py

@@ -9,12 +9,12 @@
 class container():
     """
     Brief: Opens the Dockerfile for writing
-    Param: 
+    Param:
         - self : The dockerfile object
         - base : The docker base image to start from
         - libs : Additional libraries defined by user
         - exp : The experiment name
-        - RUNenv : The commands that have to be run at 
+        - RUNenv : The commands that have to be run at
                    the beginning of a RUN in the dockerfile
                    to set up the environment
     """
@@ -58,7 +58,7 @@ def __init__(self,base,exp,libs,RUNenv,target):
     def writeDockerfileCheckout(self, cScriptName, cOnDisk):
         """
         Brief: writes to the checkout part of the Dockerfile and sets up the compile
-        Param: 
+        Param:
             - self : The dockerfile object
             - cScriptName : The name of the checkout script in the container
             - cOnDisk : The relative path to the checkout script on disk
@@ -74,7 +74,7 @@ def writeDockerfileCheckout(self, cScriptName, cOnDisk):
     def writeDockerfileMakefile(self, makefileOnDiskPath, linklineonDiskPath):
         """
         Brief: Copies the Makefile into the bldDir in the dockerfile
-        Param: 
+        Param:
             - self : The dockerfile object
             - makefileOnDiskPath : The path to Makefile on the local disk
             - linklineonDiskPath : The path to the link line script on the local disk
@@ -98,8 +98,8 @@ def writeDockerfileMakefile(self, makefileOnDiskPath, linklineonDiskPath):
 
     def writeDockerfileMkmf(self, c):
         """
-        Brief: Adds components to the build part of the Dockerfile                    
-        Param: 
+        Brief: Adds components to the build part of the Dockerfile            
+        Param:
             - self : The dockerfile object
             - c : Component from the compile yaml
         """
@@ -141,14 +141,14 @@ def writeDockerfileMkmf(self, c):
     def writeRunscript(self,RUNenv,containerRun,runOnDisk):
         """
         Brief: Writes a runscript to set up spack loads/environment
-               in order to run the executable in the container; 
+               in order to run the executable in the container;
                runscript copied into container
-        Param: 
+        Param:
             - self : The dockerfile object
-            - RUNEnv : The commands that have to be run at 
+            - RUNEnv : The commands that have to be run at
                        the beginning of a RUN in the dockerfile
-            - containerRun : The container platform used with `exec` 
-                             to run the container; apptainer 
+            - containerRun : The container platform used with `exec`
+                             to run the container; apptainer
                              or singularity used
             - runOnDisk : The path to the run script on the local disk
         """
@@ -184,17 +184,25 @@ def writeRunscript(self,RUNenv,containerRun,runOnDisk):
         self.d.write('ENTRYPOINT ["/bin/bash"]')
         self.d.close()
 
-    def build(self,containerBuild,containerRun):
+    def createBuildScript(self,containerBuild,containerRun):
         """
-        Brief: Builds the container image for the model
-        Param: 
+        Brief: Writes out the build commands for the created dockerfile in a script,
+               which builds the dockerfile and then converts the format to a singularity image file.
+        Param:
             - self : The dockerfile object
-            - containerBuild : The tool used to build the container; 
+            - containerBuild : The tool used to build the container;
                                docker or podman used
-            - containerRun : The container platform used with `exec` to 
+            - containerRun : The container platform used with `exec` to
                              run the container; apptainer or singularity used
         """
-        os.system(containerBuild+" build -f Dockerfile -t "+self.e+":"+self.target.gettargetName())
-        os.system("rm -f "+self.e+".tar "+self.e+".sif")
-        os.system(containerBuild+" save -o "+self.e+"-"+self.target.gettargetName()+".tar localhost/"+self.e+":"+self.target.gettargetName())
-        os.system(containerRun+" build --disable-cache "+self.e+"-"+self.target.gettargetName()+".sif docker-archive://"+self.e+"-"+self.target.gettargetName()+".tar")
+        self.userScript = ["#!/bin/bash\n"]
+        self.userScript.append(containerBuild+" build -f Dockerfile -t "+self.e+":"+self.target.gettargetName()+"\n")
+        self.userScript.append("rm -f "+self.e+".tar "+self.e+".sif\n")
+        self.userScript.append(containerBuild+" save -o "+self.e+"-"+self.target.gettargetName()+".tar localhost/"+self.e+":"+self.target.gettargetName()+"\n")
+        self.userScript.append(containerRun+" build --disable-cache "+self.e+"-"+self.target.gettargetName()+".sif docker-archive://"+self.e+"-"+self.target.gettargetName()+".tar\n")
+        self.userScriptFile = open("createContainer.sh","w")
+        self.userScriptFile.writelines(self.userScript)
+        self.userScriptFile.close()
+        os.chmod("createContainer.sh", 0o744)
+        self.userScriptPath = os.getcwd()+"/createContainer.sh"
+

fre/make/runFremake.py

@@ -10,12 +10,13 @@
 from multiprocessing.dummy import Pool
 from pathlib import Path
 import click
+import subprocess
 import fre.yamltools.combine_yamls as cy
 from .gfdlfremake import (
     targetfre, varsfre, yamlfre, checkout,
     makefilefre, buildDocker, buildBaremetal )
 
-def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose):
+def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose):
     ''' run fremake via click'''
     yml = yamlfile
     name = yamlfile.split(".")[0]
@@ -87,6 +88,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb
                 freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc)
                 freCheckout.finish(pc)
                 os.chmod(srcDir+"/checkout.sh", 0o744)
+                print("\nCheckout script created at "+ srcDir + "/checkout.sh \n")
                 ## TODO: Options for running on login cluster?
                 freCheckout.run()
 
@@ -126,6 +128,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb
                 # Loop through components, send component name/requires/overrides for Makefile
                 for c in fremakeYaml['src']:
                     freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides'])
+                print("\nMakefile created at " + bldDir + "/Makefile" + "\n")
                 freMakefile.writeMakefile()
 
                 ## Create a list of compile scripts to run in parallel
@@ -142,8 +145,11 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb
                     fremakeBuild.writeBuildComponents(c)
                 fremakeBuild.writeScript()
                 fremakeBuildList.append(fremakeBuild)
-                ## Run the build
-                fremakeBuild.run()
+                ## Run the build if --execute option given, otherwise print out compile script path
+                if execute:
+                    fremakeBuild.run()
+                else:
+                    print("Compile script created at "+ bldDir+"/compile.sh\n\n")
             else:
                 ###################### container stuff below #######################################
                 ## Run the checkout script
@@ -187,26 +193,32 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb
 
                 dockerBuild.writeRunscript(RUNenv,containerRun,tmpDir+"/execrunscript.sh")
 
-                ## Run the dockerfile; build the container
-                dockerBuild.build(containerBuild,containerRun)
+                # Create build script for container
+                dockerBuild.createBuildScript(containerBuild, containerRun)
+                print("Container build script created at "+dockerBuild.userScriptPath+"\n\n")
+
+                # Execute if flag is given
+                if execute:
+                    subprocess.run(args=[dockerBuild.userScriptPath], check=True)
 
                 #freCheckout.cleanup()
                 #buildDockerfile(fremakeYaml,image)
 
     if baremetalRun:
         if __name__ == '__main__':
-            # Create a multiprocessing Pool
-            pool = Pool(processes=nparallel)
-            # process data_inputs iterable with pool
-            pool.map(buildBaremetal.fremake_parallel,fremakeBuildList)
+            if execute:
+                # Create a multiprocessing Pool
+                pool = Pool(processes=nparallel)
+                # process data_inputs iterable with pool
+                pool.map(buildBaremetal.fremake_parallel,fremakeBuildList)
 
 @click.command()
-def _fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose):
+def _fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose):
     '''
     Decorator for calling _fremake_run - allows the decorated version
     of the function to be separate from the undecorated version
     '''
-    return fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose)
+    return fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose)
 
 if __name__ == "__main__":
     fremake_run()

fre/make/tests/compilation/test_fre_make_run_fremake.py

@@ -21,6 +21,6 @@
 @pytest.mark.skip(reason='failing: fix in development, see PR 275')
 def test_fre_make_run_fremake_null_model_serial_compile():
     ''' run fre make with run-fremake subcommand and build the null model experiment with gnu'''
-    runFremake.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, False)
+    runFremake.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, True, False)
     assert Path(f"{HOME_DIR}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/{EXPERIMENT}.x").exists()