Merge branch 'NOAA-GFDL:main' into createDockerfile-buildscript

fre/make/fremake.py

@@ -66,12 +66,17 @@ def make_cli():
               "--no-parallel-checkout",
               is_flag = True,
               help = no_parallel_checkout_opt_help)
+@click.option("-e",
+              "--execute",
+              is_flag = True,
+              default = False,
+              help = "Use this to run the created compilation script.")
 @click.option("-v",
               "--verbose",
               is_flag = True,
               help = verbose_opt_help)
 @click.pass_context
-def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, verbose):
+def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, execute, verbose):
     """ - Perform all fremake functions to run checkout and compile model"""
     context.forward(runFremake._fremake_run)
 

fre/make/gfdlfremake/buildBaremetal.py

@@ -10,14 +10,14 @@
 def fremake_parallel(fremakeBuildList):
     """
     Brief: Called for parallel execution purposes.  Runs the builds.
-    Param: 
+    Param:
         - fremakeBuildList : fremakeBuild object list passes by pool.map
     """
     fremakeBuildList.run()
 
 class buildBaremetal():
     """
-    Brief: Creates the build script to compile the model 
+    Brief: Creates the build script to compile the model
     Param:
         - self : The buildScript object
         - exp  : The experiment name
@@ -40,22 +40,22 @@ def __init__(self,exp,mkTemplatePath,srcDir,bldDir,target,modules,modulesInit,jo
         self.template = mkTemplatePath
         self.modules = ""
         for m in modules:
-            self.modules = self.modules +" "+ m
+            self.modules = f"{self.modules} {m}"
 
         ## Set up the top portion of the compile script
         self.setup=[   "#!/bin/sh -fx \n",
-                       "bld_dir="+self.bld+"/ \n",
-                       "src_dir="+self.src+"/ \n",
-                       "mkmf_template="+self.template+" \n"]
+                       f"bld_dir={self.bld}/ \n",
+                       f"src_dir={self.src}/ \n",
+                       f"mkmf_template={self.template} \n"]
         if self.modules != "":
             self.setup.extend(modulesInit) #extend - this is a list
-            self.setup.append("module load "+self.modules+" \n") # Append -this is a single string
+            self.setup.append(f"module load {self.modules} \n") # Append -this is a single string
 
         ## Create the build directory
-        os.system("mkdir -p "+self.bld)
+        os.system(f"mkdir -p {self.bld}")
 
         ## Create the compile script
-        self.f=open(self.bld+"/compile.sh","w")
+        self.f=open(f"{self.bld}/compile.sh","w")
         self.f.writelines(self.setup)
 
     def writeBuildComponents(self, c):
@@ -69,24 +69,30 @@ def writeBuildComponents(self, c):
         comp = c["component"]
 
         # Make the component directory
-        self.f.write("\n mkdir -p $bld_dir/"+comp+"\n")
+        self.f.write(f"\n mkdir -p $bld_dir/{comp}\n")
 
         # Get the paths needed for compiling
         pstring = ""
         for paths in c["paths"]:
             pstring = pstring+"$src_dir/"+paths+" "
 
         # Run list_paths
-        self.f.write(" list_paths -l -o $bld_dir/"+comp+"/pathnames_"+comp+" "+pstring+"\n")
-        self.f.write(" cd $bld_dir/"+comp+"\n")
+        self.f.write(f" list_paths -l -o $bld_dir/{comp}/pathnames_{comp} {pstring}\n")
+        self.f.write(f" cd $bld_dir/{comp}\n")
 
         # Create the mkmf line
         # If this lib doesnt have any code dependencies and
         # it requires the preprocessor (no -o and yes --use-cpp)
         if c["requires"] == [] and c["doF90Cpp"]:
-            self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir -p lib"+comp+".a -t $mkmf_template --use-cpp -c \""+c["cppdefs"]+"\" "+c["otherFlags"]+" $bld_dir/"+comp+"/pathnames_"+comp+" \n")
+            self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir "
+                         "-p lib"+comp+".a -t $mkmf_template --use-cpp "
+                         "-c \""+c["cppdefs"]+"\" "+c["otherFlags"]
+                         +" $bld_dir/"+comp+"/pathnames_"+comp+" \n")
         elif c["requires"] == []: # If this lib doesnt have any code dependencies (no -o)
-            self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir -p lib"+comp+".a -t $mkmf_template -c \""+c["cppdefs"]+"\" "+c["otherFlags"]+" $bld_dir/"+comp+"/pathnames_"+comp+" \n")
+            self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir "
+                         "-p lib"+comp+".a -t $mkmf_template -c \""
+                         +c["cppdefs"]+"\" "+c["otherFlags"]
+                         +" $bld_dir/"+comp+"/pathnames_"+comp+" \n")
         else: #Has requirements
             #Set up the requirements as a string to inclue after the -o
             reqstring = ""
@@ -95,9 +101,15 @@ def writeBuildComponents(self, c):
 
             #Figure out if we need the preprocessor
             if c["doF90Cpp"]:
-                self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir -p lib"+comp+".a -t $mkmf_template --use-cpp -c \""+c["cppdefs"]+"\" -o \""+reqstring+"\" "+c["otherFlags"]+" $bld_dir/"+comp+"/pathnames_"+comp+" \n")
+                self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir "
+                             "-p lib"+comp+".a -t $mkmf_template --use-cpp "
+                             "-c \""+c["cppdefs"]+"\" -o \""+reqstring+"\" "
+                             +c["otherFlags"]+" $bld_dir/"+comp+"/pathnames_"+comp+" \n")
             else:
-                self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir -p lib"+comp+".a -t $mkmf_template -c \""+c["cppdefs"]+"\" -o \""+reqstring+"\" "+c["otherFlags"]+" $bld_dir/"+comp+"/pathnames_"+comp+" \n")
+                self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir "
+                             "-p lib"+comp+".a -t $mkmf_template -c \""
+                             +c["cppdefs"]+"\" -o \""+reqstring+"\" "+c["otherFlags"]
+                             +" $bld_dir/"+comp+"/pathnames_"+comp+" \n")
 
 ##TODO: add targets input
     def writeScript(self):
@@ -106,8 +118,8 @@ def writeScript(self):
         Param:
             - self : The buildScript object
         """
-        self.f.write("cd "+self.bld+"\n")
-        self.f.write(self.make+"\n")
+        self.f.write(f"cd {self.bld}\n")
+        self.f.write(f"{self.make}\n")
         self.f.close()
 
         # Make compile script executable
@@ -120,10 +132,22 @@ def run(self):
         Param:
             - self : The dockerfile object
         """
-###### TODO make the Makefile
-        command = [self.bld+"/compile.sh","|","tee",self.bld+"/log.compile"]
-        try:
-            subprocess.run(args=command, check=True)
-        except:
-            print("There was an error running "+self.bld+"/compile.sh")
-            raise
+        command = [self.bld+"/compile.sh"]
+
+        # Run compile script
+        p1 = subprocess.Popen(command, stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
+
+        # Direct output to log file as well
+        p2 = subprocess.Popen(["tee",self.bld+"/log.compile"], stdin=p1.stdout)
+
+        # Allow process1 to receive SIGPIPE is process2 exits
+        p1.stdout.close()
+        p2.communicate()
+
+        # wait for process1 to finish before checking return code
+        p1.wait()
+        if p1.returncode != 0:
+            print(f"\nThere was an error running {self.bld}/compile.sh")
+            print(f"Check the log file: {self.bld}/log.compile")
+        else:
+            print(f"\nSuccessful run of {self.bld}/compile.sh")

fre/make/gfdlfremake/buildDocker.py

@@ -9,12 +9,12 @@
 class container():
     """
     Brief: Opens the Dockerfile for writing
-    Param: 
+    Param:
         - self : The dockerfile object
         - base : The docker base image to start from
         - libs : Additional libraries defined by user
         - exp : The experiment name
-        - RUNenv : The commands that have to be run at 
+        - RUNenv : The commands that have to be run at
                    the beginning of a RUN in the dockerfile
                    to set up the environment
     """
@@ -58,7 +58,7 @@ def __init__(self,base,exp,libs,RUNenv,target):
     def writeDockerfileCheckout(self, cScriptName, cOnDisk):
         """
         Brief: writes to the checkout part of the Dockerfile and sets up the compile
-        Param: 
+        Param:
             - self : The dockerfile object
             - cScriptName : The name of the checkout script in the container
             - cOnDisk : The relative path to the checkout script on disk
@@ -74,7 +74,7 @@ def writeDockerfileCheckout(self, cScriptName, cOnDisk):
     def writeDockerfileMakefile(self, makefileOnDiskPath, linklineonDiskPath):
         """
         Brief: Copies the Makefile into the bldDir in the dockerfile
-        Param: 
+        Param:
             - self : The dockerfile object
             - makefileOnDiskPath : The path to Makefile on the local disk
             - linklineonDiskPath : The path to the link line script on the local disk
@@ -98,8 +98,8 @@ def writeDockerfileMakefile(self, makefileOnDiskPath, linklineonDiskPath):
 
     def writeDockerfileMkmf(self, c):
         """
-        Brief: Adds components to the build part of the Dockerfile                    
-        Param: 
+        Brief: Adds components to the build part of the Dockerfile            
+        Param:
             - self : The dockerfile object
             - c : Component from the compile yaml
         """
@@ -141,14 +141,14 @@ def writeDockerfileMkmf(self, c):
     def writeRunscript(self,RUNenv,containerRun,runOnDisk):
         """
         Brief: Writes a runscript to set up spack loads/environment
-               in order to run the executable in the container; 
+               in order to run the executable in the container;
                runscript copied into container
-        Param: 
+        Param:
             - self : The dockerfile object
-            - RUNEnv : The commands that have to be run at 
+            - RUNEnv : The commands that have to be run at
                        the beginning of a RUN in the dockerfile
-            - containerRun : The container platform used with `exec` 
-                             to run the container; apptainer 
+            - containerRun : The container platform used with `exec`
+                             to run the container; apptainer
                              or singularity used
             - runOnDisk : The path to the run script on the local disk
         """
@@ -184,17 +184,25 @@ def writeRunscript(self,RUNenv,containerRun,runOnDisk):
         self.d.write('ENTRYPOINT ["/bin/bash"]')
         self.d.close()
 
-    def build(self,containerBuild,containerRun):
+    def createBuildScript(self,containerBuild,containerRun):
         """
-        Brief: Builds the container image for the model
-        Param: 
+        Brief: Writes out the build commands for the created dockerfile in a script,
+               which builds the dockerfile and then converts the format to a singularity image file.
+        Param:
             - self : The dockerfile object
-            - containerBuild : The tool used to build the container; 
+            - containerBuild : The tool used to build the container;
                                docker or podman used
-            - containerRun : The container platform used with `exec` to 
+            - containerRun : The container platform used with `exec` to
                              run the container; apptainer or singularity used
         """
-        os.system(containerBuild+" build -f Dockerfile -t "+self.e+":"+self.target.gettargetName())
-        os.system("rm -f "+self.e+".tar "+self.e+".sif")
-        os.system(containerBuild+" save -o "+self.e+"-"+self.target.gettargetName()+".tar localhost/"+self.e+":"+self.target.gettargetName())
-        os.system(containerRun+" build --disable-cache "+self.e+"-"+self.target.gettargetName()+".sif docker-archive://"+self.e+"-"+self.target.gettargetName()+".tar")
+        self.userScript = ["#!/bin/bash\n"]
+        self.userScript.append(containerBuild+" build -f Dockerfile -t "+self.e+":"+self.target.gettargetName()+"\n")
+        self.userScript.append("rm -f "+self.e+".tar "+self.e+".sif\n")
+        self.userScript.append(containerBuild+" save -o "+self.e+"-"+self.target.gettargetName()+".tar localhost/"+self.e+":"+self.target.gettargetName()+"\n")
+        self.userScript.append(containerRun+" build --disable-cache "+self.e+"-"+self.target.gettargetName()+".sif docker-archive://"+self.e+"-"+self.target.gettargetName()+".tar\n")
+        self.userScriptFile = open("createContainer.sh","w")
+        self.userScriptFile.writelines(self.userScript)
+        self.userScriptFile.close()
+        os.chmod("createContainer.sh", 0o744)
+        self.userScriptPath = os.getcwd()+"/createContainer.sh"
+

fre/make/runFremake.py

@@ -10,12 +10,13 @@
 from multiprocessing.dummy import Pool
 from pathlib import Path
 import click
+import subprocess
 import fre.yamltools.combine_yamls as cy
 from .gfdlfremake import (
     targetfre, varsfre, yamlfre, checkout,
     makefilefre, buildDocker, buildBaremetal )
 
-def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose):
+def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose):
     ''' run fremake via click'''
     yml = yamlfile
     name = yamlfile.split(".")[0]
@@ -87,6 +88,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb
                 freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc)
                 freCheckout.finish(pc)
                 os.chmod(srcDir+"/checkout.sh", 0o744)
+                print("\nCheckout script created at "+ srcDir + "/checkout.sh \n")
                 ## TODO: Options for running on login cluster?
                 freCheckout.run()
 
@@ -126,6 +128,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb
                 # Loop through components, send component name/requires/overrides for Makefile
                 for c in fremakeYaml['src']:
                     freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides'])
+                print("\nMakefile created at " + bldDir + "/Makefile" + "\n")
                 freMakefile.writeMakefile()
 
                 ## Create a list of compile scripts to run in parallel
@@ -142,8 +145,11 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb
                     fremakeBuild.writeBuildComponents(c)
                 fremakeBuild.writeScript()
                 fremakeBuildList.append(fremakeBuild)
-                ## Run the build
-                fremakeBuild.run()
+                ## Run the build if --execute option given, otherwise print out compile script path
+                if execute:
+                    fremakeBuild.run()
+                else:
+                    print("Compile script created at "+ bldDir+"/compile.sh\n\n")
             else:
                 ###################### container stuff below #######################################
                 ## Run the checkout script
@@ -187,26 +193,32 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb
 
                 dockerBuild.writeRunscript(RUNenv,containerRun,tmpDir+"/execrunscript.sh")
 
-                ## Run the dockerfile; build the container
-                dockerBuild.build(containerBuild,containerRun)
+                # Create build script for container
+                dockerBuild.createBuildScript(containerBuild, containerRun)
+                print("Container build script created at "+dockerBuild.userScriptPath+"\n\n")
+
+                # Execute if flag is given
+                if execute:
+                    subprocess.run(args=[dockerBuild.userScriptPath], check=True)
 
                 #freCheckout.cleanup()
                 #buildDockerfile(fremakeYaml,image)
 
     if baremetalRun:
         if __name__ == '__main__':
-            # Create a multiprocessing Pool
-            pool = Pool(processes=nparallel)
-            # process data_inputs iterable with pool
-            pool.map(buildBaremetal.fremake_parallel,fremakeBuildList)
+            if execute:
+                # Create a multiprocessing Pool
+                pool = Pool(processes=nparallel)
+                # process data_inputs iterable with pool
+                pool.map(buildBaremetal.fremake_parallel,fremakeBuildList)
 
 @click.command()
-def _fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose):
+def _fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose):
     '''
     Decorator for calling _fremake_run - allows the decorated version
     of the function to be separate from the undecorated version
     '''
-    return fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose)
+    return fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose)
 
 if __name__ == "__main__":
     fremake_run()

fre/make/tests/compilation/test_fre_make_run_fremake.py

@@ -21,6 +21,6 @@
 @pytest.mark.skip(reason='failing: fix in development, see PR 275')
 def test_fre_make_run_fremake_null_model_serial_compile():
     ''' run fre make with run-fremake subcommand and build the null model experiment with gnu'''
-    runFremake.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, False)
+    runFremake.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, True, False)
     assert Path(f"{HOME_DIR}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/{EXPERIMENT}.x").exists()