From 2f2c5efb69a3fd0d5a58c670ac3a0aa03c1ca914 Mon Sep 17 00:00:00 2001 From: rem1776 Date: Wed, 13 Nov 2024 15:20:48 -0500 Subject: [PATCH 1/7] inital changes for run-fremake execute option --- fre/make/fremake.py | 6 +++++- fre/make/runFremake.py | 18 ++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/fre/make/fremake.py b/fre/make/fremake.py index f39a6be0..21cd7bfb 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -66,12 +66,16 @@ def make_cli(): "--no-parallel-checkout", is_flag = True, help = no_parallel_checkout_opt_help) +@click.option("--execute", + is_flag = True, + default = False, + help = "Use this to run the created compilation script.") @click.option("-v", "--verbose", is_flag = True, help = verbose_opt_help) @click.pass_context -def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, verbose): +def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, execute, verbose): """ - Perform all fremake functions to run checkout and compile model""" context.forward(runFremake._fremake_run) diff --git a/fre/make/runFremake.py b/fre/make/runFremake.py index ffe2ec96..f57c2af0 100644 --- a/fre/make/runFremake.py +++ b/fre/make/runFremake.py @@ -15,7 +15,7 @@ targetfre, varsfre, yamlfre, checkout, makefilefre, buildDocker, buildBaremetal ) -def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose): +def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose): ''' run fremake via click''' yml = yamlfile name = yamlfile.split(".")[0] @@ -188,25 +188,27 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb dockerBuild.writeRunscript(RUNenv,containerRun,tmpDir+"/execrunscript.sh") ## Run the dockerfile; build the container - dockerBuild.build(containerBuild,containerRun) + if execute: + dockerBuild.build(containerBuild,containerRun) #freCheckout.cleanup() #buildDockerfile(fremakeYaml,image) if baremetalRun: if __name__ == '__main__': - # Create a multiprocessing Pool - pool = Pool(processes=nparallel) - # process data_inputs iterable with pool - pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) + if execute: + # Create a multiprocessing Pool + pool = Pool(processes=nparallel) + # process data_inputs iterable with pool + pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) @click.command() -def _fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose): +def _fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose): ''' Decorator for calling _fremake_run - allows the decorated version of the function to be separate from the undecorated version ''' - return fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose) + return fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose) if __name__ == "__main__": fremake_run() From c29e3116c1061c2c7ac346677a4530229c24d08b Mon Sep 17 00:00:00 2001 From: Ryan Mulhall Date: Fri, 15 Nov 2024 15:51:47 -0500 Subject: [PATCH 2/7] add a function to generate a container build script if --execute not given --- fre/make/fremake.py | 3 +- fre/make/gfdlfremake/buildBaremetal.py | 6 ++-- fre/make/gfdlfremake/buildDocker.py | 49 ++++++++++++++++++-------- fre/make/runFremake.py | 13 +++++-- 4 files changed, 51 insertions(+), 20 deletions(-) diff --git a/fre/make/fremake.py b/fre/make/fremake.py index 21cd7bfb..6dfb9a34 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -66,7 +66,8 @@ def make_cli(): "--no-parallel-checkout", is_flag = True, help = no_parallel_checkout_opt_help) -@click.option("--execute", +@click.option("-e", + "--execute", is_flag = True, default = False, help = "Use this to run the created compilation script.") diff --git a/fre/make/gfdlfremake/buildBaremetal.py b/fre/make/gfdlfremake/buildBaremetal.py index fdb4e2d8..a4ab216b 100644 --- a/fre/make/gfdlfremake/buildBaremetal.py +++ b/fre/make/gfdlfremake/buildBaremetal.py @@ -10,14 +10,14 @@ def fremake_parallel(fremakeBuildList): """ Brief: Called for parallel execution purposes. Runs the builds. - Param: + Param: - fremakeBuildList : fremakeBuild object list passes by pool.map """ fremakeBuildList.run() class buildBaremetal(): """ - Brief: Creates the build script to compile the model + Brief: Creates the build script to compile the model Param: - self : The buildScript object - exp : The experiment name @@ -50,6 +50,8 @@ def __init__(self,exp,mkTemplatePath,srcDir,bldDir,target,modules,modulesInit,jo if self.modules != "": self.setup.extend(modulesInit) #extend - this is a list self.setup.append("module load "+self.modules+" \n") # Append -this is a single string + # make sure our modules are loaded correctly + self.setup.append("test $? != 0 && echo \"**********Modules failed to load, check platform yaml\" && exit 1\n") ## Create the build directory os.system("mkdir -p "+self.bld) diff --git a/fre/make/gfdlfremake/buildDocker.py b/fre/make/gfdlfremake/buildDocker.py index 6d33d0d2..cb48b48c 100644 --- a/fre/make/gfdlfremake/buildDocker.py +++ b/fre/make/gfdlfremake/buildDocker.py @@ -9,12 +9,12 @@ class container(): """ Brief: Opens the Dockerfile for writing - Param: + Param: - self : The dockerfile object - base : The docker base image to start from - libs : Additional libraries defined by user - exp : The experiment name - - RUNenv : The commands that have to be run at + - RUNenv : The commands that have to be run at the beginning of a RUN in the dockerfile to set up the environment """ @@ -58,7 +58,7 @@ def __init__(self,base,exp,libs,RUNenv,target): def writeDockerfileCheckout(self, cScriptName, cOnDisk): """ Brief: writes to the checkout part of the Dockerfile and sets up the compile - Param: + Param: - self : The dockerfile object - cScriptName : The name of the checkout script in the container - cOnDisk : The relative path to the checkout script on disk @@ -74,7 +74,7 @@ def writeDockerfileCheckout(self, cScriptName, cOnDisk): def writeDockerfileMakefile(self, makefileOnDiskPath, linklineonDiskPath): """ Brief: Copies the Makefile into the bldDir in the dockerfile - Param: + Param: - self : The dockerfile object - makefileOnDiskPath : The path to Makefile on the local disk - linklineonDiskPath : The path to the link line script on the local disk @@ -98,8 +98,8 @@ def writeDockerfileMakefile(self, makefileOnDiskPath, linklineonDiskPath): def writeDockerfileMkmf(self, c): """ - Brief: Adds components to the build part of the Dockerfile - Param: + Brief: Adds components to the build part of the Dockerfile + Param: - self : The dockerfile object - c : Component from the compile yaml """ @@ -141,14 +141,14 @@ def writeDockerfileMkmf(self, c): def writeRunscript(self,RUNenv,containerRun,runOnDisk): """ Brief: Writes a runscript to set up spack loads/environment - in order to run the executable in the container; + in order to run the executable in the container; runscript copied into container - Param: + Param: - self : The dockerfile object - - RUNEnv : The commands that have to be run at + - RUNEnv : The commands that have to be run at the beginning of a RUN in the dockerfile - - containerRun : The container platform used with `exec` - to run the container; apptainer + - containerRun : The container platform used with `exec` + to run the container; apptainer or singularity used - runOnDisk : The path to the run script on the local disk """ @@ -187,14 +187,35 @@ def writeRunscript(self,RUNenv,containerRun,runOnDisk): def build(self,containerBuild,containerRun): """ Brief: Builds the container image for the model - Param: + Param: - self : The dockerfile object - - containerBuild : The tool used to build the container; + - containerBuild : The tool used to build the container; docker or podman used - - containerRun : The container platform used with `exec` to + - containerRun : The container platform used with `exec` to run the container; apptainer or singularity used """ os.system(containerBuild+" build -f Dockerfile -t "+self.e+":"+self.target.gettargetName()) os.system("rm -f "+self.e+".tar "+self.e+".sif") os.system(containerBuild+" save -o "+self.e+"-"+self.target.gettargetName()+".tar localhost/"+self.e+":"+self.target.gettargetName()) os.system(containerRun+" build --disable-cache "+self.e+"-"+self.target.gettargetName()+".sif docker-archive://"+self.e+"-"+self.target.gettargetName()+".tar") + + def createBuildScript(self,containerBuild,containerRun): + """ + Brief: Writes out the build commands for the created dockerfile in a script, uses the same commands as the routine above + Param: + - self : The dockerfile object + - containerBuild : The tool used to build the container; + docker or podman used + - containerRun : The container platform used with `exec` to + run the container; apptainer or singularity used + """ + self.userScript = ["#!/bin/bash\n"] + self.userScript.append(containerBuild+" build -f Dockerfile -t "+self.e+":"+self.target.gettargetName()+"\n") + self.userScript.append("rm -f "+self.e+".tar "+self.e+".sif\n") + self.userScript.append(containerBuild+" save -o "+self.e+"-"+self.target.gettargetName()+".tar localhost/"+self.e+":"+self.target.gettargetName()+"\n") + self.userScript.append(containerRun+" build --disable-cache "+self.e+"-"+self.target.gettargetName()+".sif docker-archive://"+self.e+"-"+self.target.gettargetName()+".tar\n") + self.userScriptFile = open("createContainer.sh","w") + self.userScriptFile.writelines(self.userScript) + os.chmod("createContainer.sh", 0o744) + self.userScriptPath = os.getcwd()+"/createContainer.sh" + diff --git a/fre/make/runFremake.py b/fre/make/runFremake.py index f57c2af0..0e4cfecd 100644 --- a/fre/make/runFremake.py +++ b/fre/make/runFremake.py @@ -142,8 +142,11 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec fremakeBuild.writeBuildComponents(c) fremakeBuild.writeScript() fremakeBuildList.append(fremakeBuild) - ## Run the build - fremakeBuild.run() + ## Run the build if --execute option given, otherwise print out compile script path + if execute: + fremakeBuild.run() + else: + print("Compile script created at "+ bldDir+"/compile.sh\n\n") else: ###################### container stuff below ####################################### ## Run the checkout script @@ -187,9 +190,13 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec dockerBuild.writeRunscript(RUNenv,containerRun,tmpDir+"/execrunscript.sh") - ## Run the dockerfile; build the container + ## Run the dockerfile and create the container if execute option is given + ## otherwise create a build script and print out its path if execute: dockerBuild.build(containerBuild,containerRun) + else: + dockerBuild.createBuildScript(containerBuild, containerRun) + print("Container build script created at "+dockerBuild.userScriptPath+"\n\n") #freCheckout.cleanup() #buildDockerfile(fremakeYaml,image) From 670cf66f29a752636bb4bbc290e4d2098ed6fe5f Mon Sep 17 00:00:00 2001 From: Ryan Mulhall <35538242+rem1776@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:50:15 -0500 Subject: [PATCH 3/7] add option to test_fre_make_run_fremake.py --- fre/make/tests/compilation/test_fre_make_run_fremake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/make/tests/compilation/test_fre_make_run_fremake.py b/fre/make/tests/compilation/test_fre_make_run_fremake.py index be91a547..a92cf6a7 100644 --- a/fre/make/tests/compilation/test_fre_make_run_fremake.py +++ b/fre/make/tests/compilation/test_fre_make_run_fremake.py @@ -16,6 +16,6 @@ def test_fre_make_run_fremake_null_model_serial_compile(): ''' run fre make with run-fremake subcommand and build the null model experiment with gnu''' - runFremake.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, False) + runFremake.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, True, False) assert Path(f"{HOME_DIR}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/{EXPERIMENT}.x").exists() From ed44e680752854366ce31e3df03d285ada63a555 Mon Sep 17 00:00:00 2001 From: Ryan Mulhall <35538242+rem1776@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:59:47 -0500 Subject: [PATCH 4/7] remove module check --- fre/make/gfdlfremake/buildBaremetal.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/fre/make/gfdlfremake/buildBaremetal.py b/fre/make/gfdlfremake/buildBaremetal.py index a4ab216b..48205d6b 100644 --- a/fre/make/gfdlfremake/buildBaremetal.py +++ b/fre/make/gfdlfremake/buildBaremetal.py @@ -50,8 +50,6 @@ def __init__(self,exp,mkTemplatePath,srcDir,bldDir,target,modules,modulesInit,jo if self.modules != "": self.setup.extend(modulesInit) #extend - this is a list self.setup.append("module load "+self.modules+" \n") # Append -this is a single string - # make sure our modules are loaded correctly - self.setup.append("test $? != 0 && echo \"**********Modules failed to load, check platform yaml\" && exit 1\n") ## Create the build directory os.system("mkdir -p "+self.bld) From 7b06b5a760cea13eb8edb171e07f00ee41a88eb8 Mon Sep 17 00:00:00 2001 From: Ryan Mulhall Date: Tue, 26 Nov 2024 15:01:20 -0500 Subject: [PATCH 5/7] update null model platforms for latest c5 modules --- fre/make/tests/null_example/platforms.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/make/tests/null_example/platforms.yaml b/fre/make/tests/null_example/platforms.yaml index fdfa9d4f..a1595bee 100644 --- a/fre/make/tests/null_example/platforms.yaml +++ b/fre/make/tests/null_example/platforms.yaml @@ -10,7 +10,7 @@ platforms: - name: ncrc5.intel23 compiler: intel modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] - modules: [!join [*INTEL, "/2023.1.0"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3] + modules: [!join [*INTEL, "/2023.2.0"],"fre/bronx-21",cray-hdf5/1.12.2.11, cray-netcdf/4.9.0.11] fc: ftn cc: cc mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL, ".mk"] From c52a769aefa36ba1d7a1df8386e9fd3b8b91ef38 Mon Sep 17 00:00:00 2001 From: Ryan Mulhall Date: Tue, 26 Nov 2024 15:50:55 -0500 Subject: [PATCH 6/7] changes to always create the container build script --- fre/make/gfdlfremake/buildDocker.py | 19 +++---------------- fre/make/runFremake.py | 13 +++++++------ 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/fre/make/gfdlfremake/buildDocker.py b/fre/make/gfdlfremake/buildDocker.py index cb48b48c..491d5c2c 100644 --- a/fre/make/gfdlfremake/buildDocker.py +++ b/fre/make/gfdlfremake/buildDocker.py @@ -184,24 +184,10 @@ def writeRunscript(self,RUNenv,containerRun,runOnDisk): self.d.write('ENTRYPOINT ["/bin/bash"]') self.d.close() - def build(self,containerBuild,containerRun): - """ - Brief: Builds the container image for the model - Param: - - self : The dockerfile object - - containerBuild : The tool used to build the container; - docker or podman used - - containerRun : The container platform used with `exec` to - run the container; apptainer or singularity used - """ - os.system(containerBuild+" build -f Dockerfile -t "+self.e+":"+self.target.gettargetName()) - os.system("rm -f "+self.e+".tar "+self.e+".sif") - os.system(containerBuild+" save -o "+self.e+"-"+self.target.gettargetName()+".tar localhost/"+self.e+":"+self.target.gettargetName()) - os.system(containerRun+" build --disable-cache "+self.e+"-"+self.target.gettargetName()+".sif docker-archive://"+self.e+"-"+self.target.gettargetName()+".tar") - def createBuildScript(self,containerBuild,containerRun): """ - Brief: Writes out the build commands for the created dockerfile in a script, uses the same commands as the routine above + Brief: Writes out the build commands for the created dockerfile in a script, + which builds the dockerfile and then converts the format to a singularity image file. Param: - self : The dockerfile object - containerBuild : The tool used to build the container; @@ -216,6 +202,7 @@ def createBuildScript(self,containerBuild,containerRun): self.userScript.append(containerRun+" build --disable-cache "+self.e+"-"+self.target.gettargetName()+".sif docker-archive://"+self.e+"-"+self.target.gettargetName()+".tar\n") self.userScriptFile = open("createContainer.sh","w") self.userScriptFile.writelines(self.userScript) + self.userScriptFile.close() os.chmod("createContainer.sh", 0o744) self.userScriptPath = os.getcwd()+"/createContainer.sh" diff --git a/fre/make/runFremake.py b/fre/make/runFremake.py index 0e4cfecd..17e1c28b 100644 --- a/fre/make/runFremake.py +++ b/fre/make/runFremake.py @@ -10,6 +10,7 @@ from multiprocessing.dummy import Pool from pathlib import Path import click +import subprocess import fre.yamltools.combine_yamls as cy from .gfdlfremake import ( targetfre, varsfre, yamlfre, checkout, @@ -190,13 +191,13 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec dockerBuild.writeRunscript(RUNenv,containerRun,tmpDir+"/execrunscript.sh") - ## Run the dockerfile and create the container if execute option is given - ## otherwise create a build script and print out its path + # Create build script for container + dockerBuild.createBuildScript(containerBuild, containerRun) + print("Container build script created at "+dockerBuild.userScriptPath+"\n\n") + + # Execute if flag is given if execute: - dockerBuild.build(containerBuild,containerRun) - else: - dockerBuild.createBuildScript(containerBuild, containerRun) - print("Container build script created at "+dockerBuild.userScriptPath+"\n\n") + subprocess.run(args=[dockerBuild.userScriptPath], check=True) #freCheckout.cleanup() #buildDockerfile(fremakeYaml,image) From 8fa356beb20f8cf6e75d64b6dd09182248fa6de5 Mon Sep 17 00:00:00 2001 From: Ryan Mulhall Date: Wed, 27 Nov 2024 12:11:00 -0500 Subject: [PATCH 7/7] add output for checkout and makefile creation for runfremake --- fre/make/runFremake.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fre/make/runFremake.py b/fre/make/runFremake.py index 17e1c28b..9e1c1730 100644 --- a/fre/make/runFremake.py +++ b/fre/make/runFremake.py @@ -88,6 +88,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc) freCheckout.finish(pc) os.chmod(srcDir+"/checkout.sh", 0o744) + print("\nCheckout script created at "+ srcDir + "/checkout.sh \n") ## TODO: Options for running on login cluster? freCheckout.run() @@ -127,6 +128,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec # Loop through components, send component name/requires/overrides for Makefile for c in fremakeYaml['src']: freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides']) + print("\nMakefile created at " + bldDir + "/Makefile" + "\n") freMakefile.writeMakefile() ## Create a list of compile scripts to run in parallel