adds Scorpio build

components/omega/CMakeLists.txt

@@ -34,7 +34,7 @@ if (NOT DEFINED PROJECT_NAME)
   preset()
 
   project(${OMEGA_PROJECT_NAME}
-             LANGUAGES CXX
+          LANGUAGES CXX
          )
 
   set(CMAKE_CXX_STANDARD 17) # used in E3SM

components/omega/OmegaBuild.cmake

@@ -56,19 +56,41 @@ macro(preset)
   endif()
 
   if(OMEGA_BUILD_TYPE STREQUAL "Debug")
+
     list(APPEND _PY_OPTS "-d")
-  endif()
 
-  execute_process(COMMAND ${Python_EXECUTABLE} create_scripts.py ${_PY_OPTS}
-    OUTPUT_QUIET ERROR_QUIET
-    WORKING_DIRECTORY ${OMEGA_SOURCE_DIR}
-    OUTPUT_VARIABLE _MACHINE_INFO)
+    execute_process(COMMAND ${Python_EXECUTABLE} create_scripts.py ${_PY_OPTS}
+      WORKING_DIRECTORY ${OMEGA_SOURCE_DIR}
+      OUTPUT_VARIABLE _MACHINE_INFO
+      ERROR_VARIABLE _ERROR_INFO)
+
+      message(STATUS "create_scripts.py output: ${_MACHINE_INFO}")
+      message(STATUS "create_scripts.py error: ${_ERROR_INFO}")
+
+  else()
+
+    execute_process(COMMAND ${Python_EXECUTABLE} create_scripts.py ${_PY_OPTS}
+      OUTPUT_QUIET ERROR_QUIET
+      WORKING_DIRECTORY ${OMEGA_SOURCE_DIR}
+      OUTPUT_VARIABLE _MACHINE_INFO)
+
+  endif()
 
   include(${_TMP_CMAKE_FILE})
   if(OMEGA_BUILD_TYPE STREQUAL "Release")
     file(REMOVE ${_TMP_CMAKE_FILE})
   endif()
 
+  if(OMEGA_C_COMPILER)
+    find_program(_OMEGA_C_COMPILER ${OMEGA_C_COMPILER})
+  else()
+    if (MPILIB STREQUAL "mpi-serial")
+      find_program(_OMEGA_C_COMPILER ${SCC})
+    else()
+      find_program(_OMEGA_C_COMPILER ${MPICC})
+    endif()
+  endif()
+
   if(OMEGA_CXX_COMPILER)
     find_program(_OMEGA_CXX_COMPILER ${OMEGA_CXX_COMPILER})
   else()
@@ -79,10 +101,28 @@ macro(preset)
     endif()
   endif()
 
+  if(OMEGA_Fortran_COMPILER)
+    find_program(_OMEGA_Fortran_COMPILER ${OMEGA_Fortran_COMPILER})
+  else()
+    if (MPILIB STREQUAL "mpi-serial")
+      find_program(_OMEGA_Fortran_COMPILER ${SFC})
+    else()
+      find_program(_OMEGA_Fortran_COMPILER ${MPIFC})
+    endif()
+  endif()
+
+  set(OMEGA_C_COMPILER ${_OMEGA_C_COMPILER})
+  set(CMAKE_C_COMPILER ${OMEGA_C_COMPILER})
+
   set(OMEGA_CXX_COMPILER ${_OMEGA_CXX_COMPILER})
   set(CMAKE_CXX_COMPILER ${OMEGA_CXX_COMPILER})
 
+  set(OMEGA_Fortran_COMPILER ${_OMEGA_Fortran_COMPILER})
+  set(CMAKE_Fortran_COMPILER ${OMEGA_Fortran_COMPILER})
+
+  message(STATUS "OMEGA_C_COMPILER = ${OMEGA_C_COMPILER}")
   message(STATUS "OMEGA_CXX_COMPILER = ${OMEGA_CXX_COMPILER}")
+  message(STATUS "OMEGA_Fortran_COMPILER = ${OMEGA_Fortran_COMPILER}")
   message(STATUS "MPI_EXEC = ${MPI_EXEC}")
 
 endmacro()
@@ -168,15 +208,15 @@ macro(update_variables)
   file(WRITE ${_TestMPISrcFile}  ${_TestMPISource})
 
   execute_process(
-	COMMAND ${OMEGA_CXX_COMPILER} -c ${_TestMPISrcFile} -o ${_TestMPIObjFile}
+    COMMAND ${OMEGA_CXX_COMPILER} -c ${_TestMPISrcFile} -o ${_TestMPIObjFile}
     OUTPUT_QUIET ERROR_QUIET
     RESULT_VARIABLE _MPI_TEST_RESULT
     OUTPUT_VARIABLE _MPI_TEST_OUTPUT
     ERROR_VARIABLE _MPI_TEST_ERROR)
 
   if(OMEGA_BUILD_TYPE EQUAL Release)
-	file(REMOVE ${_TestMPISrcFile})
-	file(REMOVE ${_TestMPIObjFile})
+    file(REMOVE ${_TestMPISrcFile})
+    file(REMOVE ${_TestMPIObjFile})
   endif()
 
   if (NOT _MPI_TEST_RESULT EQUAL 0)
@@ -205,47 +245,64 @@ macro(update_variables)
   # Check if CUDA or HIP is supported
   if((NOT DEFINED OMEGA_ARCH) OR (OMEGA_ARCH STREQUAL "NOT_DEFINED"))
 
-    execute_process(
-	  COMMAND ${OMEGA_CXX_COMPILER} --version
-      RESULT_VARIABLE _CXX_VER_RESULT
-      OUTPUT_VARIABLE _CXX_VER_OUTPUT)
+    if(USE_CUDA)
+      set(OMEGA_ARCH "CUDA")
+
+    elseif(USE_HIP)
+      set(OMEGA_ARCH "HIP")
 
-    if (_CXX_VER_RESULT EQUAL 0)
+    else()
+
+      execute_process(
+        COMMAND ${OMEGA_CXX_COMPILER} --version
+        RESULT_VARIABLE _CXX_VER_RESULT
+        OUTPUT_VARIABLE _CXX_VER_OUTPUT)
+
+      if (_CXX_VER_RESULT EQUAL 0)
 
-      string(REGEX MATCH "HIP|hip"       _HIP_CHECK "${_CXX_VER_OUTPUT}")
-      string(REGEX MATCH "AMD|amd"       _AMD_CHECK "${_CXX_VER_OUTPUT}")
-      string(REGEX MATCH "NVCC|nvcc"     _NVCC_CHECK "${_CXX_VER_OUTPUT}")
-      string(REGEX MATCH "NVIDIA|nvidia" _NVIDIA_CHECK "${_CXX_VER_OUTPUT}")
+        string(REGEX MATCH "HIP|hip"       _HIP_CHECK "${_CXX_VER_OUTPUT}")
+        string(REGEX MATCH "AMD|amd"       _AMD_CHECK "${_CXX_VER_OUTPUT}")
+        string(REGEX MATCH "NVCC|nvcc"     _NVCC_CHECK "${_CXX_VER_OUTPUT}")
+        string(REGEX MATCH "NVIDIA|nvidia" _NVIDIA_CHECK "${_CXX_VER_OUTPUT}")
 
-      if(_HIP_CHECK AND _AMD_CHECK)
-        set(OMEGA_ARCH "HIP")
+        if(_HIP_CHECK AND _AMD_CHECK)
+          set(OMEGA_ARCH "HIP")
 
-      elseif(_NVCC_CHECK AND _NVIDIA_CHECK)
-        set(OMEGA_ARCH "CUDA")
+        elseif(_NVCC_CHECK AND _NVIDIA_CHECK)
+          set(OMEGA_ARCH "CUDA")
 
+        else()
+          set(OMEGA_ARCH "")
+
+        endif()
       else()
         set(OMEGA_ARCH "")
 
       endif()
-    else()
-        set(OMEGA_ARCH "")
-
     endif()
   endif()
 
+  if(OMEGA_BUILD_TYPE STREQUAL "Debug")
+    message(STATUS "OMEGA_ARCH = ${OMEGA_ARCH}")
+  endif()
+
   if(OMEGA_ARCH STREQUAL "CUDA")
-    enable_language(CUDA)
+    set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
+    find_program(CMAKE_CUDA_COMPILER "nvcc")
 
-  elseif(OMEGA_ARCH STREQUAL "HIP")
-    enable_language(HIP)
+    if(OMEGA_BUILD_TYPE STREQUAL "Debug")
+      message(STATUS "CMAKE_CUDA_COMPILER = ${CMAKE_CUDA_COMPILER}")
+      message(STATUS "CMAKE_CUDA_HOST_COMPILER = ${CMAKE_CUDA_HOST_COMPILER}")
+    endif()
 
-  elseif(USE_CUDA)
-    set(OMEGA_ARCH "CUDA")
-    enable_language(CUDA)
+  elseif(OMEGA_ARCH STREQUAL "HIP")
+    set(CMAKE_HIP_HOST_COMPILER ${CMAKE_CXX_COMPILER})
+    find_program(CMAKE_HIP_COMPILER "hipcc")
 
-  elseif(USE_HIP)
-    set(OMEGA_ARCH "HIP")
-    enable_language(HIP)
+    if(OMEGA_BUILD_TYPE STREQUAL "Debug")
+      message(STATUS "CMAKE_HIP_COMPILER = ${CMAKE_HIP_COMPILER}")
+      message(STATUS "CMAKE_HIP_HOST_COMPILER = ${CMAKE_HIP_HOST_COMPILER}")
+    endif()
 
   endif()
 

components/omega/create_scripts.py

@@ -46,9 +46,10 @@ def parse_cmdline():
 
 
 # run a shell command
-def run_cmd_no_fail(cmd):
-    out = subprocess.run(cmd, shell=True, capture_output=True)
-    return str(out.stdout, 'UTF-8')
+def run_cmd_no_fail(cmd, env):
+    out = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, env=env)
+    retval = str(out.stdout, 'UTF-8').strip()
+    return retval
 
 
 # main class that extends Machines class
@@ -111,7 +112,7 @@ def get_processed_value(self, raw_value, outvar):
         for s in shell_ref_re.finditer(item_data):
             shell_cmd = s.groups()[0]
             item_data = item_data.replace(s.group(),
-                                          run_cmd_no_fail(shell_cmd))
+                                          run_cmd_no_fail(shell_cmd, outvar))
 
         for m in reference_re.finditer(item_data):
             var = m.groups()[0]
@@ -138,10 +139,10 @@ def get_processed_value(self, raw_value, outvar):
         return item_data
 
     # get module info
-    def get_modules(self, outvar):
+    def get_modules(self, outvar, exclude_envs):
 
         modcmds: typing.List[str] = []
-        outvar["__OMEGA_MODULE_COMMANDS__"] = modcmds
+        self.__OMEGA_MODULE_COMMANDS__ = modcmds
 
         module_system_node = self.get_child("module_system")
         module_system_type = self.get(module_system_node, "type")
@@ -151,6 +152,9 @@ def get_modules(self, outvar):
                    "module system is not supported."))
             exit(-1)
 
+        # get env. variables *before* applying module configurations
+        # specified in config_machines.xml
+
         out1 = subprocess.check_output("env", shell=True)
         env1 = str(out1, 'UTF-8')
 
@@ -162,6 +166,8 @@ def get_modules(self, outvar):
 
         shcmds = []
 
+        # read module commands for the specified compiler and mpi library
+
         for module_node in module_nodes:
             compiler = self.get(module_node, "compiler")
             mpilib = self.get(module_node, "mpilib")
@@ -194,26 +200,37 @@ def get_modules(self, outvar):
 
         shcmds.append("env")
 
+        # get env. variables *after* applying module configurations
+        # specified in config_machines.xml
+
         out2 = subprocess.check_output(";".join(shcmds), shell=True)
         env2 = str(out2, 'UTF-8')
 
         parsed1 = {}
+        parsed2 = {}
 
         for (name, value) in pat_envvar.findall(env1):
             parsed1[name] = value
 
         for (name, value) in pat_envvar.findall(env2):
-            if name in parsed1:
+            parsed2[name] = value
+
+        for (name, value) in parsed2.items():
+            if name in parsed1.keys():
                 if parsed1[name] != value:
                     outvar[name] = value
             else:
                 outvar[name] = value
 
+        for (name, value) in parsed1.items():
+            if name not in parsed2.keys():
+                exclude_envs.append(name)
+
     # get environmental variables info
     def get_envs(self, outvar):
 
         exports: typing.Dict[str, str] = {}
-        outvar["__OMEGA_SCRIPT_EXPORTS__"] = exports
+        self.__OMEGA_SCRIPT_EXPORTS__ = exports
 
         envvar_nodes = self.get_children("environment_variables",
                                          root=self.machine_node)
@@ -284,16 +301,17 @@ def get_mpirun(self, outvar):
     def gen_machinfo(self):
 
         outvar: typing.Dict[str, str] = {}
+        exclude_envs: typing.List[str] = []
 
         self.get_mpirun(outvar)
-        self.get_modules(outvar)
+        self.get_modules(outvar, exclude_envs)
         self.get_envs(outvar)
-        self.write_output(outvar)
+        self.write_output(outvar, exclude_envs)
         self.generate_scripts(outvar)
 
     # create a temporary cmake script to be included
     # in the main Omega cmake build system
-    def write_output(self, outvar):
+    def write_output(self, outvar, exclude_envs):
 
         with open(self.outpath, "w") as f:
             f.write("message(STATUS \"Reading E3SM machine info\")\n")
@@ -302,6 +320,9 @@ def write_output(self, outvar):
                 if not key.startswith("__OMEGA_"):
                     f.write("set(ENV{%s} \"%s\")\n" % (key, value))
 
+            for name in exclude_envs:
+                f.write("unset(ENV{%s})\n" % name)
+
             f.write(f"set(MACH {self.machname})\n")
             f.write(f"set(OS {self.machos})\n")
             f.write(f"set(COMPILER {self.compiler})\n")
@@ -325,11 +346,11 @@ def generate_scripts(self, outvar):
             f.write("#!/usr/bin/env bash\n\n")
 
             f.write("# module commands\n")
-            for cmd in outvar["__OMEGA_MODULE_COMMANDS__"]:
+            for cmd in self.__OMEGA_MODULE_COMMANDS__:
                 f.write(cmd + "\n")
 
             f.write("\n# env. variables\n")
-            for key, value in outvar["__OMEGA_SCRIPT_EXPORTS__"].items():
+            for key, value in self.__OMEGA_SCRIPT_EXPORTS__.items():
                 f.write(f"export {key}=\"{value}\"\n")
 
         with open(omega_build, "w") as f:

components/omega/external/CMakeLists.txt

@@ -19,3 +19,25 @@ endif()
 if(OMEGA_DEBUG)
   target_compile_definitions(yakl INTERFACE YAKL_DEBUG=1)
 endif()
+
+# forward env. variables to Scorpio build
+if(DEFINED ENV{NETCDF_PATH})
+  set(NetCDF_PATH "$ENV{NETCDF_PATH}")
+endif()
+if(DEFINED ENV{NETCDF_C_PATH})
+  set(NetCDF_C_PATH "$ENV{NETCDF_C_PATH}")
+endif()
+if(DEFINED ENV{NETCDF_FORTRAN_PATH})
+  set(NetCDF_Fortran_PATH "$ENV{NETCDF_FORTRAN_PATH}")
+endif()
+if(DEFINED ENV{PNETCDF_PATH})
+  set(PnetCDF_PATH "$ENV{PNETCDF_PATH}")
+endif()
+
+option(PIO_ENABLE_TOOLS "" OFF)
+
+# Add the Scorpio library
+add_subdirectory(
+  ${E3SM_EXTERNALS_ROOT}/scorpio
+  ${CMAKE_CURRENT_BINARY_DIR}/scorpio
+)

components/omega/src/CMakeLists.txt

@@ -22,7 +22,7 @@ target_link_options(
     ${OMEGA_LINK_OPTIONS}
 )
 
-target_link_libraries(${OMEGA_LIB_NAME} spdlog yakl)
+target_link_libraries(${OMEGA_LIB_NAME} spdlog yakl pioc)
 
 # build Omega executable
 if(OMEGA_BUILD_EXECUTABLE)