diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index b6baf5d55e..10ab4dcfbb 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -55,6 +55,7 @@ import traceback from concurrent.futures import ThreadPoolExecutor from datetime import datetime +from enum import Enum from textwrap import indent import easybuild.tools.environment as env @@ -73,9 +74,10 @@ from easybuild.tools.build_details import get_build_stats from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, dry_run_msg, dry_run_warning, dry_run_set_dirs from easybuild.tools.build_log import print_error, print_msg, print_warning -from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES, PYTHONPATH, EBPYTHONPREFIXES +from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES +from easybuild.tools.config import EASYBUILD_SOURCES_URL, EBPYTHONPREFIXES # noqa from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES -from easybuild.tools.config import EASYBUILD_SOURCES_URL # noqa +from easybuild.tools.config import PYTHONPATH, SEARCH_PATH_BIN_DIRS, SEARCH_PATH_LIB_DIRS from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, package_path, source_paths from easybuild.tools.environment import restore_env, sanitize_env @@ -97,7 +99,8 @@ from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX -from easybuild.tools.modules import Lmod, curr_module_paths, invalidate_module_caches_for, get_software_root +from easybuild.tools.modules import Lmod, ModEnvVarType, ModuleLoadEnvironment +from easybuild.tools.modules import curr_module_paths, invalidate_module_caches_for, get_software_root from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ALL, PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS from easybuild.tools.output import show_progress_bars, start_progress_bar, stop_progress_bar, update_progress_bar @@ -109,7 +112,7 @@ from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION -DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64') +DEFAULT_BIN_LIB_SUBDIRS = SEARCH_PATH_BIN_DIRS + SEARCH_PATH_LIB_DIRS MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP] @@ -122,6 +125,17 @@ _log = fancylogger.getLogger('easyblock') +class LibSymlink(Enum): + """ + Possible states for symlinking of lib/lib64 subdirectories: + - UNKNOWN: has not been determined yet + - LIB_TO_LIB64: 'lib' is a symlink to 'lib64' + - LIB64_TO_LIB: 'lib64' is a symlink to 'lib' + - NEITHER: neither 'lib' is a symlink to 'lib64', nor 'lib64' is a symlink to 'lib' + - """ + UNKNOWN, LIB_TO_LIB64, LIB64_TO_LIB, NEITHER = range(0, 4) + + class EasyBlock(object): """Generic support for building and installing software, base class for actual easyblocks.""" @@ -204,9 +218,15 @@ def __init__(self, ec, logfile=None): if modules_header_path is not None: self.modules_header = read_file(modules_header_path) + # environment variables on module load + self.module_load_environment = ModuleLoadEnvironment() + # determine install subdirectory, based on module name self.install_subdir = None + # track status of symlink between library directories + self.install_lib_symlink = LibSymlink.UNKNOWN + # indicates whether build should be performed in installation dir self.build_in_installdir = self.cfg['buildininstalldir'] @@ -1616,106 +1636,120 @@ def make_module_group_check(self): def make_module_req(self): """ - Generate the environment-variables to run the module. + Generate the environment-variables required to run the module. """ - requirements = self.make_module_req_guess() - - lines = ['\n'] - if os.path.isdir(self.installdir): - old_dir = change_dir(self.installdir) - else: - old_dir = None + mod_lines = ['\n'] if self.dry_run: self.dry_run_msg("List of paths that would be searched and added to module file:\n") note = "note: glob patterns are not expanded and existence checks " note += "for paths are skipped for the statements below due to dry run" - lines.append(self.module_generator.comment(note)) - - # For these environment variables, the corresponding directory must include at least one file. - # The values determine if detection is done recursively, i.e. if it accepts directories where files - # are only in subdirectories. - keys_requiring_files = { - 'PATH': False, - 'LD_LIBRARY_PATH': False, - 'LIBRARY_PATH': True, - 'CPATH': True, - 'CMAKE_PREFIX_PATH': True, - 'CMAKE_LIBRARY_PATH': True, - } + mod_lines.append(self.module_generator.comment(note)) + + if self.make_module_req_guess.__qualname__ != "EasyBlock.make_module_req_guess": + # Deprecated make_module_req_guess method used in child Easyblock + # Update environment with custom make_module_req_guess + self.log.deprecated( + "make_module_req_guess() is deprecated, use EasyBlock.module_load_environment instead.", + "6.0", + ) + self.module_load_environment.update(self.make_module_req_guess()) - for key, reqs in sorted(requirements.items()): - if isinstance(reqs, str): - self.log.warning("Hoisting string value %s into a list before iterating over it", reqs) - reqs = [reqs] + # Only inject path-like environment variables into module file + env_var_requirements = sorted([ + (envar_name, envar_val) for envar_name, envar_val in self.module_load_environment.items() + if envar_val.is_path + ]) + self.log.debug(f"Tentative module environment requirements before path expansion: {env_var_requirements}") + + for env_var, search_paths in env_var_requirements: if self.dry_run: - self.dry_run_msg(" $%s: %s" % (key, ', '.join(reqs))) - # Don't expand globs or do any filtering below for dry run - paths = reqs + self.dry_run_msg(f" ${env_var}:{', '.join(search_paths)}") + # Don't expand globs or do any filtering for dry run + mod_req_paths = search_paths else: - # Expand globs but only if the string is non-empty - # empty string is a valid value here (i.e. to prepend the installation prefix, cfr $CUDA_HOME) - paths = sum((glob.glob(path) if path else [path] for path in reqs), []) # sum flattens to list - - # If lib64 is just a symlink to lib we fixup the paths to avoid duplicates - lib64_is_symlink = (all(os.path.isdir(path) for path in ['lib', 'lib64']) and - os.path.samefile('lib', 'lib64')) - if lib64_is_symlink: - fixed_paths = [] - for path in paths: - if (path + os.path.sep).startswith('lib64' + os.path.sep): - # We only need CMAKE_LIBRARY_PATH if there is a separate lib64 path, so skip symlink - if key == 'CMAKE_LIBRARY_PATH': - continue - path = path.replace('lib64', 'lib', 1) - fixed_paths.append(path) - if fixed_paths != paths: - self.log.info("Fixed symlink lib64 in paths for %s: %s -> %s", key, paths, fixed_paths) - paths = fixed_paths - # remove duplicate paths preserving order - paths = nub(paths) - if key in keys_requiring_files: - # only retain paths that contain at least one file - recursive = keys_requiring_files[key] - retained_paths = [] - for pth in paths: - fullpath = os.path.join(self.installdir, pth) - if os.path.isdir(fullpath) and dir_contains_files(fullpath, recursive=recursive): - retained_paths.append(pth) - if retained_paths != paths: - self.log.info("Only retaining paths for %s that contain at least one file: %s -> %s", - key, paths, retained_paths) - paths = retained_paths - - if paths: - lines.append(self.module_generator.prepend_paths(key, paths)) + mod_req_paths = [] + for path in search_paths: + mod_req_paths.extend(self.expand_module_search_path(path, path_type=search_paths.type)) + + if mod_req_paths: + mod_req_paths = nub(mod_req_paths) # remove duplicates + mod_lines.append(self.module_generator.prepend_paths(env_var, mod_req_paths)) + if self.dry_run: self.dry_run_msg('') - if old_dir is not None: - change_dir(old_dir) + return "".join(mod_lines) + + def expand_module_search_path(self, search_path, path_type=ModEnvVarType.PATH_WITH_FILES): + """ + Expand given path glob and return list of suitable paths to be used as search paths: + - Paths are relative to installation prefix root + - Paths must point to existing files/directories + - Search paths to a 'lib64' symlinked to 'lib' are discarded to avoid duplicates + - :path_type: ModEnvVarType that controls requirements for population of directories + - PATH: no requirements, can be empty + - PATH_WITH_FILES: must contain at least one file in them (default) + - PATH_WITH_TOP_FILES: increase stricness to require files in top level directory + """ + # Expand globs but only if the string is non-empty + # empty string is a valid value here (i.e. to prepend the installation prefix root directory) + abs_glob = os.path.join(self.installdir, search_path) + exp_search_paths = [abs_glob] if search_path == "" else glob.glob(abs_glob) + + # Explicitly check symlink state between lib dirs if it is still undefined (e.g. --module-only) + if self.install_lib_symlink == LibSymlink.UNKNOWN: + self.check_install_lib_symlink() + + retained_search_paths = [] + for abs_path in exp_search_paths: + # return relative paths + tentative_path = os.path.relpath(abs_path, start=self.installdir) + tentative_path = '' if tentative_path == '.' else tentative_path # use empty string instead of dot + + # avoid duplicate entries between symlinked library dirs + tent_path_sep = tentative_path + os.path.sep + if self.install_lib_symlink == LibSymlink.LIB64_TO_LIB and tent_path_sep.startswith('lib64' + os.path.sep): + self.log.debug("Discarded search path to symlinked lib64 directory: %s", tentative_path) + continue + if self.install_lib_symlink == LibSymlink.LIB_TO_LIB64 and tent_path_sep.startswith('lib' + os.path.sep): + self.log.debug("Discarded search path to symlinked lib directory: %s", tentative_path) + continue + + check_dir_files = path_type in (ModEnvVarType.PATH_WITH_FILES, ModEnvVarType.PATH_WITH_TOP_FILES) + if os.path.isdir(abs_path) and check_dir_files: + # only retain paths to directories that contain at least one file + recursive = path_type == ModEnvVarType.PATH_WITH_FILES + if not dir_contains_files(abs_path, recursive=recursive): + self.log.debug("Discarded search path to empty directory: %s", tentative_path) + continue + + retained_search_paths.append(tentative_path) - return ''.join(lines) + return retained_search_paths + + def check_install_lib_symlink(self): + """Check symlink state between library directories in installation prefix""" + lib_dir = os.path.join(self.installdir, 'lib') + lib64_dir = os.path.join(self.installdir, 'lib64') + + self.install_lib_symlink = LibSymlink.NEITHER + if os.path.exists(lib_dir) and os.path.exists(lib64_dir): + if os.path.islink(lib_dir) and os.path.samefile(lib_dir, lib64_dir): + self.install_lib_symlink = LibSymlink.LIB_TO_LIB64 + elif os.path.islink(lib64_dir) and os.path.samefile(lib_dir, lib64_dir): + self.install_lib_symlink = LibSymlink.LIB64_TO_LIB def make_module_req_guess(self): """ - A dictionary of possible directories to look for. - """ - lib_paths = ['lib', 'lib32', 'lib64'] - return { - 'PATH': ['bin', 'sbin'], - 'LD_LIBRARY_PATH': lib_paths, - 'LIBRARY_PATH': lib_paths, - 'CPATH': ['include'], - 'MANPATH': ['man', os.path.join('share', 'man')], - 'PKG_CONFIG_PATH': [os.path.join(x, 'pkgconfig') for x in lib_paths + ['share']], - 'ACLOCAL_PATH': [os.path.join('share', 'aclocal')], - 'CLASSPATH': ['*.jar'], - 'XDG_DATA_DIRS': ['share'], - 'GI_TYPELIB_PATH': [os.path.join(x, 'girepository-*') for x in lib_paths], - 'CMAKE_PREFIX_PATH': [''], - 'CMAKE_LIBRARY_PATH': ['lib64'], # lib and lib32 are searched through the above - } + A dictionary of common search path variables to be loaded by environment modules + Each key contains the list of known directories related to the search path + """ + self.log.deprecated( + "make_module_req_guess() is deprecated, use EasyBlock.module_load_environment instead", + '6.0', + ) + return self.module_load_environment.as_dict def load_module(self, mod_paths=None, purge=True, extra_modules=None, verbose=True): """ @@ -3136,18 +3170,19 @@ def post_install_step(self): # However for each in $LIBRARY_PATH (where is often /lib) it searches /../lib64 first. # So we create /lib64 as a symlink to /lib to make it prefer EB installed libraries. # See https://github.com/easybuilders/easybuild-easyconfigs/issues/5776 - if build_option('lib64_lib_symlink'): - if os.path.exists(lib_dir) and not os.path.exists(lib64_dir): - # create *relative* 'lib64' symlink to 'lib'; - # see https://github.com/easybuilders/easybuild-framework/issues/3564 - symlink('lib', lib64_dir, use_abspath_source=False) + if build_option('lib64_lib_symlink') and os.path.exists(lib_dir) and not os.path.exists(lib64_dir): + # create *relative* 'lib64' symlink to 'lib'; + # see https://github.com/easybuilders/easybuild-framework/issues/3564 + symlink('lib', lib64_dir, use_abspath_source=False) # symlink lib to lib64, which is helpful on OpenSUSE; # see https://github.com/easybuilders/easybuild-framework/issues/3549 - if build_option('lib_lib64_symlink'): - if os.path.exists(lib64_dir) and not os.path.exists(lib_dir): - # create *relative* 'lib' symlink to 'lib64'; - symlink('lib64', lib_dir, use_abspath_source=False) + if build_option('lib_lib64_symlink') and os.path.exists(lib64_dir) and not os.path.exists(lib_dir): + # create *relative* 'lib' symlink to 'lib64'; + symlink('lib64', lib_dir, use_abspath_source=False) + + # refresh symlink state in install_lib_symlink class variable + self.check_install_lib_symlink() self.run_post_install_commands() self.apply_post_install_patches() diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index dca2d587cc..9c836aa992 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -89,6 +89,7 @@ def __init__(self, *args, **kwargs): self.installdir = self.master.installdir self.modules_tool = self.master.modules_tool self.module_generator = self.master.module_generator + self.module_load_environment = self.master.module_load_environment self.robot_path = self.master.robot_path self.is_extension = True self.unpack_options = None diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index e02d9b923b..b5bb9f8cb3 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -168,13 +168,16 @@ LOCAL_VAR_NAMING_CHECK_WARN = WARN LOCAL_VAR_NAMING_CHECKS = [LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG, LOCAL_VAR_NAMING_CHECK_WARN] - OUTPUT_STYLE_AUTO = 'auto' OUTPUT_STYLE_BASIC = 'basic' OUTPUT_STYLE_NO_COLOR = 'no_color' OUTPUT_STYLE_RICH = 'rich' OUTPUT_STYLES = (OUTPUT_STYLE_AUTO, OUTPUT_STYLE_BASIC, OUTPUT_STYLE_NO_COLOR, OUTPUT_STYLE_RICH) +SEARCH_PATH_BIN_DIRS = ['bin'] +SEARCH_PATH_HEADER_DIRS = ['include'] +SEARCH_PATH_LIB_DIRS = ['lib', 'lib64'] + PYTHONPATH = 'PYTHONPATH' EBPYTHONPREFIXES = 'EBPYTHONPREFIXES' PYTHON_SEARCH_PATH_TYPES = [PYTHONPATH, EBPYTHONPREFIXES] diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 7feba289f3..e063c29b15 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -41,12 +41,13 @@ import os import re import shlex +from enum import Enum from easybuild.base import fancylogger from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_warning -from easybuild.tools.config import ERROR, IGNORE, PURGE, UNLOAD, UNSET -from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, LOADED_MODULES_ACTIONS +from easybuild.tools.config import ERROR, EBROOT_ENV_VAR_ACTIONS, IGNORE, LOADED_MODULES_ACTIONS, PURGE +from easybuild.tools.config import SEARCH_PATH_BIN_DIRS, SEARCH_PATH_HEADER_DIRS, SEARCH_PATH_LIB_DIRS, UNLOAD, UNSET from easybuild.tools.config import build_option, get_modules_tool, install_path from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env, setvar, unset_env_vars from easybuild.tools.filetools import convert_name, mkdir, normalize_path, path_matches, read_file, which, write_file @@ -55,6 +56,7 @@ from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.utilities import get_subclasses, nub + # software root/version environment variable name prefixes ROOT_ENV_VAR_NAME_PREFIX = "EBROOT" VERSION_ENV_VAR_NAME_PREFIX = "EBVERSION" @@ -131,6 +133,199 @@ _log = fancylogger.getLogger('modules', fname=False) +class ModEnvVarType(Enum): + """ + Possible types of ModuleEnvironmentVariable: + - STRING: (list of) strings with no further meaning + - PATH: (list of) of paths to existing directories or files + - PATH_WITH_FILES: (list of) of paths to existing directories containing + one or more files + - PATH_WITH_TOP_FILES: (list of) of paths to existing directories + containing one or more files in its top directory + - """ + STRING, PATH, PATH_WITH_FILES, PATH_WITH_TOP_FILES = range(0, 4) + + +class ModuleEnvironmentVariable: + """ + Environment variable data structure for modules + Contents of environment variable is a list of unique strings + """ + + def __init__(self, contents, var_type=None, delim=os.pathsep): + """ + Initialize new environment variable + Actual contents of the environment variable are held in self.contents + By default, environment variable is a (list of) paths with files in them + Existence of paths and their contents are not checked at init + """ + self.contents = contents + self.delim = delim + + if var_type is None: + var_type = ModEnvVarType.PATH_WITH_FILES + self.type = var_type + + self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) + + def __repr__(self): + return repr(self.contents) + + def __str__(self): + return self.delim.join(self.contents) + + def __iter__(self): + return iter(self.contents) + + @property + def contents(self): + return self._contents + + @contents.setter + def contents(self, value): + """Enforce that contents is a list of strings""" + if isinstance(value, str): + value = [value] + + try: + str_list = [str(path) for path in value] + except TypeError as err: + raise TypeError("ModuleEnvironmentVariable.contents must be a list of strings") from err + + self._contents = nub(str_list) # remove duplicates and keep order + + @property + def type(self): + return self._type + + @type.setter + def type(self, value): + """Convert type to VarType""" + if isinstance(value, ModEnvVarType): + self._type = value + else: + try: + self._type = ModEnvVarType[value] + except KeyError as err: + raise EasyBuildError(f"Cannot create ModuleEnvironmentVariable with type {value}") from err + + def append(self, item): + """Shortcut to append to list of contents""" + self.contents += [item] + + def extend(self, item): + """Shortcut to extend list of contents""" + self.contents += item + + def prepend(self, item): + """Shortcut to prepend item to list of contents""" + self.contents = [item] + self.contents + + def update(self, item): + """Shortcut to replace list of contents with item""" + self.contents = item + + def remove(self, *args): + """Shortcut to remove items from list of contents""" + try: + self.contents.remove(*args) + except ValueError: + # item is not in the list, move along + self.log.debug(f"ModuleEnvironmentVariable does not contain item: {' '.join(args)}") + + @property + def is_path(self): + path_like_types = [ + ModEnvVarType.PATH, + ModEnvVarType.PATH_WITH_FILES, + ModEnvVarType.PATH_WITH_TOP_FILES, + ] + return self.type in path_like_types + + +class ModuleLoadEnvironment: + """Changes to environment variables that should be made when environment module is loaded""" + + def __init__(self): + """ + Initialize default environment definition + Paths are relative to root of installation directory + """ + self.ACLOCAL_PATH = [os.path.join('share', 'aclocal')] + self.CLASSPATH = ['*.jar'] + self.CMAKE_LIBRARY_PATH = ['lib64'] # only needed for installations with standalone lib64 + self.CMAKE_PREFIX_PATH = [''] + self.CPATH = SEARCH_PATH_HEADER_DIRS + self.GI_TYPELIB_PATH = [os.path.join(x, 'girepository-*') for x in SEARCH_PATH_LIB_DIRS] + self.LD_LIBRARY_PATH = SEARCH_PATH_LIB_DIRS + self.LIBRARY_PATH = SEARCH_PATH_LIB_DIRS + self.MANPATH = ['man', os.path.join('share', 'man')] + self.PATH = SEARCH_PATH_BIN_DIRS + ['sbin'] + self.PKG_CONFIG_PATH = [os.path.join(x, 'pkgconfig') for x in SEARCH_PATH_LIB_DIRS + ['share']] + self.XDG_DATA_DIRS = ['share'] + + def __setattr__(self, name, value): + """ + Specific restrictions for ModuleLoadEnvironment attributes: + - attribute names are uppercase + - attributes are instances of ModuleEnvironmentVariable + """ + if name != name.upper(): + raise EasyBuildError(f"Names of ModuleLoadEnvironment attributes must be uppercase, got '{name}'") + try: + (contents, kwargs) = value + except ValueError: + contents, kwargs = value, {} + + if not isinstance(kwargs, dict): + contents, kwargs = value, {} + + # special variables that require files in their top directories + if name in ('LD_LIBRARY_PATH', 'PATH'): + kwargs.update({'var_type': ModEnvVarType.PATH_WITH_TOP_FILES}) + + return super().__setattr__(name, ModuleEnvironmentVariable(contents, **kwargs)) + + def __iter__(self): + """Make the class iterable""" + yield from self.__dict__ + + def items(self): + """ + Return key-value pairs for each attribute that is a ModuleEnvironmentVariable + - key = attribute name + - value = its "contents" attribute + """ + for attr in self.__dict__: + yield attr, getattr(self, attr) + + def update(self, new_env): + """Update contents of environment from given dictionary""" + try: + for envar_name, envar_contents in new_env.items(): + setattr(self, envar_name, envar_contents) + except AttributeError as err: + raise EasyBuildError("Cannot update ModuleLoadEnvironment from a non-dict variable") from err + + @property + def as_dict(self): + """ + Return dict with mapping of ModuleEnvironmentVariables names with their contents + """ + return dict(self.items()) + + @property + def environ(self): + """ + Return dict with mapping of ModuleEnvironmentVariables names with their contents + Equivalent in shape to os.environ + """ + mapping = {} + for envar_name, envar_contents in self.items(): + mapping.update({envar_name: str(envar_contents)}) + return mapping + + class ModulesTool(object): """An abstract interface to a tool that deals with modules.""" # name of this modules tool (used in log/warning/error messages) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 2eb654ecae..009f1ae281 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -41,7 +41,7 @@ import easybuild.tools.systemtools as st from easybuild.base import fancylogger -from easybuild.framework.easyblock import EasyBlock, get_easyblock_instance +from easybuild.framework.easyblock import EasyBlock, LibSymlink, get_easyblock_instance from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.framework.easyconfig.tools import avail_easyblocks, process_easyconfig @@ -52,7 +52,7 @@ from easybuild.tools.filetools import change_dir, copy_dir, copy_file, mkdir, read_file, remove_dir, remove_file from easybuild.tools.filetools import verify_checksum, write_file from easybuild.tools.module_generator import module_generator -from easybuild.tools.modules import EnvironmentModules, Lmod, reset_module_caches +from easybuild.tools.modules import EnvironmentModules, Lmod, ModEnvVarType, reset_module_caches from easybuild.tools.version import get_git_revision, this_is_easybuild @@ -434,12 +434,14 @@ def test_make_module_req(self): # create fake directories and files that should be guessed os.makedirs(eb.installdir) + for path in ('bin', ('bin', 'testdir'), 'sbin', 'share', ('share', 'man'), 'lib', 'lib64'): + path_components = (path, ) if isinstance(path, str) else path + os.mkdir(os.path.join(eb.installdir, *path_components)) + eb.install_lib_symlink = LibSymlink.NEITHER + write_file(os.path.join(eb.installdir, 'foo.jar'), 'foo.jar') write_file(os.path.join(eb.installdir, 'bla.jar'), 'bla.jar') - for path in ('bin', ('bin', 'testdir'), 'sbin', 'share', ('share', 'man'), 'lib', 'lib64'): - if isinstance(path, str): - path = (path, ) - os.mkdir(os.path.join(eb.installdir, *path)) + write_file(os.path.join(eb.installdir, 'share', 'man', 'pi'), 'Man page') # this is not a path that should be picked up os.mkdir(os.path.join(eb.installdir, 'CPATH')) @@ -501,6 +503,7 @@ def test_make_module_req(self): write_file(os.path.join(eb.installdir, 'lib', 'libfoo.so'), 'test') shutil.rmtree(os.path.join(eb.installdir, 'lib64')) os.symlink('lib', os.path.join(eb.installdir, 'lib64')) + eb.install_lib_symlink = LibSymlink.LIB64_TO_LIB with eb.module_generator.start_module_creation(): guess = eb.make_module_req() if get_module_syntax() == 'Tcl': @@ -519,8 +522,18 @@ def test_make_module_req(self): self.assertEqual(len(re.findall(r'^prepend_path\("%s", pathJoin\(root, "lib"\)\)$' % var, guess, re.M)), 1) - # check for behavior when a string value is used as dict value by make_module_req_guesses - eb.make_module_req_guess = lambda: {'PATH': 'bin'} + # nuke default module load environment + default_mod_load_vars = [ + 'ACLOCAL_PATH', 'CLASSPATH', 'CMAKE_PREFIX_PATH', 'CMAKE_LIBRARY_PATH', 'CPATH', 'GI_TYPELIB_PATH', + 'LD_LIBRARY_PATH', 'LIBRARY_PATH', 'MANPATH', 'PATH', 'PKG_CONFIG_PATH', 'XDG_DATA_DIRS', + ] + for env_var in default_mod_load_vars: + delattr(eb.module_load_environment, env_var) + + self.assertEqual(len(vars(eb.module_load_environment)), 0) + + # check for behavior when a string value is used as value of module_load_environment + eb.module_load_environment.PATH = 'bin' with eb.module_generator.start_module_creation(): txt = eb.make_module_req() if get_module_syntax() == 'Tcl': @@ -532,7 +545,7 @@ def test_make_module_req(self): # check for correct behaviour if empty string is specified as one of the values # prepend-path statements should be included for both the 'bin' subdir and the install root - eb.make_module_req_guess = lambda: {'PATH': ['bin', '']} + eb.module_load_environment.PATH = ['bin', ''] with eb.module_generator.start_module_creation(): txt = eb.make_module_req() if get_module_syntax() == 'Tcl': @@ -545,7 +558,7 @@ def test_make_module_req(self): self.fail("Unknown module syntax: %s" % get_module_syntax()) # check for correct order of prepend statements when providing a list (and that no duplicates are allowed) - eb.make_module_req_guess = lambda: {'LD_LIBRARY_PATH': ['lib/pathC', 'lib/pathA', 'lib/pathB', 'lib/pathA']} + eb.module_load_environment.LD_LIBRARY_PATH = ['lib/pathC', 'lib/pathA', 'lib/pathB', 'lib/pathA'] for path in ['pathA', 'pathB', 'pathC']: os.mkdir(os.path.join(eb.installdir, 'lib', path)) write_file(os.path.join(eb.installdir, 'lib', path, 'libfoo.so'), 'test') @@ -573,7 +586,8 @@ def test_make_module_req(self): # If PATH or LD_LIBRARY_PATH contain only folders, do not add an entry sub_lib_path = os.path.join('lib', 'path_folders') sub_path_path = os.path.join('bin', 'path_folders') - eb.make_module_req_guess = lambda: {'LD_LIBRARY_PATH': sub_lib_path, 'PATH': sub_path_path} + eb.module_load_environment.LD_LIBRARY_PATH = sub_lib_path + eb.module_load_environment.PATH = sub_path_path for path in (sub_lib_path, sub_path_path): full_path = os.path.join(eb.installdir, path, 'subpath') os.makedirs(full_path) @@ -3036,6 +3050,132 @@ def test_create_easyblock_without_logfile(self): os.remove(eb.logfile) + def test_expand_module_search_path(self): + """Testcase for expand_module_search_path""" + top_dir = os.path.abspath(os.path.dirname(__file__)) + toy_ec = os.path.join(top_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + eb = EasyBlock(EasyConfig(toy_ec)) + eb.installdir = config.install_path() + test_emsp = eb.expand_module_search_path # shortcut + + # create test directories and files + os.makedirs(eb.installdir) + test_directories = ( + 'empty_dir', + 'dir_empty_subdir', + ('dir_empty_subdir', 'empty_subdir'), + 'dir_with_file', + 'dir_full_subdirs', + ('dir_full_subdirs', 'subdir1'), + ('dir_full_subdirs', 'subdir2'), + ) + for path in test_directories: + path_components = (path, ) if isinstance(path, str) else path + os.mkdir(os.path.join(eb.installdir, *path_components)) + + write_file(os.path.join(eb.installdir, 'dir_with_file', 'file.txt'), 'test file') + write_file(os.path.join(eb.installdir, 'dir_full_subdirs', 'subdir1', 'file11.txt'), 'test file 1.1') + write_file(os.path.join(eb.installdir, 'dir_full_subdirs', 'subdir1', 'file12.txt'), 'test file 1.2') + write_file(os.path.join(eb.installdir, 'dir_full_subdirs', 'subdir2', 'file21.txt'), 'test file 2.1') + + self.assertEqual(eb.install_lib_symlink, LibSymlink.UNKNOWN) + eb.check_install_lib_symlink() + self.assertEqual(eb.install_lib_symlink, LibSymlink.NEITHER) + + self.assertEqual(test_emsp("nonexistent", ModEnvVarType.PATH), []) + self.assertEqual(test_emsp("nonexistent", ModEnvVarType.PATH_WITH_FILES), []) + self.assertEqual(test_emsp("nonexistent", ModEnvVarType.PATH_WITH_TOP_FILES), []) + self.assertEqual(test_emsp("empty_dir", ModEnvVarType.PATH), ["empty_dir"]) + self.assertEqual(test_emsp("empty_dir", ModEnvVarType.PATH_WITH_FILES), []) + self.assertEqual(test_emsp("empty_dir", ModEnvVarType.PATH_WITH_TOP_FILES), []) + self.assertEqual(test_emsp("dir_empty_subdir", ModEnvVarType.PATH), ["dir_empty_subdir"]) + self.assertEqual(test_emsp("dir_empty_subdir", ModEnvVarType.PATH_WITH_FILES), []) + self.assertEqual(test_emsp("dir_empty_subdir", ModEnvVarType.PATH_WITH_TOP_FILES), []) + self.assertEqual(test_emsp("dir_with_file", ModEnvVarType.PATH), ["dir_with_file"]) + self.assertEqual(test_emsp("dir_with_file", ModEnvVarType.PATH_WITH_FILES), ["dir_with_file"]) + self.assertEqual(test_emsp("dir_with_file", ModEnvVarType.PATH_WITH_TOP_FILES), ["dir_with_file"]) + self.assertEqual(test_emsp("dir_full_subdirs", ModEnvVarType.PATH), ["dir_full_subdirs"]) + self.assertEqual(test_emsp("dir_full_subdirs", ModEnvVarType.PATH_WITH_FILES), ["dir_full_subdirs"]) + self.assertEqual(test_emsp("dir_full_subdirs", ModEnvVarType.PATH_WITH_TOP_FILES), []) + + # test globs + ref_expanded_paths = ["dir_empty_subdir/empty_subdir"] + self.assertEqual(test_emsp("dir_empty_subdir/*", ModEnvVarType.PATH), ref_expanded_paths) + self.assertEqual(test_emsp("dir_empty_subdir/*", ModEnvVarType.PATH_WITH_FILES), []) + self.assertEqual(test_emsp("dir_empty_subdir/*", ModEnvVarType.PATH_WITH_TOP_FILES), []) + ref_expanded_paths = ["dir_full_subdirs/subdir1", "dir_full_subdirs/subdir2"] + self.assertEqual(sorted(test_emsp("dir_full_subdirs/*", ModEnvVarType.PATH)), ref_expanded_paths) + self.assertEqual(sorted(test_emsp("dir_full_subdirs/*", ModEnvVarType.PATH_WITH_FILES)), ref_expanded_paths) + self.assertEqual(sorted(test_emsp("dir_full_subdirs/*", ModEnvVarType.PATH_WITH_TOP_FILES)), ref_expanded_paths) + ref_expanded_paths = ["dir_full_subdirs/subdir2/file21.txt"] + self.assertEqual(test_emsp("dir_full_subdirs/subdir2/*", ModEnvVarType.PATH), ref_expanded_paths) + self.assertEqual(test_emsp("dir_full_subdirs/subdir2/*", ModEnvVarType.PATH_WITH_FILES), ref_expanded_paths) + self.assertEqual(test_emsp("dir_full_subdirs/subdir2/*", ModEnvVarType.PATH_WITH_TOP_FILES), ref_expanded_paths) + self.assertEqual(test_emsp("nonexistent/*", True), []) + self.assertEqual(test_emsp("nonexistent/*", ModEnvVarType.PATH), []) + self.assertEqual(test_emsp("nonexistent/*", ModEnvVarType.PATH_WITH_FILES), []) + self.assertEqual(test_emsp("nonexistent/*", ModEnvVarType.PATH_WITH_TOP_FILES), []) + + # state of install_lib_symlink should not have changed + self.assertEqual(eb.install_lib_symlink, LibSymlink.NEITHER) + + # test just one lib directory + os.mkdir(os.path.join(eb.installdir, "lib")) + eb.check_install_lib_symlink() + self.assertEqual(eb.install_lib_symlink, LibSymlink.NEITHER) + self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib"]) + self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), []) + self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), []) + write_file(os.path.join(eb.installdir, "lib", "libtest.so"), "not actually a lib") + self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib"]) + self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), ["lib"]) + self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"]) + + # test both lib and lib64 directories + os.mkdir(os.path.join(eb.installdir, "lib64")) + eb.check_install_lib_symlink() + self.assertEqual(eb.install_lib_symlink, LibSymlink.NEITHER) + self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH)), ["lib", "lib64"]) + self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES), ["lib"]) + self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"]) + write_file(os.path.join(eb.installdir, "lib64", "libtest.so"), "not actually a lib") + self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH)), ["lib", "lib64"]) + self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES)), ["lib", "lib64"]) + self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES)), ["lib", "lib64"]) + + # test lib64 symlinked to lib + remove_dir(os.path.join(eb.installdir, "lib64")) + os.symlink("lib", os.path.join(eb.installdir, "lib64")) + eb.check_install_lib_symlink() + self.assertEqual(eb.install_lib_symlink, LibSymlink.LIB64_TO_LIB) + self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib"]) + self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), ["lib"]) + self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"]) + self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH), []) + self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_FILES), []) + self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_TOP_FILES), []) + self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH), ["lib"]) + self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES), ["lib"]) + self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"]) + + # test lib symlinked to lib64 + remove_dir(os.path.join(eb.installdir, "lib")) + remove_file(os.path.join(eb.installdir, "lib64")) + os.mkdir(os.path.join(eb.installdir, "lib64")) + write_file(os.path.join(eb.installdir, "lib64", "libtest.so"), "not actually a lib") + os.symlink("lib64", os.path.join(eb.installdir, "lib")) + eb.check_install_lib_symlink() + self.assertEqual(eb.install_lib_symlink, LibSymlink.LIB_TO_LIB64) + self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), []) + self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), []) + self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), []) + self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH), ["lib64"]) + self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_FILES), ["lib64"]) + self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib64"]) + self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH), ["lib64"]) + self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES), ["lib64"]) + self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib64"]) + def suite(): """ return all the tests in this file """ diff --git a/test/framework/modules.py b/test/framework/modules.py index bdd1e595a0..ef3d986c06 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1599,6 +1599,145 @@ def test_get_setenv_value_from_modulefile(self): res = self.modtool.get_setenv_value_from_modulefile('toy/0.0', 'NO_SUCH_VARIABLE_SET') self.assertEqual(res, None) + def test_module_environment_variable(self): + """Test for ModuleEnvironmentVariable object""" + test_paths = ['lib', 'lib64'] + mod_envar = mod.ModuleEnvironmentVariable(test_paths) + self.assertTrue(hasattr(mod_envar, 'contents')) + self.assertTrue(hasattr(mod_envar, 'type')) + self.assertTrue(hasattr(mod_envar, 'delim')) + self.assertEqual(mod_envar.contents, test_paths) + self.assertEqual(repr(mod_envar), repr(test_paths)) + self.assertEqual(str(mod_envar), 'lib:lib64') + + mod_envar_custom_delim = mod.ModuleEnvironmentVariable(test_paths, delim='|') + self.assertEqual(mod_envar_custom_delim.contents, test_paths) + self.assertEqual(repr(mod_envar_custom_delim), repr(test_paths)) + self.assertEqual(str(mod_envar_custom_delim), 'lib|lib64') + + mod_envar_custom_type = mod.ModuleEnvironmentVariable(test_paths, var_type='STRING') + self.assertEqual(mod_envar_custom_type.contents, test_paths) + self.assertEqual(mod_envar_custom_type.type, mod.ModEnvVarType.STRING) + self.assertEqual(mod_envar_custom_type.is_path, False) + mod_envar_custom_type.type = 'PATH' + self.assertEqual(mod_envar_custom_type.type, mod.ModEnvVarType.PATH) + self.assertEqual(mod_envar_custom_type.is_path, True) + + mod_envar_custom_type.type = mod.ModEnvVarType.PATH + self.assertEqual(mod_envar_custom_type.is_path, True) + + mod_envar_custom_type.type = 'PATH_WITH_FILES' + self.assertEqual(mod_envar_custom_type.type, mod.ModEnvVarType.PATH_WITH_FILES) + self.assertEqual(mod_envar_custom_type.is_path, True) + + mod_envar_custom_type.type = mod.ModEnvVarType.PATH_WITH_FILES + self.assertEqual(mod_envar_custom_type.is_path, True) + + mod_envar_custom_type.type = 'PATH_WITH_TOP_FILES' + self.assertEqual(mod_envar_custom_type.type, mod.ModEnvVarType.PATH_WITH_TOP_FILES) + self.assertEqual(mod_envar_custom_type.is_path, True) + + mod_envar_custom_type.type = mod.ModEnvVarType.PATH_WITH_TOP_FILES + self.assertEqual(mod_envar_custom_type.is_path, True) + + self.assertRaises(EasyBuildError, setattr, mod_envar_custom_type, 'type', 'NONEXISTENT') + self.assertRaises(EasyBuildError, mod.ModuleEnvironmentVariable, test_paths, 'NONEXISTENT') + + mod_envar.contents = [] + self.assertEqual(mod_envar.contents, []) + self.assertRaises(TypeError, setattr, mod_envar, 'contents', None) + mod_envar.contents = (1, 3, 2, 3) + self.assertEqual(mod_envar.contents, ['1', '3', '2']) + mod_envar.contents = 'include' + self.assertEqual(mod_envar.contents, ['include']) + + mod_envar.append('share') + self.assertEqual(mod_envar.contents, ['include', 'share']) + mod_envar.append('share') + self.assertEqual(mod_envar.contents, ['include', 'share']) + self.assertRaises(TypeError, mod_envar.append, 'arg1', 'arg2') + + mod_envar.extend(test_paths) + self.assertEqual(mod_envar.contents, ['include', 'share', 'lib', 'lib64']) + mod_envar.extend(test_paths) + self.assertEqual(mod_envar.contents, ['include', 'share', 'lib', 'lib64']) + mod_envar.extend(test_paths + ['lib128']) + self.assertEqual(mod_envar.contents, ['include', 'share', 'lib', 'lib64', 'lib128']) + self.assertRaises(TypeError, mod_envar.append, ['list1'], ['list2']) + + mod_envar.remove('lib128') + self.assertEqual(mod_envar.contents, ['include', 'share', 'lib', 'lib64']) + mod_envar.remove('nonexistent') + self.assertEqual(mod_envar.contents, ['include', 'share', 'lib', 'lib64']) + self.assertRaises(TypeError, mod_envar.remove, 'arg1', 'arg2') + + mod_envar.prepend('bin') + self.assertEqual(mod_envar.contents, ['bin', 'include', 'share', 'lib', 'lib64']) + + mod_envar.update('new_path') + self.assertEqual(mod_envar.contents, ['new_path']) + mod_envar.update(['new_path_1', 'new_path_2']) + self.assertEqual(mod_envar.contents, ['new_path_1', 'new_path_2']) + self.assertRaises(TypeError, mod_envar.update, 'arg1', 'arg2') + + def test_module_load_environment(self): + """Test for ModuleLoadEnvironment object""" + mod_load_env = mod.ModuleLoadEnvironment() + + # test setting attributes + test_contents = ['lib', 'lib64'] + mod_load_env.TEST_VAR = test_contents + self.assertTrue(hasattr(mod_load_env, 'TEST_VAR')) + self.assertEqual(mod_load_env.TEST_VAR.contents, test_contents) + + error_pattern = "Names of ModuleLoadEnvironment attributes must be uppercase, got 'test_lower'" + self.assertErrorRegex(EasyBuildError, error_pattern, setattr, mod_load_env, 'test_lower', test_contents) + + mod_load_env.TEST_STR = 'some/path' + self.assertTrue(hasattr(mod_load_env, 'TEST_STR')) + self.assertEqual(mod_load_env.TEST_STR.contents, ['some/path']) + + mod_load_env.TEST_VARTYPE = (test_contents, {'var_type': "STRING"}) + self.assertTrue(hasattr(mod_load_env, 'TEST_VARTYPE')) + self.assertEqual(mod_load_env.TEST_VARTYPE.contents, test_contents) + self.assertEqual(mod_load_env.TEST_VARTYPE.type, mod.ModEnvVarType.STRING) + + mod_load_env.TEST_VARTYPE.type = "PATH" + self.assertEqual(mod_load_env.TEST_VARTYPE.type, mod.ModEnvVarType.PATH) + self.assertRaises(TypeError, setattr, mod_load_env, 'TEST_UNKNONW', (test_contents, {'unkown_param': True})) + + # test retrieving environment + ref_load_env = mod_load_env.__dict__.copy() + self.assertCountEqual(list(mod_load_env), ref_load_env.keys()) + + ref_load_env_item_list = list(ref_load_env.items()) + self.assertCountEqual(list(mod_load_env.items()), ref_load_env_item_list) + + ref_load_env_item_list = dict(ref_load_env.items()) + self.assertCountEqual(mod_load_env.as_dict, ref_load_env_item_list) + + ref_load_env_environ = {key: str(value) for key, value in ref_load_env.items()} + self.assertDictEqual(mod_load_env.environ, ref_load_env_environ) + + # test updating environment + new_test_env = { + 'TEST_VARTYPE': 'replaced_path', + 'TEST_NEW_VAR': ['new_path1', 'new_path2'], + } + mod_load_env.update(new_test_env) + self.assertTrue(hasattr(mod_load_env, 'TEST_VARTYPE')) + self.assertEqual(mod_load_env.TEST_VARTYPE.contents, ['replaced_path']) + self.assertEqual(mod_load_env.TEST_VARTYPE.type, mod.ModEnvVarType.PATH_WITH_FILES) + self.assertTrue(hasattr(mod_load_env, 'TEST_NEW_VAR')) + self.assertEqual(mod_load_env.TEST_NEW_VAR.contents, ['new_path1', 'new_path2']) + self.assertEqual(mod_load_env.TEST_NEW_VAR.type, mod.ModEnvVarType.PATH_WITH_FILES) + + # check that previous variables still exist + self.assertTrue(hasattr(mod_load_env, 'TEST_VAR')) + self.assertEqual(mod_load_env.TEST_VAR.contents, test_contents) + self.assertTrue(hasattr(mod_load_env, 'TEST_STR')) + self.assertEqual(mod_load_env.TEST_STR.contents, ['some/path']) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index 5727dcd0e0..2957973d83 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -73,6 +73,10 @@ def __init__(self, *args, **kwargs): setvar('TOY', '%s-%s' % (self.name, self.version)) + # extra paths for environment variables to consider + if self.name == 'toy': + self.module_load_environment.CPATH.append('toy-headers') + def prepare_for_extensions(self): """ Prepare for installing toy extensions. diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index b386ba550a..a4711f7589 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -4285,6 +4285,48 @@ def test_toy_python(self): self.assertTrue(ebpythonprefixes_regex.search(toy_mod_txt), f"Pattern '{ebpythonprefixes_regex.pattern}' found in: {toy_mod_txt}") + def test_toy_multiple_ecs_module(self): + """ + Verify whether module file is correct when multiple easyconfigs are being installed. + """ + test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + + # modify 'toy' easyconfig so toy-headers subdirectory is created, + # which is taken into account by EB_toy easyblock for $CPATH + test_toy_ec = os.path.join(self.test_prefix, 'test-toy.eb') + toy_ec_txt = read_file(toy_ec) + toy_ec_txt += "\npostinstallcmds += ['mkdir %(installdir)s/toy-headers']" + toy_ec_txt += "\npostinstallcmds += ['touch %(installdir)s/toy-headers/toy.h']" + write_file(test_toy_ec, toy_ec_txt) + + # modify 'toy-app' easyconfig so toy-headers subdirectory is created, + # which is consider by EB_toy easyblock for $CPATH, + # but should *not* be actually used because software name is not 'toy' + toy_app_ec = os.path.join(test_ecs, 't', 'toy-app', 'toy-app-0.0.eb') + test_toy_app_ec = os.path.join(self.test_prefix, 'test-toy-app.eb') + toy_ec_txt = read_file(toy_app_ec) + toy_ec_txt += "\npostinstallcmds += ['mkdir %(installdir)s/toy-headers']" + toy_ec_txt += "\npostinstallcmds += ['touch %(installdir)s/toy-headers/toy.h']" + write_file(test_toy_app_ec, toy_ec_txt) + + self.run_test_toy_build_with_output(ec_file=test_toy_ec, extra_args=[test_toy_app_ec], raise_error=True) + + toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_mod += '.lua' + toy_modtxt = read_file(toy_mod) + regex = re.compile('prepend[-_]path.*CPATH.*toy-headers', re.M) + self.assertTrue(regex.search(toy_modtxt), + f"Pattern '{regex.pattern}' should be found in: {toy_modtxt}") + + toy_app_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy-app', '0.0') + if get_module_syntax() == 'Lua': + toy_app_mod += '.lua' + toy_app_modtxt = read_file(toy_app_mod) + self.assertFalse(regex.search(toy_app_modtxt), + f"Pattern '{regex.pattern}' should *not* be found in: {toy_app_modtxt}") + def suite(): """ return all the tests in this file """