diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst index 2ba3858f7..66145f8ca 100644 --- a/doc/source/NewSiteConfigs.rst +++ b/doc/source/NewSiteConfigs.rst @@ -277,11 +277,12 @@ Remember to activate the ``lua`` module environment and have MacTeX in your sear It is recommended to save the output of concretize in a log file and inspect that log file using the :ref:`show_duplicate_packages.py ` utility. This is done to find and eliminate duplicate package specifications which can cause issues at the module creation step below. Note that in the unified environment, there may be deliberate duplicates; consult the specs in spack.yaml to determine which ones are desired. +See the :ref:`documentation ` for usage information including command line options. .. code-block:: console spack concretize 2>&1 | tee log.concretize - util/show_duplicate_packages.py -d log.concretize + util/show_duplicate_packages.py -d [-c] log.concretize spack install [--verbose] [--fail-fast] 11. Create lmod module files @@ -526,11 +527,12 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi It is recommended to save the output of concretize in a log file and inspect that log file using the :ref:`show_duplicate_packages.py ` utility. This is done to find and eliminate duplicate package specifications which can cause issues at the module creation step below. Note that in the unified environment, there may be deliberate duplicates; consult the specs in spack.yaml to determine which ones are desired. +See the :ref:`documentation ` for usage information including command line options. .. code-block:: console spack concretize 2>&1 | tee log.concretize - util/show_duplicate_packages.py -d log.concretize + util/show_duplicate_packages.py -d [-c] log.concretize spack install [--verbose] [--fail-fast] 13. Create tcl module files (replace ``tcl`` with ``lmod`` if you have manually installed lmod) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 1228df208..5726946e7 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -750,7 +750,7 @@ The following instructions install a new spack environment on a pre-configured s emacs envs/unified-dev.hera/site/*.yaml # Process/concretize the specs; optionally check for duplicate packages - spack concretize | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize + spack concretize | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d [-c] log.concretize # Optional step for systems with a pre-configured spack mirror, see below. diff --git a/doc/source/Utilities.rst b/doc/source/Utilities.rst index 7e314a276..fad759688 100644 --- a/doc/source/Utilities.rst +++ b/doc/source/Utilities.rst @@ -18,7 +18,7 @@ The utility located at util/show_duplicate_packages.py parses the output of ``sp spack concretize |& tee log.concretize ${SPACK_STACK_DIR}/util/show_duplicate_packages.py log.concretize -The ``-d`` option shows only a list of the duplicates, as opposed to the default behavior, which is to show a print-out of all packages with colorized duplicates. In any case, the identification of any duplicates will yield a return code of 1. The ``-i`` option can be invoked multiple times to skip specific package names. +The ``-d`` option shows only a list of the duplicates, as opposed to the default behavior, which is to show a print-out of all packages with colorized duplicates. In any case, the identification of any duplicates will yield a return code of 1. The ``-i`` option can be invoked multiple times to skip specific package names. The ``-c`` option can be used to ignore duplicates associated with different compilers; in an environment with, say, GCC and Intel copies of any given package, those two copies of a package will not be reported as duplicates. .. _Permissions_Checker: diff --git a/util/show_duplicate_packages.py b/util/show_duplicate_packages.py index 364c61edd..fd9b605f1 100755 --- a/util/show_duplicate_packages.py +++ b/util/show_duplicate_packages.py @@ -21,7 +21,7 @@ def colorize_spec(line, package_name, colorize=False): c1 = r'\033[93m' ; c2 = r'\033[0m' return re.sub("(\w{7}\s+)(%s)@"%package_name, f"\\1{c1}\\2{c2}@", line) -def show_duplicate_packages(txt_to_check, ignore_list=[], only_show_dups=False): +def show_duplicate_packages(txt_to_check, ignore_list=[], only_show_dups=False, include_compiler=False): dd = defaultdict(set) for line in txt_to_check.split("\n"): line = line.replace("^", "") @@ -29,6 +29,9 @@ def show_duplicate_packages(txt_to_check, ignore_list=[], only_show_dups=False): if not package_name: continue if [package_name[0]] in ignore_list: continue line = " ".join(line.split()[1:]) + if include_compiler: + compiler = re.findall(r"%[^@]+", line)[0] + package_name[0] += compiler dd[package_name[0]].add(line) duplicates_found = False for key in sorted(dd.keys()): @@ -38,7 +41,7 @@ def show_duplicate_packages(txt_to_check, ignore_list=[], only_show_dups=False): else: colorize = multiple if multiple: duplicates_found = True for line in dd[key]: - print(colorize_spec(line, key, colorize=colorize)) + print(colorize_spec(line, re.sub(r"%.+", "", key), colorize=colorize)) sys.stderr.write("===\n%suplicates found%s\n" % (("D","!") if duplicates_found else ("No d","."))) sys.stderr.flush() return int(duplicates_found) @@ -48,12 +51,12 @@ def show_duplicate_packages(txt_to_check, ignore_list=[], only_show_dups=False): parser.add_argument("filename", nargs="?", help="'log.concretize' or other concretization output; if not set, stdin will be used") parser.add_argument("-d", action="store_true", help="Only show duplicates (default output is colorized list of all packages)") parser.add_argument("-i", default=[], nargs="*", action="append", help="Ignore package name (e.g., 'hdf5', 'netcdf-c')") + parser.add_argument("-c", action="store_true", help="Ignore duplicates that have different compilers") args = parser.parse_args() if args.filename: with open(args.filename, "r") as f: txt_to_check = f.read() else: txt_to_check = sys.stdin.read() - ret = show_duplicate_packages(txt_to_check, only_show_dups=args.d, ignore_list=args.i) + ret = show_duplicate_packages(txt_to_check, only_show_dups=args.d, ignore_list=args.i, include_compiler=args.c) sys.exit(ret) - diff --git a/util/util_tests.sh b/util/util_tests.sh index 58a105be8..8d6bcfbd3 100755 --- a/util/util_tests.sh +++ b/util/util_tests.sh @@ -45,9 +45,11 @@ run_and_check 1 "check_permissions G" ${SPACK_STACK_DIR}/util/check_permissions. # Check show_duplicate_packages.py cd ${SPACK_STACK_DIR}/util/checks -echo -e " - abcdefg hdf6@1.2.3\n - tuvwxyz hdf6@1.2.3" > fakeconcrete.A +echo -e " - abcdefg hdf6@1.2.3%intel\n - tuvwxyz hdf6@1.2.3%gcc" > fakeconcrete.A run_and_check 1 "show_duplicate_packages.py A1" ${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.A run_and_check 1 "show_duplicate_packages.py A2" "cat fakeconcrete.A | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py" +run_and_check 0 "show_duplicate_packages.py A3" ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -c fakeconcrete.A +run_and_check 0 "show_duplicate_packages.py A4" "cat fakeconcrete.A | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -c" echo -e " - abcdefg hdf6@1.2.3\n - tuvwxyz hdf6@1.2.4" > fakeconcrete.B run_and_check 1 "show_duplicate_packages.py B1" ${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.B run_and_check 1 "show_duplicate_packages.py B2" "cat fakeconcrete.B | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py"