Merge pull request JCSDA#732 from AlexanderRichert-NOAA/dupcheck_comp…

…iler

Allow show_duplicate_packages.py to ignore dup's from different compilers

doc/source/NewSiteConfigs.rst

@@ -277,11 +277,12 @@ Remember to activate the ``lua`` module environment and have MacTeX in your sear
 It is recommended to save the output of concretize in a log file and inspect that log file using the :ref:`show_duplicate_packages.py <Duplicate_Checker>` utility.
 This is done to find and eliminate duplicate package specifications which can cause issues at the module creation step below.
 Note that in the unified environment, there may be deliberate duplicates; consult the specs in spack.yaml to determine which ones are desired.
+See the :ref:`documentation <Duplicate_Checker>` for usage information including command line options.
 
 .. code-block:: console
 
    spack concretize 2>&1 | tee log.concretize
-   util/show_duplicate_packages.py -d log.concretize
+   util/show_duplicate_packages.py -d [-c] log.concretize
    spack install [--verbose] [--fail-fast]
 
 11. Create lmod module files
@@ -526,11 +527,12 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi
 It is recommended to save the output of concretize in a log file and inspect that log file using the :ref:`show_duplicate_packages.py <Duplicate_Checker>` utility.
 This is done to find and eliminate duplicate package specifications which can cause issues at the module creation step below.
 Note that in the unified environment, there may be deliberate duplicates; consult the specs in spack.yaml to determine which ones are desired.
+See the :ref:`documentation <Duplicate_Checker>` for usage information including command line options.
 
 .. code-block:: console
 
    spack concretize 2>&1 | tee log.concretize
-   util/show_duplicate_packages.py -d log.concretize
+   util/show_duplicate_packages.py -d [-c] log.concretize
    spack install [--verbose] [--fail-fast]
 
 13. Create tcl module files (replace ``tcl`` with ``lmod`` if you have manually installed lmod)

doc/source/PreConfiguredSites.rst

@@ -750,7 +750,7 @@ The following instructions install a new spack environment on a pre-configured s
    emacs envs/unified-dev.hera/site/*.yaml
 
    # Process/concretize the specs; optionally check for duplicate packages
-   spack concretize | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize
+   spack concretize | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d [-c] log.concretize
 
    # Optional step for systems with a pre-configured spack mirror, see below.
 

doc/source/Utilities.rst

@@ -18,7 +18,7 @@ The utility located at util/show_duplicate_packages.py parses the output of ``sp
    spack concretize |& tee log.concretize
    ${SPACK_STACK_DIR}/util/show_duplicate_packages.py log.concretize
 
-The ``-d`` option shows only a list of the duplicates, as opposed to the default behavior, which is to show a print-out of all packages with colorized duplicates. In any case, the identification of any duplicates will yield a return code of 1. The ``-i`` option can be invoked multiple times to skip specific package names.
+The ``-d`` option shows only a list of the duplicates, as opposed to the default behavior, which is to show a print-out of all packages with colorized duplicates. In any case, the identification of any duplicates will yield a return code of 1. The ``-i`` option can be invoked multiple times to skip specific package names. The ``-c`` option can be used to ignore duplicates associated with different compilers; in an environment with, say, GCC and Intel copies of any given package, those two copies of a package will not be reported as duplicates.
 
 .. _Permissions_Checker:
 

util/show_duplicate_packages.py

@@ -21,14 +21,17 @@ def colorize_spec(line, package_name, colorize=False):
     c1 = r'\033[93m' ; c2 = r'\033[0m'
     return re.sub("(\w{7}\s+)(%s)@"%package_name, f"\\1{c1}\\2{c2}@", line)
 
-def show_duplicate_packages(txt_to_check, ignore_list=[], only_show_dups=False):
+def show_duplicate_packages(txt_to_check, ignore_list=[], only_show_dups=False, include_compiler=False):
     dd = defaultdict(set)
     for line in txt_to_check.split("\n"):
         line = line.replace("^", "")
         package_name = re.findall("\s\w{7}\s+(\^?[^\s@]+)@", line)
         if not package_name: continue
         if [package_name[0]] in ignore_list: continue
         line = "  ".join(line.split()[1:])
+        if include_compiler:
+          compiler = re.findall(r"%[^@]+", line)[0]
+          package_name[0] += compiler
         dd[package_name[0]].add(line)
     duplicates_found = False
     for key in sorted(dd.keys()):
@@ -38,7 +41,7 @@ def show_duplicate_packages(txt_to_check, ignore_list=[], only_show_dups=False):
         else: colorize = multiple
         if multiple: duplicates_found = True
         for line in dd[key]:
-            print(colorize_spec(line, key, colorize=colorize))
+            print(colorize_spec(line, re.sub(r"%.+", "", key), colorize=colorize))
     sys.stderr.write("===\n%suplicates found%s\n" % (("D","!") if duplicates_found else ("No d",".")))
     sys.stderr.flush()
     return int(duplicates_found)
@@ -48,12 +51,12 @@ def show_duplicate_packages(txt_to_check, ignore_list=[], only_show_dups=False):
     parser.add_argument("filename", nargs="?", help="'log.concretize' or other concretization output; if not set, stdin will be used")
     parser.add_argument("-d", action="store_true", help="Only show duplicates (default output is colorized list of all packages)")
     parser.add_argument("-i", default=[], nargs="*", action="append", help="Ignore package name (e.g., 'hdf5', 'netcdf-c')")
+    parser.add_argument("-c", action="store_true", help="Ignore duplicates that have different compilers")
     args = parser.parse_args()
     if args.filename:
         with open(args.filename, "r") as f:
             txt_to_check = f.read()
     else:
         txt_to_check = sys.stdin.read()
-    ret = show_duplicate_packages(txt_to_check, only_show_dups=args.d, ignore_list=args.i)
+    ret = show_duplicate_packages(txt_to_check, only_show_dups=args.d, ignore_list=args.i, include_compiler=args.c)
     sys.exit(ret)
-

util/util_tests.sh

@@ -45,9 +45,11 @@ run_and_check 1 "check_permissions G" ${SPACK_STACK_DIR}/util/check_permissions.
 
 # Check show_duplicate_packages.py
 cd ${SPACK_STACK_DIR}/util/checks
-echo -e " -  abcdefg [email protected]\n -  tuvwxyz [email protected]" > fakeconcrete.A
+echo -e " -  abcdefg [email protected]%intel\n -  tuvwxyz [email protected]%gcc" > fakeconcrete.A
 run_and_check 1 "show_duplicate_packages.py A1" ${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.A
 run_and_check 1 "show_duplicate_packages.py A2" "cat fakeconcrete.A | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py"
+run_and_check 0 "show_duplicate_packages.py A3" ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -c fakeconcrete.A
+run_and_check 0 "show_duplicate_packages.py A4" "cat fakeconcrete.A | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -c"
 echo -e " -  abcdefg [email protected]\n -  tuvwxyz [email protected]" > fakeconcrete.B
 run_and_check 1 "show_duplicate_packages.py B1" ${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.B
 run_and_check 1 "show_duplicate_packages.py B2" "cat fakeconcrete.B | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py"