Merge branch 'develop' into experiments/osu-micro-benchmarks

.github/workflows/run.yml

@@ -219,22 +219,23 @@ jobs:
             --disable-logger \
             workspace setup --dry-run
 
-      - name: Dry run laghos/mpi-only on LLNL-Magma-Penguin-icelake-OmniPath with allocation modifier
+      - name: Dry run dynamic laghos/mpi-only on LLNL-Magma-Penguin-icelake-OmniPath with allocation modifier
         run: |
-          ./bin/benchpark setup laghos/mpi-only LLNL-Magma-Penguin-icelake-OmniPath workspace/
+          ./bin/benchpark experiment init --dest=laghos-mpi-only laghos
+          ./bin/benchpark setup ./laghos-mpi-only LLNL-Magma-Penguin-icelake-OmniPath workspace/
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/laghos/mpi-only/LLNL-Magma-Penguin-icelake-OmniPath/workspace \
+            --workspace-dir workspace/laghos-mpi-only/LLNL-Magma-Penguin-icelake-OmniPath/workspace \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run
 
       - name: Dry run laghos/mpi-only on LLNL-Ruby-icelake-OmniPath with allocation modifier
         run: |
-          ./bin/benchpark setup laghos/mpi-only LLNL-Ruby-icelake-OmniPath workspace/
+          ./bin/benchpark setup laghos-mpi-only LLNL-Ruby-icelake-OmniPath workspace/
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/laghos/mpi-only/LLNL-Ruby-icelake-OmniPath/workspace \
+            --workspace-dir workspace/laghos-mpi-only/LLNL-Ruby-icelake-OmniPath/workspace \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run

.gitlab/ci/test.yml

@@ -7,13 +7,19 @@
         ARCHCONFIG: LLNL-Dane-DELL-sapphirerapids-OmniPath
 
 test_run:
+  resource_group: $HOST
   stage: test
   tags:
     - $HOST
-    - shell
+    - batch
   <<: *test_clusters
+  variables:
+    SCHEDULER_PARAMETERS: -N 1 -t 1h
   rules:
-    - changes:
+    - if: |
+        $CI_PIPELINE_SOURCE == "merge_request_event" ||
+        $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      changes:
         - .gitlab-ci.yml
         - .gitlab/ci/*
         - experiments/**
@@ -32,4 +38,5 @@ test_run:
     - cd ./workspace/saxpy/openmp/$ARCHCONFIG/workspace/
     - ramble --workspace-dir . --disable-progress-bar --disable-logger workspace setup
     # Run Saxpy Experiments
-    - ramble --workspace-dir . --disable-progress-bar --disable-logger on
+    - ramble --workspace-dir . --disable-progress-bar --disable-logger
+      on --executor '{execute_experiment}' --where '{n_nodes} == 1'

docs/_static/css/custom.css

@@ -89,3 +89,7 @@ a:visited {
 html.writer-html4 .rst-content dl:not(.docutils) > dt, html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) > dt {
     color: #7fa866
 }
+
+.rst-content div[class^="highlight"] pre {
+    max-height: 600px;
+}

docs/llnl-tutorial.rst

@@ -26,9 +26,9 @@ system specification in Benchpark::
 
     benchpark system init --dest=ruby-system cts cluster=ruby
 
-To run the cuda, strong scaling version of the AMG20223 benchmark, initialize it for experiments::
+To run the openmp, strong scaling version of the AMG20223 benchmark, initialize it for experiments::
 
-    benchpark experiment init --dest=amg2023-benchmark amg2023 cuda=oui strong=oui
+    benchpark experiment init --dest=amg2023-benchmark amg2023 openmp=oui
 
 Then setup the workspace directory for the system and experiment together::
 
@@ -44,22 +44,22 @@ Then setup the Ramble experiment workspace, this builds all software and may tak
     cd ./workspace/amg2023-benchmark/Cts-6d48f81/workspace/
     ramble --workspace-dir . --disable-progress-bar workspace setup
 
-Next, we run the Saxpy experiments, which will launch jobs through the
-scheduler on Tioga::
+Next, we run the AMG2023 experiments, which will launch jobs through the
+scheduler on the CTS system::
 
     ramble --workspace-dir . --disable-progress-bar on
 
 ------
 Tioga
 ------
 
-This second tutorial will guide you through the process of using the cuda 
+This second tutorial will guide you through the process of using the ROCm
 version of the Saxpy benchmark on Tioga. 
 The parameters for initializing the system are slightly different due to the 
 different variants defined for the system. For example, the variant ``~gtl`` turns off gtl-enabled MPI, ``+gtl`` turns it on::
 
     benchpark system init --dest=tioga-system tioga ~gtl
-    benchpark experiment init --dest=saxpy-benchmark saxpy cuda=oui
+    benchpark experiment init --dest=saxpy-benchmark saxpy rocm=oui
     benchpark setup ./saxpy-benchmark ./tioga-system workspace/
     . workspace/setup.sh
     cd ./workspace/saxpy-benchmark/Tioga-975af3c/workspace/

docs/modifiers.rst

@@ -99,7 +99,7 @@ setup step::
 
     benchpark experiment init --dest=</path/to/experiment_root> caliper=<caliper_modifier> </output/path/to/workspace> 
 
-Valid values for ``<caliper>`` are found in the **Caliper Variant**
+Valid values for ``<caliper_variant>`` are found in the **Caliper Variant**
 column of the table below.  Benchpark will link the experiment to Caliper,
 and inject appropriate Caliper configuration at runtime.  After the experiments
 in the workspace have completed running, a ``.cali`` file

experiments/laghos/experiment.py

@@ -0,0 +1,80 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from benchpark.error import BenchparkError
+from benchpark.directives import variant
+from benchpark.experiment import Experiment
+from benchpark.scaling import StrongScaling
+from benchpark.expr.builtin.caliper import Caliper
+
+
+class Laghos(
+    Experiment,
+    StrongScaling,
+    Caliper,
+):
+
+    variant(
+        "workload",
+        default="triplept",
+        description="triplept or other problem",
+    )
+
+    variant(
+        "version",
+        default="develop",
+        description="app version",
+    )
+
+    def compute_applications_section(self):
+        # TODO: Replace with conflicts clause
+        scaling_modes = {
+            "strong": self.spec.satisfies("strong=oui"),
+            "single_node": self.spec.satisfies("single_node=oui"),
+        }
+
+        scaling_mode_enabled = [key for key, value in scaling_modes.items() if value]
+        if len(scaling_mode_enabled) != 1:
+            raise BenchparkError(
+                f"Only one type of scaling per experiment is allowed for application package {self.name}"
+            )
+
+        # Number of initial nodes
+        num_nodes = {"n_nodes": 1}
+
+        if self.spec.satisfies("single_node=oui"):
+            for pk, pv in num_nodes.items():
+                self.add_experiment_variable(pk, pv, True)
+        elif self.spec.satisfies("strong=oui"):
+            scaled_variables = self.generate_strong_scaling_params(
+                {tuple(num_nodes.keys()): list(num_nodes.values())},
+                int(self.spec.variants["scaling-factor"][0]),
+                int(self.spec.variants["scaling-iterations"][0]),
+            )
+            for pk, pv in scaled_variables.items():
+                self.add_experiment_variable(pk, pv, True)
+
+        self.add_experiment_variable(
+            "n_ranks", "{sys_cores_per_node} * {n_nodes}", True
+        )
+
+    def compute_spack_section(self):
+        # get package version
+        app_version = self.spec.variants["version"][0]
+
+        # get system config options
+        # TODO: Get compiler/mpi/package handles directly from system.py
+        system_specs = {}
+        system_specs["compiler"] = "default-compiler"
+        system_specs["mpi"] = "default-mpi"
+
+        # set package spack specs
+        # empty package_specs value implies external package
+        self.add_spack_spec(system_specs["mpi"])
+        # self.add_spack_spec(system_specs["blas"])
+
+        self.add_spack_spec(
+            self.name, [f"laghos@{app_version} +metis", system_specs["compiler"]]
+        )

repo/laghos/application.py

@@ -19,9 +19,29 @@ class Laghos(ExecutableApplication):
             'lagrangian','spatial-discretization','unstructured-grid',
             'network-latency-bound','network-collectives','unstructured-grid']
 
-    executable('p', 'laghos -p 3 -m {laghos}/data/box01_hex.mesh -rs 5 -ms 500', use_mpi=True)
-
-    workload('problem', executables=['p'])
+    executable('prob', 'laghos -p {problem} -m {mesh} -rs {rs} -rp {rp} -ms {ms}', use_mpi=True)
+
+    workload('triplept', executables=['prob'])
+
+    workload_variable('mesh', default='{laghos}/data/box01_hex.mesh',
+            description='mesh file',
+            workloads=['triplept'])
+
+    workload_variable('problem', default='3',
+            description='problem number',
+            workloads=['triplept'])
+
+    workload_variable('rs', default='5',
+            description='number of serial refinements',
+            workloads=['triplept'])
+
+    workload_variable('rp', default='0',
+            description='number of parallel refinements',
+            workloads=['triplept'])
+
+    workload_variable('ms', default='500',
+            description='max number of steps',
+            workloads=['triplept'])
 
     figure_of_merit('Major kernels total time',
                     log_file='{experiment_run_dir}/{experiment_name}.out',

repo/laghos/package.py

@@ -33,7 +33,8 @@ class Laghos(MakefilePackage):
     depends_on("caliper", when="+caliper")
     depends_on("adiak", when="+caliper")
 
-    depends_on("mfem@develop", when="@develop")
+    depends_on("[email protected]+optimize+pic+shared", when="@develop")
+    depends_on("mfem@develop^[email protected]+optimize+pic+shared", when="@develop")
     depends_on("[email protected]:", when="@3.1")
     depends_on("[email protected]:4.1", when="@3.0")
     # Recommended mfem version for laghos v2.0 is: ^[email protected]

systems/cts/externals/base/00-packages.yaml

@@ -47,4 +47,6 @@ packages:
     buildable: false
     externals:
     - spec: [email protected]
-      prefix: /usr/tce/packages/fftw/fftw-3.3.10
+      prefix: /usr/tce/packages/fftw/fftw-3.3.10
+  zlib-api:                                                                     
+    require: zlib

systems/genericx86/externals/base/00-packages.yaml

@@ -0,0 +1,8 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+packages:
+  zlib-api:                                                                     
+    require: zlib

systems/sierra/externals/base/00-packages.yaml

@@ -49,3 +49,5 @@ packages:
       buildable: false
   mpi:
     buildable: false
+  zlib-api:                                                                     
+      require: zlib

systems/tioga/externals/base/00-packages.yaml

@@ -184,3 +184,5 @@ packages:
     buildable: false
   rocsolver:
     buildable: false
+  zlib-api:                                                                     
+    require: zlib