diff --git a/.gitlab/tests/shared_slurm_clusters.yml b/.gitlab/tests/shared_slurm_clusters.yml index cb810c3bc..849eab64e 100644 --- a/.gitlab/tests/shared_slurm_clusters.yml +++ b/.gitlab/tests/shared_slurm_clusters.yml @@ -94,7 +94,7 @@ run_tests_slurm_dane: VARIANT: [+openmp, ''] - HOST: dane ARCHCONFIG: llnl-cluster - BENCHMARK: [laghos, lammps, raja-perf, remhos, phloem] + BENCHMARK: [laghos, lammps, raja-perf, remhos, phloem, mpi-pingpong] # Strong scaling runs for "benchpark analyze" test - HOST: dane ARCHCONFIG: llnl-cluster diff --git a/experiments/mpi-pingpong/experiment.py b/experiments/mpi-pingpong/experiment.py new file mode 100644 index 000000000..bb9c85b9f --- /dev/null +++ b/experiments/mpi-pingpong/experiment.py @@ -0,0 +1,51 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from benchpark.caliper import Caliper +from benchpark.directives import maintainers, variant +from benchpark.experiment import Experiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType + + +class MpiPingpong( + Experiment, + ProgrammingModel( + ProgrammingModelType.Mpionly, + ProgrammingModelType.Rocm, + ProgrammingModelType.Cuda, + ), + Caliper, +): + + variant( + "workload", + default="run", + ) + + maintainers("stephanielam3211") + + def compute_applications_section(self): + n_nodes = 64 # max number of nodes + expr_vars = { + "n_nodes": n_nodes, + "iterations": 10000, + "msg_size": 16, + } + + for pk, pv in expr_vars.items(): + self.add_experiment_variable(pk, pv, True) + + self.add_experiment_variable("n_ranks", "{n_nodes}*{sys_cores_per_node}", True) + + self.add_experiment_variable("partner_rank", "{n_ranks}-1", True) + + self.set_required_variables( + n_resources="{n_ranks}", + process_problem_size="{msg_size}", + total_problem_size="{msg_size}", + ) + + def compute_package_section(self): + self.add_package_spec(self.name, ["mpi-pingpong@develop"]) diff --git a/repos/ramble_applications/mpi_pingpong/application.py b/repos/ramble_applications/mpi_pingpong/application.py new file mode 100644 index 000000000..bb657ac1c --- /dev/null +++ b/repos/ramble_applications/mpi_pingpong/application.py @@ -0,0 +1,22 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +import sys + +from ramble.appkit import * + + +class MpiPingpong(ExecutableApplication): + name = "mpi-pingpong" + + tags = ["single-node", "builtin-caliper"] + + executable( + "pingpong", + "pingpong -i {iterations} -m {msg_size} -p 0,{partner_rank} -n {n_nodes} -s {sys_cores_per_node}//{sys_sockets_per_node} -c {sys_cores_per_node} -b {experiment_run_dir}/{experiment_name}_metadata.json", + use_mpi=True, + ) + + workload("run", executables=["pingpong"]) diff --git a/repos/spack_repo/benchpark/packages/mpi_pingpong/package.py b/repos/spack_repo/benchpark/packages/mpi_pingpong/package.py new file mode 100644 index 000000000..7de04c742 --- /dev/null +++ b/repos/spack_repo/benchpark/packages/mpi_pingpong/package.py @@ -0,0 +1,65 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from spack.package import * + + +class MpiPingpong(CMakePackage, ROCmPackage, CudaPackage): + + git = "https://github.com/LLNL/microbenchmarks.git" + + version("develop", branch="develop") + + variant("caliper", default=False, description="Enable Caliper/Adiak support") + variant("rocm", default=False, description="Enable Rocm support") + variant("cuda", default=False, description="Enable CUDA support") + variant("mpi", default=True, description="Enable MPI support") + + depends_on("mpi", when="+mpi") + depends_on("caliper", when="+caliper") + depends_on("adiak", when="+caliper") + depends_on("hip", when="+rocm") + depends_on("cuda", when="+cuda") + + conflicts("+cuda +rocm", msg="Enable only one of +cuda or +rocm") + + root_cmakelists_dir = "repo/pingpong" + + def cmake_args(self): + if self.spec.satisfies("+caliper"): + args = ["-DUSE_CALIPER=ON"] + else: + args = ["-DUSE_CALIPER=OFF"] + + if self.spec.satisfies("+rocm"): + args.append( + self.define("CMAKE_EXE_LINKER_FLAGS", self.spec["mpi"].libs.ld_flags) + ) + args.append("-DUSE_HIP=ON") + args.append(self.define("ROCM_PATH", self.spec["hip"].prefix)) + hip_vars = self.spec["hip"].variants + if "amdgpu_targets" in hip_vars: + vals = [t for t in hip_vars["amdgpu_targets"].value if t != "none"] + if vals: + archs = ",".join(vals) + args.append(self.define("CMAKE_HIP_ARCHITECTURES", archs)) + else: + args.append("-DUSE_HIP=OFF") + + if self.spec.satisfies("+cuda"): + args.append(self.define("USE_CUDA", "ON")) + v = self.spec.variants.get("cuda_arch", None) + if v: + vals = [a for a in v.value if a != "none"] + if vals: + args.append(self.define("CMAKE_CUDA_ARCHITECTURES", ";".join(vals))) + else: + args.append(self.define("USE_CUDA", "OFF")) + + return args + + def install(self, spec, prefix): + mkdirp(prefix.bin) + install(join_path(self.build_directory, "pingpong"), prefix.bin) diff --git a/systems/llnl-cluster/system.py b/systems/llnl-cluster/system.py index df8fe3c86..2c9419b10 100644 --- a/systems/llnl-cluster/system.py +++ b/systems/llnl-cluster/system.py @@ -51,6 +51,7 @@ class LlnlCluster(System): "cpu_arch": "sapphirerapids", "sys_cores_per_node": 112, "sys_cores_os_reserved_per_node": 0, # No explicit core reservation, first thread on each core reserved (2 threads per core) + "sys_sockets_per_node": 2, "sys_cores_os_reserved_per_node_list": None, "sys_mem_per_node_GB": 256, "sys_cpu_mem_per_node_MB": 210, diff --git a/systems/llnl-elcapitan/system.py b/systems/llnl-elcapitan/system.py index 9547767a6..f3275f6ad 100644 --- a/systems/llnl-elcapitan/system.py +++ b/systems/llnl-elcapitan/system.py @@ -30,6 +30,7 @@ class LlnlElcapitan(System): "sys_cores_per_node": 56, "sys_cores_os_reserved_per_node": 8, "sys_cores_os_reserved_per_node_list": [0, 8, 16, 24, 32, 40, 48, 56], + "sys_sockets_per_node": 2, "sys_gpus_per_node": 8, "sys_mem_per_node_GB": 512, "sys_cpu_mem_per_node_MB": 2048, diff --git a/systems/llnl-sierra/system.py b/systems/llnl-sierra/system.py index 553bded65..a71172303 100644 --- a/systems/llnl-sierra/system.py +++ b/systems/llnl-sierra/system.py @@ -37,6 +37,7 @@ class LlnlSierra(System): 22, 23, ], # First two cores on each socket reserved. + "sys_sockets_per_node": 2, "sys_gpus_per_node": 4, "sys_mem_per_node_GB": 256, "sys_cpu_mem_per_node_MB": 40,