From 6dfa885dc55f8406a63dda01ec05eeed11a551e4 Mon Sep 17 00:00:00 2001 From: Grace Nansamba Date: Sun, 12 Jan 2025 16:27:50 -0800 Subject: [PATCH 01/16] added array of numProc and problemSize --- experiments/amg2023/experiment.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/experiments/amg2023/experiment.py b/experiments/amg2023/experiment.py index 5866eea6e..577a141d6 100644 --- a/experiments/amg2023/experiment.py +++ b/experiments/amg2023/experiment.py @@ -68,11 +68,21 @@ def compute_applications_section(self): ) # Number of processes in each dimension - num_procs = {"px": 2, "py": 2, "pz": 2} + # num_procs = {"px": 2, "py": 2, "pz": 2} + num_procs = { + "px": ['4', '8', '8', '8'], + "py": ['4', '4', '8', '8'], + "pz": ['4', '4', '4', '8'] + } # Per-process size (in zones) in each dimension - problem_sizes = {"nx": 80, "ny": 80, "nz": 80} - + # problem_sizes = {"nx": 80, "ny": 80, "nz": 80} + problem_sizes = { + "nx": ['32', '16', '32', '32'], + "ny": ['32', '32', '16', '32'], + "nz": ['16', '32', '32', '16'] + } + if self.spec.satisfies("+single_node"): n_resources = 1 # TODO: Check if n_ranks / n_resources_per_node <= 1 From 8921e3180a275b17fd5b414b860b10f40fcf6977 Mon Sep 17 00:00:00 2001 From: Grace Nansamba Date: Thu, 16 Jan 2025 09:44:20 -0800 Subject: [PATCH 02/16] made an array with ints --- experiments/amg2023/experiment.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/experiments/amg2023/experiment.py b/experiments/amg2023/experiment.py index 577a141d6..e510d824e 100644 --- a/experiments/amg2023/experiment.py +++ b/experiments/amg2023/experiment.py @@ -69,19 +69,19 @@ def compute_applications_section(self): # Number of processes in each dimension # num_procs = {"px": 2, "py": 2, "pz": 2} - num_procs = { - "px": ['4', '8', '8', '8'], - "py": ['4', '4', '8', '8'], - "pz": ['4', '4', '4', '8'] - } + num_procs = { + "px": [4, 8, 8, 8], + "py": [4, 4, 8, 8], + "pz": [4, 4, 4, 8] + } # Per-process size (in zones) in each dimension # problem_sizes = {"nx": 80, "ny": 80, "nz": 80} - problem_sizes = { - "nx": ['32', '16', '32', '32'], - "ny": ['32', '32', '16', '32'], - "nz": ['16', '32', '32', '16'] - } + problem_sizes = { + "nx": [32, 16, 32, 32], + "ny": [32, 32, 16, 32], + "nz": [16, 32, 32, 16] + } if self.spec.satisfies("+single_node"): n_resources = 1 From 85d8d8b79d5163e94a65ad1aecea4705dbaeceeb Mon Sep 17 00:00:00 2001 From: Grace Nansamba Date: Fri, 17 Jan 2025 16:14:37 -0800 Subject: [PATCH 03/16] added strong and weak scaling options --- experiments/kripke/experiment.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/experiments/kripke/experiment.py b/experiments/kripke/experiment.py index 7fff0420e..70c610d8b 100644 --- a/experiments/kripke/experiment.py +++ b/experiments/kripke/experiment.py @@ -61,10 +61,10 @@ def compute_applications_section(self): } # Number of processes in each dimension - num_procs = {"npx": 2, "npy": 2, "npz": 1} + num_procs = {"npx": 4, "npy": 4, "npz": 4} # Number of zones in each dimension, per process - problem_sizes = {"nzx": 64, "nzy": 64, "nzz": 32} + problem_sizes = {"nzx": 64, "nzy": 128, "nzz": 128} for k, v in input_variables.items(): self.add_experiment_variable(k, v, True) @@ -90,6 +90,12 @@ def compute_applications_section(self): for nk, nv in scaled_variables.items(): self.add_experiment_variable(nk, nv, True) elif self.spec.satisfies("+strong"): + # Number of processes in each dimension + num_procs = {"npx": 2, "npy": 2, "npz": 1} + + # Number of zones in each dimension, per process + problem_sizes = {"nzx": 64, "nzy": 64, "nzz": 32} + scaled_variables = self.generate_strong_scaling_params( {tuple(num_procs.keys()): list(num_procs.values())}, int(self.spec.variants["scaling-factor"][0]), From 2f73c15f20b4f892dbcc826076403d8f764d0c58 Mon Sep 17 00:00:00 2001 From: Grace Nansamba Date: Mon, 3 Feb 2025 20:35:15 -0800 Subject: [PATCH 04/16] added caliper new stats --- modifiers/caliper/modifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modifiers/caliper/modifier.py b/modifiers/caliper/modifier.py index 9eaa29103..a79b88adc 100644 --- a/modifiers/caliper/modifier.py +++ b/modifiers/caliper/modifier.py @@ -42,7 +42,7 @@ class Caliper(SpackModifier): env_var_modification( "CALI_CONFIG", - "spot(output={}{})".format(_cali_datafile, "${CALI_CONFIG_MODE}"), + "spot(output={},profile.mpi,comm.stats)".format(_cali_datafile, "${CALI_CONFIG_MODE}"), method="set", modes=[_default_mode], ) From 7a58fb936c90ab175753be3c17d7110237604c1f Mon Sep 17 00:00:00 2001 From: Grace Nansamba Date: Tue, 4 Feb 2025 10:36:25 -0800 Subject: [PATCH 05/16] added comm stats --- modifiers/caliper-mpi/modifier.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modifiers/caliper-mpi/modifier.py b/modifiers/caliper-mpi/modifier.py index f742cd874..e634e48c5 100644 --- a/modifiers/caliper-mpi/modifier.py +++ b/modifiers/caliper-mpi/modifier.py @@ -21,9 +21,10 @@ class CaliperMpi(CaliperBase): env_var_modification( "CALI_CONFIG", - "spot(output={},profile.mpi,mpi.message.size,mpi.message.count)".format( - _cali_datafile - ), + #"spot(output={},profile.mpi,mpi.message.size,mpi.message.count)".format( + # _cali_datafile + #), + "spot(output={},profile.mpi,comm.stats)".format(_cali_datafile), method="set", modes=["mpi"], ) From 0bc5a28fdf2ff280bcd8e478a4b970fbc9ed800d Mon Sep 17 00:00:00 2001 From: Grace Nansamba Date: Tue, 4 Feb 2025 12:24:54 -0800 Subject: [PATCH 06/16] changed the path to point to annotated github --- repo/kripke/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repo/kripke/package.py b/repo/kripke/package.py index 03e6ac2a1..584c6301e 100644 --- a/repo/kripke/package.py +++ b/repo/kripke/package.py @@ -11,8 +11,8 @@ class Kripke(CMakePackage, CudaPackage, ROCmPackage): transport proxy/mini app. """ - homepage = "https://computing.llnl.gov/projects/co-design/kripke" - git = "https://github.com/LLNL/Kripke.git" + homepage = "https://github.com/gracenansamba/Kripke.git" + git = "https://github.com/gracenansamba/Kripke.git" tags = ["proxy-app"] From 603d0275e06c6b588d3a559315173bf63a73b9bd Mon Sep 17 00:00:00 2001 From: Grace Nansamba Date: Tue, 4 Feb 2025 22:14:36 -0800 Subject: [PATCH 07/16] addedcaliper@master --- repo/kripke/package.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/repo/kripke/package.py b/repo/kripke/package.py index 584c6301e..a49b0c12b 100644 --- a/repo/kripke/package.py +++ b/repo/kripke/package.py @@ -20,7 +20,7 @@ class Kripke(CMakePackage, CudaPackage, ROCmPackage): license("BSD-3-Clause") - version("develop", branch="develop", submodules=False) + version("comm_cali", branch="comm_cali", submodules=False, git="https://github.com/gracenansamba/Kripke.git") version( "1.2.7.0", submodules=False, commit="db920c1f5e1dcbb9e949d120e7d86efcdb777635" ) @@ -52,12 +52,10 @@ class Kripke(CMakePackage, CudaPackage, ROCmPackage): variant("mpi", default=True, description="Build with MPI.") variant("openmp", default=False, description="Build with OpenMP enabled.") variant("caliper", default=False, description="Build with Caliper support enabled.") - + depends_on('chai@2024.02', when='@develop') - - depends_on("mpi", when="+mpi") + depends_on("caliper@master", when="+caliper") depends_on("chai+mpi", when="+mpi") - depends_on("caliper", when="+caliper") depends_on("adiak@0.4:", when="+caliper") conflicts("^blt@:0.3.6", when="+rocm") From a5fb5510d53bbf868af74565aa2e2f28b6cc0d8c Mon Sep 17 00:00:00 2001 From: Grace Nansamba Date: Wed, 5 Feb 2025 20:47:54 -0800 Subject: [PATCH 08/16] added caliper variant --- experiments/kripke/experiment.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/experiments/kripke/experiment.py b/experiments/kripke/experiment.py index 70c610d8b..0dad3825a 100644 --- a/experiments/kripke/experiment.py +++ b/experiments/kripke/experiment.py @@ -33,10 +33,27 @@ class Kripke( variant( "version", - default="develop", + default="comm_cali", description="app version", ) + variant( + "caliper", + default="none", + values=( + "none", + "time", + "mpi", + "cuda", + "topdown-counters-all", + "topdown-counters-toplevel", + "topdown-all", + "topdown-toplevel", + ), + multi=True, + description="caliper mode", + ) + def compute_applications_section(self): # TODO: Replace with conflicts clause scaling_modes = { From 329c3c7948f33eefdddcf9805da3cba1db271f16 Mon Sep 17 00:00:00 2001 From: Grace Nansamba Date: Thu, 6 Feb 2025 20:48:41 -0800 Subject: [PATCH 09/16] adjusted problem size --- experiments/kripke/experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiments/kripke/experiment.py b/experiments/kripke/experiment.py index 0dad3825a..e0bddc5c0 100644 --- a/experiments/kripke/experiment.py +++ b/experiments/kripke/experiment.py @@ -81,7 +81,7 @@ def compute_applications_section(self): num_procs = {"npx": 4, "npy": 4, "npz": 4} # Number of zones in each dimension, per process - problem_sizes = {"nzx": 64, "nzy": 128, "nzz": 128} + problem_sizes = {"nzx": 256, "nzy": 128, "nzz": 128} for k, v in input_variables.items(): self.add_experiment_variable(k, v, True) From ba559ccbc45f5a7403e7120a45c9b4073ec8c05b Mon Sep 17 00:00:00 2001 From: Grace Nansamba Date: Tue, 18 Feb 2025 17:14:29 -0800 Subject: [PATCH 10/16] added caliper version variant --- experiments/kripke/experiment.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/experiments/kripke/experiment.py b/experiments/kripke/experiment.py index da44b84d8..c1c53319d 100644 --- a/experiments/kripke/experiment.py +++ b/experiments/kripke/experiment.py @@ -36,6 +36,23 @@ class Kripke( default="develop", description="app version", ) + + variant( + "caliper", + default="none", + values=( + "none", + "time", + "mpi", + "cuda", + "topdown-counters-all", + "topdown-counters-toplevel", + "topdown-all", + "topdown-toplevel", + ), + multi=True, + description="caliper mode", + ) def compute_applications_section(self): # TODO: Replace with conflicts clause @@ -61,10 +78,10 @@ def compute_applications_section(self): } # Number of processes in each dimension - num_procs = {"npx": 2, "npy": 2, "npz": 1} + num_procs = {"npx": 2, "npy": 2, "npz": 2} # Number of zones in each dimension, per process - problem_sizes = {"nzx": 64, "nzy": 64, "nzz": 32} + problem_sizes = {"nzx": 128, "nzy": 128, "nzz": 128} for k, v in input_variables.items(): self.add_experiment_variable(k, v, True) @@ -90,6 +107,12 @@ def compute_applications_section(self): for nk, nv in scaled_variables.items(): self.add_experiment_variable(nk, nv, True) elif self.spec.satisfies("+strong"): + # Number of processes in each dimension + num_procs = {"npx": 2, "npy": 2, "npz": 1} + + # Number of zones in each dimension, per process + problem_sizes = {"nzx": 64, "nzy": 64, "nzz": 32} + scaled_variables = self.generate_strong_scaling_params( {tuple(num_procs.keys()): list(num_procs.values())}, int(self.spec.variants["scaling-factor"][0]), From 31e987b6b72e3969223da5ebf31a506f62dec99e Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Wed, 19 Feb 2025 11:29:08 -0800 Subject: [PATCH 11/16] Experiment params --- experiments/kripke/experiment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/experiments/kripke/experiment.py b/experiments/kripke/experiment.py index e0bddc5c0..45ad7f24c 100644 --- a/experiments/kripke/experiment.py +++ b/experiments/kripke/experiment.py @@ -78,10 +78,10 @@ def compute_applications_section(self): } # Number of processes in each dimension - num_procs = {"npx": 4, "npy": 4, "npz": 4} + num_procs = {"npx": 2, "npy": 2, "npz": 2} # Number of zones in each dimension, per process - problem_sizes = {"nzx": 256, "nzy": 128, "nzz": 128} + problem_sizes = {"nzx": 128, "nzy": 128, "nzz": 128} for k, v in input_variables.items(): self.add_experiment_variable(k, v, True) From e9fa9443be533d6cad2224e5e6b8da37a037c8fa Mon Sep 17 00:00:00 2001 From: Grace Nansamba Date: Wed, 19 Feb 2025 13:57:20 -0800 Subject: [PATCH 12/16] removed repeated code --- experiments/kripke/experiment.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/experiments/kripke/experiment.py b/experiments/kripke/experiment.py index b7a583ac1..c1c53319d 100644 --- a/experiments/kripke/experiment.py +++ b/experiments/kripke/experiment.py @@ -33,7 +33,7 @@ class Kripke( variant( "version", - default="comm_cali", + default="develop", description="app version", ) @@ -54,23 +54,6 @@ class Kripke( description="caliper mode", ) - variant( - "caliper", - default="none", - values=( - "none", - "time", - "mpi", - "cuda", - "topdown-counters-all", - "topdown-counters-toplevel", - "topdown-all", - "topdown-toplevel", - ), - multi=True, - description="caliper mode", - ) - def compute_applications_section(self): # TODO: Replace with conflicts clause scaling_modes = { From 0bcbf2eb6dc9a8ae0d4f7224dea28313f87dd4d7 Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Thu, 20 Feb 2025 11:32:17 -0800 Subject: [PATCH 13/16] Fix build --- experiments/kripke/experiment.py | 2 +- repo/kripke/package.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/experiments/kripke/experiment.py b/experiments/kripke/experiment.py index c1c53319d..cadd413b6 100644 --- a/experiments/kripke/experiment.py +++ b/experiments/kripke/experiment.py @@ -33,7 +33,7 @@ class Kripke( variant( "version", - default="develop", + default="comm_cali", description="app version", ) diff --git a/repo/kripke/package.py b/repo/kripke/package.py index a49b0c12b..680dc8421 100644 --- a/repo/kripke/package.py +++ b/repo/kripke/package.py @@ -54,6 +54,7 @@ class Kripke(CMakePackage, CudaPackage, ROCmPackage): variant("caliper", default=False, description="Build with Caliper support enabled.") depends_on('chai@2024.02', when='@develop') + depends_on('chai@2024.02', when='@comm_cali') depends_on("caliper@master", when="+caliper") depends_on("chai+mpi", when="+mpi") depends_on("adiak@0.4:", when="+caliper") From 01a751f86fc518cb7995b098f044cf8519ac8d79 Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Thu, 20 Feb 2025 11:56:17 -0800 Subject: [PATCH 14/16] MPI gtl flags --- repo/kripke/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/repo/kripke/package.py b/repo/kripke/package.py index 680dc8421..1a08a171c 100644 --- a/repo/kripke/package.py +++ b/repo/kripke/package.py @@ -106,6 +106,7 @@ def cmake_args(self): "-DRAJA_DIR=%s" % self.spec["raja"].prefix, "-Dchai_DIR=%s" % self.spec["chai"].prefix, "-DENABLE_CHAI=ON", + "-DMPI_CXX_LINK_FLAGS='%s'" % self.spec['mpi'].libs.ld_flags, ] ) From a70fff19edf1faff1421015056ba672245060657 Mon Sep 17 00:00:00 2001 From: Grace Nansamba Date: Wed, 26 Feb 2025 20:05:15 -0800 Subject: [PATCH 15/16] made problem sizes smaller --- experiments/kripke/experiment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/experiments/kripke/experiment.py b/experiments/kripke/experiment.py index cadd413b6..d68b0bf6a 100644 --- a/experiments/kripke/experiment.py +++ b/experiments/kripke/experiment.py @@ -78,10 +78,10 @@ def compute_applications_section(self): } # Number of processes in each dimension - num_procs = {"npx": 2, "npy": 2, "npz": 2} + num_procs = {"npx": 4, "npy": 4, "npz": 4} # Number of zones in each dimension, per process - problem_sizes = {"nzx": 128, "nzy": 128, "nzz": 128} + problem_sizes = {"nzx": 16, "nzy": 32, "nzz": 32} for k, v in input_variables.items(): self.add_experiment_variable(k, v, True) From ddf84a4c232dc7c78b430445d161abb0ec930a3b Mon Sep 17 00:00:00 2001 From: Grace Nansamba Date: Wed, 5 Mar 2025 11:08:16 -0800 Subject: [PATCH 16/16] adjusted sizes --- experiments/kripke/experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiments/kripke/experiment.py b/experiments/kripke/experiment.py index d68b0bf6a..e24151795 100644 --- a/experiments/kripke/experiment.py +++ b/experiments/kripke/experiment.py @@ -78,7 +78,7 @@ def compute_applications_section(self): } # Number of processes in each dimension - num_procs = {"npx": 4, "npy": 4, "npz": 4} + num_procs = {"npx": 2, "npy": 2, "npz": 2} # Number of zones in each dimension, per process problem_sizes = {"nzx": 16, "nzy": 32, "nzz": 32}