From 6494badf201a239751023b81d530aa8b0136737d Mon Sep 17 00:00:00 2001 From: Amr Akmal Moustafa Abouelmagd Date: Wed, 18 Mar 2026 10:37:03 -0700 Subject: [PATCH 1/2] Added configs to be part of Caliper metadata --- laghos.cpp | 69 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/laghos.cpp b/laghos.cpp index 1023dbb2..89c5641f 100644 --- a/laghos.cpp +++ b/laghos.cpp @@ -139,6 +139,16 @@ int main(int argc, char *argv[]) bool enable_nc = true; + #ifdef LAGHOS_USE_CALIPER + cali_config_set("CALI_CALIPER_ATTRIBUTE_DEFAULT_SCOPE", "process"); + CALI_CXX_MARK_FUNCTION; + + MPI_Comm adiak_mpi_comm = MPI_COMM_WORLD; + void* adiak_mpi_comm_ptr = &adiak_mpi_comm; + adiak::init(adiak_mpi_comm_ptr); + adiak::collect_all(); + #endif + OptionsParser args(argc, argv); args.AddOption(&dim, "-dim", "--dimension", "Dimension of the problem."); args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use."); @@ -237,7 +247,54 @@ int main(int argc, char *argv[]) if (Mpi::Root()) { args.PrintUsage(cout); } return 1; } - if (Mpi::Root()) { args.PrintOptions(cout); } + + if (Mpi::Root()) + { + args.PrintOptions(cout); + + #ifdef LAGHOS_USE_CALIPER + adiak::value("dimension", dim); + adiak::value("mesh", std::string(mesh_file)); + adiak::value("elem-per-mpi", elem_per_mpi); + adiak::value("xelems", nx); + adiak::value("yelems", ny); + adiak::value("zelems", nz); + adiak::value("blast-energy", blast_energy); + adiak::value("xwidth", (double)Sx); + adiak::value("ywidth", (double)Sy); + adiak::value("zwidth", (double)Sz); + adiak::value("refine-serial", rs_levels); + adiak::value("refine-parallel", rp_levels); + adiak::value("problem", problem); + adiak::value("order-kinematic", order_v); + adiak::value("order-thermo", order_e); + adiak::value("order-intrule", order_q); + adiak::value("ode-solver", ode_solver_type); + adiak::value("t-final", t_final); + adiak::value("cfl", cfl); + adiak::value("cg-tol", cg_tol); + adiak::value("ftz-tol", ftz_tol); + adiak::value("delta-tol", delta_tol); + adiak::value("cg-max-steps", cg_max_iter); + adiak::value("max-steps", max_tsteps); + adiak::value("partial-assembly", (int)p_assembly); + adiak::value("impose-viscosity", (int)impose_visc); + adiak::value("visualization", (int)visualization); + adiak::value("visualization-steps", vis_steps); + adiak::value("visit", (int)visit); + adiak::value("print", (int)gfprint); + adiak::value("outputfilename", std::string(basename)); + adiak::value("device", std::string(device)); + adiak::value("checks", (int)check); + adiak::value("exact-error", (int)check_exact_sedov); + adiak::value("mem", (int)mem_usage); + adiak::value("fom", (int)fom); + adiak::value("gpu-aware-mpi", (int)gpu_aware_mpi); + adiak::value("dev-pool-size", dev_pool_size); + adiak::value("conforming", (int)enable_nc); + adiak::value("dev", dev); + #endif + } if (check_exact_sedov) { @@ -247,16 +304,6 @@ int main(int argc, char *argv[]) MFEM_VERIFY(strncmp(mesh_file, "default", 7) == 0, "check: mesh_file"); } -#ifdef LAGHOS_USE_CALIPER - cali_config_set("CALI_CALIPER_ATTRIBUTE_DEFAULT_SCOPE", "process"); - CALI_CXX_MARK_FUNCTION; - - MPI_Comm adiak_mpi_comm = MPI_COMM_WORLD; - void* adiak_mpi_comm_ptr = &adiak_mpi_comm; - adiak::init(adiak_mpi_comm_ptr); - adiak::collect_all(); -#endif - #ifdef LAGHOS_USE_DEVICE_UMPIRE auto &rm = umpire::ResourceManager::getInstance(); const char * allocator_name = "laghos_device_alloc"; From ffa540171822f6c93be65736b7b8d58fc058b6df Mon Sep 17 00:00:00 2001 From: Amr Akmal Moustafa Abouelmagd Date: Mon, 30 Mar 2026 23:04:43 -0700 Subject: [PATCH 2/2] Addressed code review comments --- laghos.cpp | 126 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 86 insertions(+), 40 deletions(-) diff --git a/laghos.cpp b/laghos.cpp index 89c5641f..dee6d123 100644 --- a/laghos.cpp +++ b/laghos.cpp @@ -87,6 +87,24 @@ static long GetMaxRssMB(); static void display_banner(std::ostream&); static void Checks(const int ti, const double norm, int &checks); +#ifdef LAGHOS_USE_CALIPER + static void RecordAdiakMetadata(int dim, const char *mesh_file, int elem_per_mpi, + int nx, int ny, int nz, double blast_energy, + double Sx, double Sy, double Sz, + int rs_levels, int rp_levels, int problem, + int order_v, int order_e, int order_q, + int ode_solver_type, double t_final, double cfl, + double cg_tol, double ftz_tol, double delta_tol, + int cg_max_iter, int max_tsteps, + bool p_assembly, bool impose_visc, + bool visualization, int vis_steps, bool visit, + bool gfprint, const char *basename, + const char *device, bool check, + bool check_exact_sedov, bool mem_usage, + bool fom, bool gpu_aware_mpi, int dev_pool_size, + bool enable_nc, int dev); +#endif + int main(int argc, char *argv[]) { // Initialize MPI. @@ -253,46 +271,14 @@ int main(int argc, char *argv[]) args.PrintOptions(cout); #ifdef LAGHOS_USE_CALIPER - adiak::value("dimension", dim); - adiak::value("mesh", std::string(mesh_file)); - adiak::value("elem-per-mpi", elem_per_mpi); - adiak::value("xelems", nx); - adiak::value("yelems", ny); - adiak::value("zelems", nz); - adiak::value("blast-energy", blast_energy); - adiak::value("xwidth", (double)Sx); - adiak::value("ywidth", (double)Sy); - adiak::value("zwidth", (double)Sz); - adiak::value("refine-serial", rs_levels); - adiak::value("refine-parallel", rp_levels); - adiak::value("problem", problem); - adiak::value("order-kinematic", order_v); - adiak::value("order-thermo", order_e); - adiak::value("order-intrule", order_q); - adiak::value("ode-solver", ode_solver_type); - adiak::value("t-final", t_final); - adiak::value("cfl", cfl); - adiak::value("cg-tol", cg_tol); - adiak::value("ftz-tol", ftz_tol); - adiak::value("delta-tol", delta_tol); - adiak::value("cg-max-steps", cg_max_iter); - adiak::value("max-steps", max_tsteps); - adiak::value("partial-assembly", (int)p_assembly); - adiak::value("impose-viscosity", (int)impose_visc); - adiak::value("visualization", (int)visualization); - adiak::value("visualization-steps", vis_steps); - adiak::value("visit", (int)visit); - adiak::value("print", (int)gfprint); - adiak::value("outputfilename", std::string(basename)); - adiak::value("device", std::string(device)); - adiak::value("checks", (int)check); - adiak::value("exact-error", (int)check_exact_sedov); - adiak::value("mem", (int)mem_usage); - adiak::value("fom", (int)fom); - adiak::value("gpu-aware-mpi", (int)gpu_aware_mpi); - adiak::value("dev-pool-size", dev_pool_size); - adiak::value("conforming", (int)enable_nc); - adiak::value("dev", dev); + RecordAdiakMetadata(dim, mesh_file, elem_per_mpi, nx, ny, nz, + blast_energy, Sx, Sy, Sz, rs_levels, rp_levels, + problem, order_v, order_e, order_q, ode_solver_type, + t_final, cfl, cg_tol, ftz_tol, delta_tol, + cg_max_iter, max_tsteps, p_assembly, impose_visc, + visualization, vis_steps, visit, gfprint, basename, + device, check, check_exact_sedov, mem_usage, fom, + gpu_aware_mpi, dev_pool_size, enable_nc, dev); #endif } @@ -1262,6 +1248,66 @@ static void display_banner(std::ostream &os) << " /____/ " << endl << endl; } +#ifdef LAGHOS_USE_CALIPER +static void RecordAdiakMetadata(int dim, const char *mesh_file, int elem_per_mpi, + int nx, int ny, int nz, double blast_energy, + double Sx, double Sy, double Sz, + int rs_levels, int rp_levels, int problem, + int order_v, int order_e, int order_q, + int ode_solver_type, double t_final, double cfl, + double cg_tol, double ftz_tol, double delta_tol, + int cg_max_iter, int max_tsteps, + bool p_assembly, bool impose_visc, + bool visualization, int vis_steps, bool visit, + bool gfprint, const char *basename, + const char *device, bool check, + bool check_exact_sedov, bool mem_usage, + bool fom, bool gpu_aware_mpi, int dev_pool_size, + bool enable_nc, int dev) +{ + adiak::value("dimension", dim); + adiak::value("mesh", std::string(mesh_file)); + adiak::value("elem-per-mpi", elem_per_mpi); + adiak::value("xelems", nx); + adiak::value("yelems", ny); + adiak::value("zelems", nz); + adiak::value("blast-energy", blast_energy); + adiak::value("xwidth", (double)Sx); + adiak::value("ywidth", (double)Sy); + adiak::value("zwidth", (double)Sz); + adiak::value("refine-serial", rs_levels); + adiak::value("refine-parallel", rp_levels); + adiak::value("problem", problem); + adiak::value("order-kinematic", order_v); + adiak::value("order-thermo", order_e); + adiak::value("order-intrule", order_q); + adiak::value("ode-solver", ode_solver_type); + adiak::value("t-final", t_final); + adiak::value("cfl", cfl); + adiak::value("cg-tol", cg_tol); + adiak::value("ftz-tol", ftz_tol); + adiak::value("delta-tol", delta_tol); + adiak::value("cg-max-steps", cg_max_iter); + adiak::value("max-steps", max_tsteps); + adiak::value("partial-assembly", (int)p_assembly); + adiak::value("impose-viscosity", (int)impose_visc); + adiak::value("visualization", (int)visualization); + adiak::value("visualization-steps", vis_steps); + adiak::value("visit", (int)visit); + adiak::value("print", (int)gfprint); + adiak::value("outputfilename", std::string(basename)); + adiak::value("device", std::string(device)); + adiak::value("checks", (int)check); + adiak::value("exact-error", (int)check_exact_sedov); + adiak::value("mem", (int)mem_usage); + adiak::value("fom", (int)fom); + adiak::value("gpu-aware-mpi", (int)gpu_aware_mpi); + adiak::value("dev-pool-size", dev_pool_size); + adiak::value("conforming", (int)enable_nc); + adiak::value("dev", dev); +} +#endif + static long GetMaxRssMB() { struct rusage usage;