diff --git a/mini-apps/balsaraFsgrid/Makefile b/mini-apps/balsaraFsgrid/Makefile index d467a80c47..66dacc93a0 100644 --- a/mini-apps/balsaraFsgrid/Makefile +++ b/mini-apps/balsaraFsgrid/Makefile @@ -2,6 +2,7 @@ ARCH = ${VLASIATOR_ARCH} include ../../MAKE/Makefile.${ARCH} +LIBRARY_PREFIX=$(CURDIR)/../../libraries-${ARCH} MATHFLAGS = -DDP -DDPF -D${VECTORCLASS} -DACC_SEMILAG_PQM -DTRANS_SEMILAG_PPM LIBS = ${LIB_BOOST} LIBS += ${LIB_JEMALLOC} @@ -9,24 +10,67 @@ LIBS += ${LIB_PROFILE} LIBS += ${LIB_ZOLTAN} LIBS += ${LIB_VLSV} +INC_EIGEN = -isystem ../../submodules/eigen/ INC_FSGRID=-I../../submodules/fsgrid/ INC_DCCRG=-I../../submodules/dccrg/ +CXXFLAGS := $(filter-out -DASTERIX_MLP,$(CXXFLAGS)) +CXXFLAGS := $(filter-out -DASTERIX_OCTREE,$(CXXFLAGS)) +CXXFLAGS := $(filter-out -DASTERIX_ZFP,$(CXXFLAGS)) +LIBS := $(filter-out LIB_NN_COMPRESSOR,$(LIBS)) +LIBS := $(filter-out LIB_ZFP,$(LIBS)) +LIBS := $(filter-out LIB_OCTREE_COMPRESSOR,$(LIBS)) + all: main .PHONY: clean clean: rm *.o main -ionosphere.o: ../../sysboundary/ionosphere.h ../../sysboundary/ionosphere.cpp ../../backgroundfield/backgroundfield.h ../../projects/project.h - ${CMP} ${CXXFLAGS} ${FLAGS} ${MATHFLAGS} -c ../../sysboundary/ionosphere.cpp ${INC_DCCRG} ${INC_FSGRID} ${INC_ZOLTAN} ${INC_BOOST} ${INC_EIGEN} ${INC_VECTORCLASS} -Wno-comment + + +ionosphere.o: ../../sysboundary/ionosphere.h ../../sysboundary/ionosphere.cpp ../../backgroundfield/backgroundfield.h ../../projects/project.h ../../logger.h + ${CMP} ${CXXFLAGS} ${FLAGS} ${MATHFLAGS} -c ../../sysboundary/ionosphere.cpp ${INC_DCCRG} ${INC_FSGRID} ${INC_ZOLTAN} ${INC_BOOST} ${INC_EIGEN} ${INC_VECTORCLASS} ${INC_PROFILE} ${INC_JEMALLOC} -Wno-comment + +sysboundarycondition.o: ../../sysboundary/sysboundarycondition.h ../../sysboundary/sysboundarycondition.cpp ../../sysboundary/donotcompute.h ../../sysboundary/donotcompute.cpp ../../sysboundary/ionosphere.h ../../sysboundary/ionosphere.cpp ../../sysboundary/copysphere.h ../../sysboundary/copysphere.cpp ../../sysboundary/outflow.h ../../sysboundary/outflow.cpp ../../sysboundary/setmaxwellian.h ../../sysboundary/setmaxwellian.cpp + ${CMP} ${CXXFLAGS} ${FLAGS} ${MATHFLAGS} -c ../../sysboundary/sysboundarycondition.cpp ${INC_DCCRG} ${INC_FSGRID} ${INC_ZOLTAN} ${INC_BOOST} ${INC_EIGEN} ${INC_PROFILE} ${INC_JEMALLOC} + +parameters.o: ../../parameters.h ../../parameters.cpp ../../readparameters.h ../../logger.h + $(CMP) $(CXXFLAGS) $(FLAGS) ${MATHFLAGS} -c ../../parameters.cpp ${INC_BOOST} ${INC_EIGEN} ${INC_DCCRG} ${INC_ZOLTAN} ${INC_FSGRID} ${INC_PROFILE} ${INC_JEMALLOC} + +readparameters.o: ../../readparameters.h ../../readparameters.cpp ../../version.h ../../version.cpp + $(CMP) $(CXXFLAGS) $(FLAGS) -c ../../readparameters.cpp ${INC_BOOST} ${INC_EIGEN} + +version.o: ../../version.cpp + ${CMP} ${CXXFLAGS} ${FLAGS} -c ../../version.cpp + +../../version.cpp: + make -C../.. version.cpp + +velocity_mesh_parameters.o: ../../velocity_mesh_parameters.h ../../velocity_mesh_parameters.cpp + ${CMP} ${CXXFLAGS} ${FLAGS} -c ../../velocity_mesh_parameters.cpp ${INC_DCCRG} ${INC_ZOLTAN} ${INC_BOOST} ${INC_FSGRID} ${INC_PROFILE} ${INC_JEMALLOC} + +spatial_cell.o: ../../spatial_cells/spatial_cell_cpu.cpp + $(CMP) $(CXXFLAGS) ${MATHFLAGS} $(FLAGS) -c ../../spatial_cells/spatial_cell_cpu.cpp -o spatial_cell.o $(INC_BOOST) ${INC_DCCRG} ${INC_EIGEN} ${INC_ZOLTAN} ${INC_VECTORCLASS} ${INC_FSGRID} ${INC_PROFILE} ${INC_JEMALLOC} + +arch_moments.o: ../../vlasovsolver/arch_moments.cpp + ${CMP} ${CXXFLAGS} ${FLAG_OPENMP} ${MATHFLAGS} ${FLAGS} -c ../../vlasovsolver/arch_moments.cpp ${INC_DCCRG} ${INC_BOOST} ${INC_ZOLTAN} ${INC_PROFILE} ${INC_FSGRID} ${INC_JEMALLOC} + +logger.o: ../../logger.h ../../logger.cpp + ${CMP} ${CXXFLAGS} ${FLAGS} -c ../../logger.cpp ${INC_MPI} + +common.o: ../../common.h ../../common.cpp + $(CMP) $(CXXFLAGS) $(FLAGS) -c ../../common.cpp fs_common.o: ../../fieldsolver/fs_limiters.h ../../fieldsolver/fs_limiters.cpp - ${CMP} ${CXXFLAGS} ${FLAGS} ${MATHFLAGS} -c ../../fieldsolver/fs_common.cpp -I$(CURDIR) ${INC_BOOST} ${INC_EIGEN} ${INC_DCCRG} ${INC_FSGRID} ${INC_PROFILE} ${INC_ZOLTAN} + ${CMP} ${CXXFLAGS} ${FLAGS} ${MATHFLAGS} -c ../../fieldsolver/fs_common.cpp -I$(CURDIR) ${INC_BOOST} ${INC_EIGEN} ${INC_DCCRG} ${INC_FSGRID} ${INC_PROFILE} ${INC_ZOLTAN} ${INC_JEMALLOC} + +fieldtracing.o: ../../fieldtracing/fieldtracing.h ../../fieldtracing/fieldtracing.cpp ../../logger.h + ${CMP} ${CXXFLAGS} ${FLAGS} ${MATHFLAGS} -c ../../fieldtracing/fieldtracing.cpp ${INC_DCCRG} ${INC_FSGRID} ${INC_BOOST} ${INC_ZOLTAN} ${INC_EIGEN} ${INC_PROFILE} ${INC_JEMALLOC} main.o: main.cpp - ${CMP} ${CXXFLAGS} ${FLAGS} ${MATHFLAGS} -c ./main.cpp ${INC_VLSV} + ${CMP} ${CXXFLAGS} ${FLAGS} ${MATHFLAGS} -c ./main.cpp ${INC_VLSV} ${INC_BOOST} ${INC_PROFILE} ${INC_JEMALLOC} ${INC_DCCRG} ${INC_ZOLTAN} ${INC_FSGRID} ${INC_EIGEN} -main: main.o fs_common.o - ${LNK} ${LDFLAGS} -o main $^ $(LIBS) -lgomp +main: main.o logger.o common.o ionosphere.o parameters.o sysboundarycondition.o readparameters.o version.o spatial_cell.o arch_moments.o fs_common.o fieldtracing.o velocity_mesh_parameters.o + ${LNK} ${LDFLAGS} -o main $^ $(LIBS) -lgomp diff --git a/mini-apps/balsaraFsgrid/main.cpp b/mini-apps/balsaraFsgrid/main.cpp index c3257f9215..078f2dba38 100644 --- a/mini-apps/balsaraFsgrid/main.cpp +++ b/mini-apps/balsaraFsgrid/main.cpp @@ -2,13 +2,141 @@ #include "../../fieldsolver/fs_common.h" #include "../../fieldsolver/fs_limiters.h" #include +#include #include #include +#include +#include "../../logger.h" +#include "../../object_wrapper.h" +Logger logFile, diagnostic; using namespace std; +using namespace SBC; using namespace fsgrids; +// uint Parameters::ohmHallTerm = 0; + +struct Param { + static uint ohmHallTerm; + static Real xmin; /*!< X-coordinate of the lower left corner of the spatial grid. */ + static Real xmax; /*!< X-coordinate of the upper right corner of the spatial grid. */ + static Real ymin; /*!< Y-coordinate of the lower left corner of the spatial grid. */ + static Real ymax; /*!< Y-coordinate of the upper right corner of the spatial grid. */ + static Real zmin; /*!< Z-coordinate of the lower left corner of the spatial grid. */ + static Real zmax; /*!< Z-coordinate of the upper right corner of the spatial grid. */ + static Real dx_ini; /*!< Initial size of spatial cell in x-direction. */ + static Real dy_ini; /*!< Initial size of spatial cell in y-direction. */ + static Real dz_ini; /*!< Initial size of spatial cell in z-direction. */ + static int xcells_ini; + static int ycells_ini; + static int zcells_ini; + static int amrMaxSpatialRefLevel; + static std::array manualFsGridDecomposition; + static void addParameters(); + static void getParameters(); +}; +typedef Param Pb; +uint Pb::ohmHallTerm; +Real Pb::xmin; /*!< X-coordinate of the lower left corner of the spatial grid. */ +Real Pb::xmax; /*!< X-coordinate of the upper right corner of the spatial grid. */ +Real Pb::ymin; /*!< Y-coordinate of the lower left corner of the spatial grid. */ +Real Pb::ymax; /*!< Y-coordinate of the upper right corner of the spatial grid. */ +Real Pb::zmin; /*!< Z-coordinate of the lower left corner of the spatial grid. */ +Real Pb::zmax; /*!< Z-coordinate of the upper right corner of the spatial grid. */ +Real Pb::dx_ini; /*!< Initial size of spatial cell in x-direction. */ +Real Pb::dy_ini; /*!< Initial size of spatial cell in y-direction. */ +Real Pb::dz_ini; /*!< Initial size of spatial cell in z-direction. */ +int Pb::xcells_ini; +int Pb::ycells_ini; +int Pb::zcells_ini; + +int Pb::amrMaxSpatialRefLevel=0; +std::array Pb::manualFsGridDecomposition = {0,0,0}; + +ObjectWrapper objectWrapper; +SysBoundary sysBoundary; +ObjectWrapper& getObjectWrapper() { + return objectWrapper; +} +std::vector localCellDummy; +const std::vector& getLocalCells() { return localCellDummy; } +SysBoundary::SysBoundary() {} +SysBoundary::~SysBoundary() {} +species::Species::~Species() {} + + +int globalflags::bailingOut = 0; +bool globalflags::writeRestart = false; +bool globalflags::writeRecover = false; +bool globalflags::balanceLoad = false; +bool globalflags::doRefine = false; +bool globalflags::ionosphereJustSolved = false; +void Pb::addParameters() { + Readparameters::add( + "fieldsolver.ohmHallTerm", + "Enable/choose spatial order of the Hall term in Ohm's law. 0: off, 1: 1st spatial order, 2: 2nd spatial order", + 0); + Readparameters::add( + "fieldsolver.manualFsGridDecompositionX", + "Manual FsGridDecomposition for field solver grid.", 0); + Readparameters::add( + "fieldsolver.manualFsGridDecompositionY", + "Manual FsGridDecomposition for field solver grid.", 0); + Readparameters::add( + "fieldsolver.manualFsGridDecompositionZ", + "Manual FsGridDecomposition for field solver grid.", 0); + Readparameters::add("gridbuilder.x_min", "Minimum value of the x-coordinate.", NAN); + Readparameters::add("gridbuilder.x_max", "Maximum value of the x-coordinate.", NAN); + Readparameters::add("gridbuilder.y_min", "Minimum value of the y-coordinate.", NAN); + Readparameters::add("gridbuilder.y_max", "Maximum value of the y-coordinate.", NAN); + Readparameters::add("gridbuilder.z_min", "Minimum value of the z-coordinate.", NAN); + Readparameters::add("gridbuilder.z_max", "Maximum value of the z-coordinate.", NAN); + Readparameters::add("AMR.max_spatial_level", "Maximum absolute spatial mesh refinement level", (uint)0); + Readparameters::add("gridbuilder.x_length", "Number of cells in x-direction in initial grid.", 0); + Readparameters::add("gridbuilder.y_length", "Number of cells in y-direction in initial grid.", 0); + Readparameters::add("gridbuilder.z_length", "Number of cells in z-direction in initial grid.", 0); + Readparameters::add("gridbuilder.dx_ini", "Number of cells in x-direction in initial grid.",NAN); + Readparameters::add("gridbuilder.dy_ini", "Number of cells in y-direction in initial grid.",NAN); + Readparameters::add("gridbuilder.dz_ini", "Number of cells in z-direction in initial grid.",NAN); +} -uint Parameters::ohmHallTerm = 0; +void Pb::getParameters() { + int myRank; + + MPI_Comm_rank(MPI_COMM_WORLD, &myRank); + Readparameters::get("fieldsolver.ohmHallTerm", Pb::ohmHallTerm); + Readparameters::get("gridbuilder.x_min", Pb::xmin); + Readparameters::get("gridbuilder.x_max", Pb::xmax); + Readparameters::get("gridbuilder.y_min", Pb::ymin); + Readparameters::get("gridbuilder.y_max", Pb::ymax); + Readparameters::get("gridbuilder.z_min", Pb::zmin); + Readparameters::get("gridbuilder.z_max", Pb::zmax); + Readparameters::get("AMR.max_spatial_level", Pb::amrMaxSpatialRefLevel); + Readparameters::get("gridbuilder.x_length", Pb::xcells_ini); + Readparameters::get("gridbuilder.y_length", Pb::ycells_ini); + Readparameters::get("gridbuilder.z_length", Pb::zcells_ini); + Readparameters::get("gridbuilder.dx_ini",Pb::dx_ini); + Readparameters::get("gridbuilder.dy_ini",Pb::dy_ini); + Readparameters::get("gridbuilder.dz_ini",Pb::dz_ini); + if ( isnan(Pb::dx_ini) || isnan(Pb::dy_ini) || isnan(Pb::dz_ini)) { + if ( Pb::xcells_ini == 0 || Pb::ycells_ini == 0 || Pb::zcells_ini == 0 ) { + if (myRank == MASTER_RANK) + std::cerr << "one of d[x,y or z]_ini and [x,y or z]cells_ini not given, exiting!\nPleae either set the d[x,y,z]_ini or [x,y,z]cells_ini variables! " << std::endl; + exit(1); + } + std::cout << "gridbuilder.d[x,y or z]_ini not set! Will calculate these values from _length and _max/min" << std::endl; + Pb::dx_ini = (Pb::xmax - Pb::xmin) / Pb::xcells_ini; + Pb::dy_ini = (Pb::ymax - Pb::ymin) / Pb::ycells_ini; + Pb::dz_ini = (Pb::zmax - Pb::zmin) / Pb::zcells_ini; + } + + fsgrid::Task_t temp_task_t; + Readparameters::get("fieldsolver.manualFsGridDecompositionX", temp_task_t); + Pb::manualFsGridDecomposition[0] = temp_task_t; + Readparameters::get("fieldsolver.manualFsGridDecompositionY", temp_task_t); + Pb::manualFsGridDecomposition[1] = temp_task_t; + Readparameters::get("fieldsolver.manualFsGridDecompositionZ", temp_task_t); + Pb::manualFsGridDecomposition[2] = temp_task_t; +} // Very simplified version of CalculateDerivatives from fieldsolver/derivatives.cpp void calculateDerivatives(const fsgrid::FsStencil& stencil, fsgrids::perbspan perb, @@ -26,7 +154,7 @@ void calculateDerivatives(const fsgrid::FsStencil& stencil, fsgrids::perbspan pe dPerB[fsgrids::dperb::dPERBydx] = limiter(leftPerB[fsgrids::bfield::PERBY], centPerB[fsgrids::bfield::PERBY], rghtPerB[fsgrids::bfield::PERBY]); dPerB[fsgrids::dperb::dPERBzdx] = limiter(leftPerB[fsgrids::bfield::PERBZ], centPerB[fsgrids::bfield::PERBZ], rghtPerB[fsgrids::bfield::PERBZ]); - if (Parameters::ohmHallTerm < 2) { + if (Pb::ohmHallTerm < 2) { dPerB[fsgrids::dperb::dPERBydxx] = 0.0; dPerB[fsgrids::dperb::dPERBzdxx] = 0.0; } else { @@ -44,7 +172,7 @@ void calculateDerivatives(const fsgrid::FsStencil& stencil, fsgrids::perbspan pe dPerB[fsgrids::dperb::dPERBxdy] = limiter(leftPerB[fsgrids::bfield::PERBX], centPerB[fsgrids::bfield::PERBX], rghtPerB[fsgrids::bfield::PERBX]); dPerB[fsgrids::dperb::dPERBzdy] = limiter(leftPerB[fsgrids::bfield::PERBZ], centPerB[fsgrids::bfield::PERBZ], rghtPerB[fsgrids::bfield::PERBZ]); - if (Parameters::ohmHallTerm < 2) { + if (Pb::ohmHallTerm < 2) { dPerB[fsgrids::dperb::dPERBxdyy] = 0.0; dPerB[fsgrids::dperb::dPERBzdyy] = 0.0; } else { @@ -61,7 +189,7 @@ void calculateDerivatives(const fsgrid::FsStencil& stencil, fsgrids::perbspan pe dPerB[fsgrids::dperb::dPERBxdz] = limiter(leftPerB[fsgrids::bfield::PERBX], centPerB[fsgrids::bfield::PERBX], rghtPerB[fsgrids::bfield::PERBX]); dPerB[fsgrids::dperb::dPERBydz] = limiter(leftPerB[fsgrids::bfield::PERBY], centPerB[fsgrids::bfield::PERBY], rghtPerB[fsgrids::bfield::PERBY]); - if (Parameters::ohmHallTerm < 2) { + if (Pb::ohmHallTerm < 2) { dPerB[fsgrids::dperb::dPERBxdzz] = 0.0; dPerB[fsgrids::dperb::dPERBydzz] = 0.0; } else { @@ -70,7 +198,7 @@ void calculateDerivatives(const fsgrid::FsStencil& stencil, fsgrids::perbspan pe } } - if (Parameters::ohmHallTerm < 2) { + if (Pb::ohmHallTerm < 2) { dPerB[fsgrids::dperb::dPERBxdyz] = 0.0; dPerB[fsgrids::dperb::dPERBydxz] = 0.0; dPerB[fsgrids::dperb::dPERBzdxy] = 0.0; @@ -122,31 +250,24 @@ int main(int argc, char** argv) { } const int masterProcessID = 0; - // Parse parameters - if (argc == 1) { - cerr << "Running with default options. Run main --help to see available settings." << endl; - } - for (int i = 1; i < argc; i++) { - cerr << "Unknown command line option \"" << argv[i] << "\"" << endl; - cerr << endl; - cerr << "main" << endl; - cerr << "Paramters:" << endl; - cerr << " none! :D" << endl; - - return 1; - } - + Readparameters readparameters(argc,argv); + Pb::addParameters(); + readparameters.parse(true, true); // 2nd parsing for specific population parameters + Pb::getParameters(); + Readparameters::helpMessage(); + phiprof::initialize(); + // Set up fsgrids - const std::array fsGridDimensions = {5, 5, 5}; + const std::array fsGridDimensions = {5, 5, 5}; const std::array periodicity{true, true, true}; - const std::array gridSpacing{P::dx_ini / pow(2, P::amrMaxSpatialRefLevel), - P::dy_ini / pow(2, P::amrMaxSpatialRefLevel), - P::dz_ini / pow(2, P::amrMaxSpatialRefLevel)}; - const std::array physicalGlobalStart{P::xmin, P::ymin, P::zmin}; - const auto decomposition = P::manualFsGridDecomposition; + const std::array gridSpacing{Pb::dx_ini / pow(2, Pb::amrMaxSpatialRefLevel), + Pb::dy_ini / pow(2, Pb::amrMaxSpatialRefLevel), + Pb::dz_ini / pow(2, Pb::amrMaxSpatialRefLevel)}; + const std::array physicalGlobalStart{Pb::xmin, Pb::ymin, Pb::zmin}; + const auto decomposition = Pb::manualFsGridDecomposition; MPI_Comm parentComm = MPI_COMM_WORLD; const auto numFsProcs = [&]() { @@ -162,7 +283,6 @@ int main(int argc, char** argv) { fsgrid::FsData technical(fsgrid.getNumStorageCells()); fsgrid::FsData> perb(fsgrid.getNumStorageCells()); fsgrid::FsData> dperb(fsgrid.getNumStorageCells()); - // Fill in values for (int i = 0; i < 5; i++) { for (int j = 0; j < 5; j++) { @@ -193,7 +313,7 @@ int main(int argc, char** argv) { for (int j = 0; j < 5; j++) { for (int k = 0; k < 5; k++) { const auto stencil = fsgrid.makeStencil(i, j, k); - calculateDerivatives(stencil, perb, dperb, technical, fsgrid); + calculateDerivatives(stencil, perb.view(), dperb.view(), technical.view(), fsgrid); } } }