Commit 48fc1cb6 authored by frey_m's avatar frey_m
Browse files

AMR test case: Add ippl timings.

modified:   ippl/test/AMR/Solver.cpp
modified:   ippl/test/AMR/testGaussian.cpp
parent a97645df
#include "Solver.h"
#include "Ippl.h"
void
Solver::solve_for_accel(container_t& rhs,
container_t& phi,
......@@ -9,7 +11,8 @@ Solver::solve_for_accel(container_t& rhs,
int finest_level,
Real offset)
{
static IpplTimings::TimerRef edge2centerTimer = IpplTimings::getTimer("gradient-edge2center");
Real tol = 1.e-10;
Real abs_tol = 1.e-14;
......@@ -48,6 +51,7 @@ Solver::solve_for_accel(container_t& rhs,
abs_tol);
// Average edge-centered gradients to cell centers and fill the values in ghost cells.
IpplTimings::startTimer(edge2centerTimer);
for (int lev = base_level; lev <= finest_level; lev++)
{
#ifdef UNIQUE_PTR
......@@ -79,6 +83,7 @@ Solver::solve_for_accel(container_t& rhs,
grad_phi[lev].mult(-1.0, 0, 3);
}
#endif
IpplTimings::stopTimer(edge2centerTimer);
}
......@@ -92,6 +97,12 @@ Solver::solve_with_f90(container_t& rhs,
Real tol,
Real abs_tol)
{
static IpplTimings::TimerRef initSolverTimer = IpplTimings::getTimer("init-solver");
static IpplTimings::TimerRef doSolveTimer = IpplTimings::getTimer("do-solve");
static IpplTimings::TimerRef gradientTimer = IpplTimings::getTimer("gradient");
IpplTimings::startTimer(initSolverTimer);
int nlevs = finest_level - base_level + 1;
int mg_bc[2*BL_SPACEDIM];
......@@ -99,7 +110,7 @@ Solver::solve_with_f90(container_t& rhs,
// This tells the solver that we are using Dirichlet bc's
if (Geometry::isAllPeriodic()) {
// if ( ParallelDescriptor::IOProcessor() )
std::cerr << "Periodic BC" << std::endl;
// std::cerr << "Periodic BC" << std::endl;
for (int dir = 0; dir < BL_SPACEDIM; ++dir) {
// periodic BC
......@@ -108,7 +119,7 @@ Solver::solve_with_f90(container_t& rhs,
}
} else {
// if ( ParallelDescriptor::IOProcessor() )
std::cerr << "Dirichlet BC" << std::endl;
// std::cerr << "Dirichlet BC" << std::endl;
for (int dir = 0; dir < BL_SPACEDIM; ++dir) {
// Dirichlet BC
......@@ -177,8 +188,14 @@ Solver::solve_with_f90(container_t& rhs,
int always_use_bnorm = 0;
int need_grad_phi = 1;
fmg.set_verbose(1);
IpplTimings::stopTimer(initSolverTimer);
IpplTimings::startTimer(doSolveTimer);
fmg.solve(phi_p, rhs_p, tol, abs_tol, always_use_bnorm, need_grad_phi);
IpplTimings::stopTimer(doSolveTimer);
IpplTimings::startTimer(gradientTimer);
#ifdef UNIQUE_PTR
const Array<Array<MultiFab*> >& g_phi_edge = BoxLib::GetArrOfArrOfPtrs(grad_phi_edge);
for (int ilev = 0; ilev < nlevs; ++ilev) {
......@@ -191,7 +208,7 @@ Solver::solve_with_f90(container_t& rhs,
fmg.get_fluxes(grad_phi_edge[amr_level], ilev);
}
#endif
IpplTimings::stopTimer(gradientTimer);
}
#ifdef USEHYPRE
......
......@@ -53,25 +53,33 @@ void doSolve(AmrOpal& myAmrOpal, PartBunchBase* bunch,
int nLevels,
Inform& msg)
{
static IpplTimings::TimerRef solvTimer = IpplTimings::getTimer("solv");
static IpplTimings::TimerRef allocTimer = IpplTimings::getTimer("allocate-memory-grid");
static IpplTimings::TimerRef assignTimer = IpplTimings::getTimer("assign-charge");
// =======================================================================
// 4. prepare for multi-level solve
// =======================================================================
rhs.resize(nLevels);
phi.resize(nLevels);
grad_phi.resize(nLevels);
IpplTimings::startTimer(allocTimer);
for (int lev = 0; lev < nLevels; ++lev) {
initGridData(rhs, phi, grad_phi, myAmrOpal.boxArray()[lev], lev);
}
IpplTimings::stopTimer(allocTimer);
// Define the density on level 0 from all particles at all levels
int base_level = 0;
int finest_level = myAmrOpal.finestLevel();
IpplTimings::startTimer(assignTimer);
dynamic_cast<AmrPartBunch*>(bunch)->AssignDensity(0, false, rhs, base_level, 1, finest_level);
IpplTimings::stopTimer(assignTimer);
double totCharge = totalCharge(rhs, finest_level, geom);
msg << "Total Charge: " << totCharge << " C" << endl
......@@ -98,7 +106,6 @@ void doSolve(AmrOpal& myAmrOpal, PartBunchBase* bunch,
// solve
Solver sol;
IpplTimings::startTimer(solvTimer);
sol.solve_for_accel(rhs,
phi,
grad_phi,
......@@ -115,14 +122,13 @@ void doSolve(AmrOpal& myAmrOpal, PartBunchBase* bunch,
rhs[i].mult(1.0 / constant, 0, 1);
#endif
}
IpplTimings::stopTimer(solvTimer);
}
void doBoxLib(const Vektor<size_t, 3>& nr, size_t nParticles,
int nLevels, size_t maxBoxSize, Inform& msg)
{
static IpplTimings::TimerRef distTimer = IpplTimings::getTimer("dist");
static IpplTimings::TimerRef distTimer = IpplTimings::getTimer("init-distribution");
static IpplTimings::TimerRef regridTimer = IpplTimings::getTimer("regrid");
// ========================================================================
// 1. initialize physical domain (just single-level)
// ========================================================================
......@@ -211,8 +217,10 @@ void doBoxLib(const Vektor<size_t, 3>& nr, size_t nParticles,
// ========================================================================
// 3. multi-level redistribute
// ========================================================================
IpplTimings::startTimer(regridTimer);
for (int i = 0; i <= myAmrOpal.finestLevel() && i < myAmrOpal.maxLevel(); ++i)
myAmrOpal.regrid(i /*lbase*/, 0.0 /*time*/);
IpplTimings::stopTimer(regridTimer);
container_t rhs;
container_t phi;
......@@ -286,6 +294,13 @@ int main(int argc, char *argv[]) {
IpplTimings::stopTimer(mainTimer);
IpplTimings::print();
std::string timefile = std::string(argv[0])
+ "-timing-cores-"
+ std::to_string(Ippl::getNodes())
+ "-threads-1.dat";
IpplTimings::print(timefile);
return 0;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment