Commit 03d4f489 authored by Tuelin Kaman's avatar Tuelin Kaman

SAAMG test cases for elliptical and arbitrary geometries; parallel decomposition and mapping

parent e49b8232
......@@ -824,8 +824,8 @@ void PartBunch::resizeMesh() {
double xmax = fs_m->solver_m->getXRangeMax();
double ymin = fs_m->solver_m->getYRangeMin();
double ymax = fs_m->solver_m->getYRangeMax();
double zmin = fs_m->solver_m->getZRangeMin();
double zmax = fs_m->solver_m->getZRangeMax();
double zmin = rmin_m[2]; //fs_m->solver_m->getZRangeMin();
double zmax = rmax_m[2]; //fs_m->solver_m->getZRangeMax();
if(xmin > rmin_m[0] || xmax < rmax_m[0] ||
ymin > rmin_m[1] || ymax < rmax_m[1]) {
......@@ -846,9 +846,9 @@ void PartBunch::resizeMesh() {
boundp();
get_bounds(rmin_m, rmax_m);
}
Vector_t mymin = Vector_t(xmin, ymin , zmin);
Vector_t mymax = Vector_t(xmax, ymax , zmax);
// extend domain with extra "ghost" point
Vector_t mymin = Vector_t(xmin, ymin , zmin-hr_m[2]);
Vector_t mymax = Vector_t(xmax, ymax , zmax+hr_m[2]);
for(int i = 0; i < 3; i++)
hr_m[i] = (mymax[i] - mymin[i]) / nr_m[i];
......@@ -877,8 +877,8 @@ void PartBunch::computeSelfFields() {
if(fs_m->hasValidSolver()) {
// if(fs_m->getFieldSolverType() == "SAAMG" && !isGridFixed())
// resizeMesh();
if (fs_m->getFieldSolverType() == "SAAMG")
resizeMesh();
INFOMSG("after resizeMesh" << hr_m << endl);
//scatter charges onto grid
......@@ -926,7 +926,9 @@ void PartBunch::computeSelfFields() {
INFOMSG("*** FINISHED DUMPING SCALAR FIELD ***" << endl);
#endif
// charge density is in rho_m
IpplTimings::startTimer(compPotenTimer_m);
fs_m->solver_m->computePotential(rho_m, hr_scaled);
IpplTimings::stopTimer(compPotenTimer_m);
//do the multiplication of the grid-cube volume coming
//from the discretization of the convolution integral.
......@@ -1126,7 +1128,9 @@ void PartBunch::computeSelfFields_cycl(double gamma, Vector_t const meanR, Vekto
/// now charge density is in rho_m
/// calculate Possion equation (without coefficient: -1/(eps))
IpplTimings::startTimer(compPotenTimer_m);
fs_m->solver_m->computePotential(rho_m, hr_scaled);
IpplTimings::stopTimer(compPotenTimer_m);
/// additional work of FFT solver
/// now the scalar potential is given back in rho_m
......
......@@ -1718,7 +1718,7 @@ void ParallelCyclotronTracker::Tracker_RK4() {
// main integration loop
*gmsg << "* ---------------------------- Start tracking ----------------------------" << endl;
for(; step_m < maxSteps_m; step_m++) {
// *gmsg << "step_m=" << step_m << endl;
*gmsg << "step_m= " << step_m << endl;
bool dumpEachTurn = false;
if(initialTotalNum_m > 2) {
// single particle dumping
......@@ -4060,9 +4060,9 @@ void ParallelCyclotronTracker::initDistInGlobalFrame() {
globalToLocal(itsBunch->R, phi, meanR);
itsBunch->R *= Vector_t(0.001); // mm --> m
itsBunch->boundp();
checkNumPart(std::string("* Before repartation: "));
checkNumPart(std::string("* Before repartition: "));
repartition();
checkNumPart(std::string("* After repartation: "));
checkNumPart(std::string("* After repartition: "));
itsBunch->R *= Vector_t(1000.0); // m --> mm
localToGlobal(itsBunch->R, phi, meanR);
}
......
This diff is collapsed.
......@@ -47,12 +47,19 @@ public:
int getStartId() {return startId;}
double getXRangeMin() { return Geo_mincoords_m(0); }
double getXRangeMax() { return Geo_maxcoords_m(0); }
double getYRangeMin() { return Geo_mincoords_m(1); }
double getYRangeMax() { return Geo_maxcoords_m(1); }
double getZRangeMin() { return Geo_mincoords_m(2); }
double getZRangeMax() { return Geo_maxcoords_m(2); }
double getXRangeMin(){ return intersectMinCoords_m(0); }
double getXRangeMax(){ return intersectMaxCoords_m(0); }
double getYRangeMin(){ return intersectMinCoords_m(1); }
double getYRangeMax(){ return intersectMaxCoords_m(1); }
double getZRangeMin(){ return intersectMinCoords_m(2); }
double getZRangeMax(){ return intersectMaxCoords_m(2); }
void setXRangeMin(double xmin){ intersectMinCoords_m(0) = xmin; }
void setXRangeMax(double xmax){ intersectMaxCoords_m(0) = xmax; }
void setYRangeMin(double ymin){ intersectMinCoords_m(1) = ymin; }
void setYRangeMax(double ymax){ intersectMaxCoords_m(1) = ymax; }
void setZRangeMin(double zmin){ intersectMinCoords_m(2) = zmin; }
void setZRangeMax(double zmax){ intersectMaxCoords_m(2) = zmax; }
bool hasGeometryChanged() { return hasGeometryChanged_m; }
......@@ -99,6 +106,8 @@ private:
Vector_t Geo_hr_m;
Vector_t Geo_mincoords_m;
Vector_t Geo_maxcoords_m;
Vector_t intersectMinCoords_m;
Vector_t intersectMaxCoords_m;
// Conversion from (x,y,z) to index in xyz plane
inline int toCoordIdx(int idx, int idy, int idz);
......
......@@ -60,9 +60,7 @@ void EllipticDomain::Compute(Vector_t hr){
for(x = 0; x < nr[0]; x++) {
for(y = 0; y < nr[1]; y++) {
if(isInside(x, y, 1)) {
//IdxMap[toCoordIdx(x, y)] = idx++;
IdxMap[toCoordIdx(x, y)] = idx;
CoordMap[idx++] = toCoordIdx(x, y);
nxy_m++;
......@@ -116,10 +114,86 @@ void EllipticDomain::Compute(Vector_t hr){
}
void EllipticDomain::Compute(Vector_t hr, NDIndex<3> localId){
//there is nothing to be done if the mesh spacings have not changed
if(hr[0] == getHr()[0] && hr[1] == getHr()[1] && hr[2] == getHr()[2]) {
hasGeometryChanged_m = false;
return;
}
setHr(hr);
hasGeometryChanged_m = true;
//reset number of points inside domain
nxy_m = 0;
// clear previous coordinate maps
IdxMap.clear();
CoordMap.clear();
//clear previous intersection points
IntersectYDir.clear();
IntersectXDir.clear();
// build a index and coordinate map
register int idx = 0;
register int x, y;
for(x = localId[0].first(); x<= localId[0].last(); x++) {
for(y = localId[1].first(); y <= localId[1].last(); y++) {
if(isInside(x, y, 1)) {
IdxMap[toCoordIdx(x, y)] = idx;
CoordMap[idx++] = toCoordIdx(x, y);
nxy_m++;
}
}
}
switch(interpolationMethod) {
case CONSTANT:
break;
case LINEAR:
case QUADRATIC:
double smajsq = SemiMajor * SemiMajor;
double sminsq = SemiMinor * SemiMinor;
double yd = 0.0;
double xd = 0.0;
double pos = 0.0;
double mx = (nr[0] - 1) * hr[0] / 2.0;
double my = (nr[1] - 1) * hr[1] / 2.0;
//calculate intersection with the ellipse
for(x = localId[0].first(); x <= localId[0].last(); x++) {
pos = x * hr[0] - mx;
if (pos <= -SemiMajor || pos >= SemiMajor)
{
IntersectYDir.insert(std::pair<int, double>(x, 0));
IntersectYDir.insert(std::pair<int, double>(x, 0));
}else{
yd = std::abs(sqrt(sminsq - sminsq * pos * pos / smajsq)); // + 0.5*nr[1]*hr[1]);
IntersectYDir.insert(std::pair<int, double>(x, yd));
IntersectYDir.insert(std::pair<int, double>(x, -yd));
}
}
for(y = localId[0].first(); y < localId[1].last(); y++) {
pos = y * hr[1] - my;
if (pos <= -SemiMinor || pos >= SemiMinor)
{
IntersectXDir.insert(std::pair<int, double>(y, 0));
IntersectXDir.insert(std::pair<int, double>(y, 0));
}else{
xd = std::abs(sqrt(smajsq - smajsq * pos * pos / sminsq)); // + 0.5*nr[0]*hr[0]);
IntersectXDir.insert(std::pair<int, double>(y, xd));
IntersectXDir.insert(std::pair<int, double>(y, -xd));
}
}
}
}
void EllipticDomain::Compute(Vector_t hr, NDIndex<3> localId, Vector_t globalMeanR, Vektor<double, 4> globalToLocalQuaternion){
}
void EllipticDomain::getBoundaryStencil(int x, int y, int z, double &W, double &E, double &S, double &N, double &F, double &B, double &C, double &scaleFactor) {
scaleFactor = 1.0;
......
......@@ -57,19 +57,15 @@ MGPoissonSolver::MGPoissonSolver(PartBunch &beam,Mesh_t *mesh, FieldLayout_t *fl
if(currentGeometry->getFilename() == "") {
if(currentGeometry->getTopology() == "ELLIPTIC"){
bp = new EllipticDomain(currentGeometry->getA(), currentGeometry->getB(), orig_nr_m, hr_m, interpl);
}
else if(currentGeometry->getTopology() == "BOXCORNER") {
} else if (currentGeometry->getTopology() == "BOXCORNER") {
bp = new BoxCornerDomain(currentGeometry->getA(), currentGeometry->getB(), currentGeometry->getC(), currentGeometry->getLength(),currentGeometry->getL1(), currentGeometry->getL2(), orig_nr_m, hr_m, interpl);
bp->Compute(itsBunch_m->get_hr());
} else {
ERRORMSG("Geometry not known" << endl);
exit(1);
}
} else {
localId = layout_m->getLocalNDIndex();
} else
bp = new ArbitraryDomain(currentGeometry, orig_nr_m, hr_m, interpl);
}
Map = 0;
A = Teuchos::null;
......@@ -82,7 +78,6 @@ MGPoissonSolver::MGPoissonSolver(PartBunch &beam,Mesh_t *mesh, FieldLayout_t *fl
nLHS = Options::nLHS;
SetupMLList();
SetupBelosList();
// setup Belos solver
if(numBlocks_m == 0 || recycleBlocks_m == 0)
solver = rcp(new Belos::BlockCGSolMgr<double, MV, OP>());
......@@ -124,7 +119,9 @@ void MGPoissonSolver::computeMap(NDIndex<3> localId) {
delete Map;
if(useRCB_m)
redistributeWithRCB(localId);
else
else if ( bp->getType() == "Geometric" )
IPPLToMap3DGeo(localId);
else if ( bp->getType() == "Elliptic" )
IPPLToMap3D(localId);
}
extrapolateLHS();
......@@ -136,10 +133,8 @@ void MGPoissonSolver::extrapolateLHS() {
// Pik (x) := (x − xi ) Pi+1,k−1(x) − (x − xi+k ) Pi,k−1(x) /(xi+k − xi )
// k = 1, . . . , n, i = 0, . . . , n − k.
if(LHS == Teuchos::null) {
LHS = rcp(new Epetra_Vector(*Map));
LHS->PutScalar(1.0);
}
LHS = rcp(new Epetra_Vector(*Map));
LHS->PutScalar(0.0);
std::deque< Epetra_Vector >::iterator it = OldLHS.begin();
......@@ -178,28 +173,27 @@ void MGPoissonSolver::computePotential(Field_t &rho, Vector_t hr) {
nr_m[0] = orig_nr_m[0];
nr_m[1] = orig_nr_m[1];
nr_m[2] = orig_nr_m[2];
bp->setMinMaxZ(itsBunch_m->get_origin()[2], itsBunch_m->get_maxExtend()[2]);
bp->setNr(nr_m);
layout_m = &itsBunch_m->getFieldLayout();
localId = layout_m->getLocalNDIndex();
IpplTimings::startTimer(FunctionTimer1_m);
bp->Compute(hr, localId, itsBunch_m->getGlobalMeanR(), itsBunch_m->getGlobalToLocalQuaternion());
if ( bp->getType() == "Geometric" )
bp->Compute(hr, localId, itsBunch_m->getGlobalMeanR(), itsBunch_m->getGlobalToLocalQuaternion());
else if ( bp->getType() == "Elliptic" )
bp->Compute(hr);
IpplTimings::stopTimer(FunctionTimer1_m);
// Define the Map
IpplTimings::startTimer(FunctionTimer2_m);
computeMap(localId); // Define the Map
computeMap(localId);
IpplTimings::stopTimer(FunctionTimer2_m);
// Allocate the RHS with the new Epetra Map
if(RHS == Teuchos::null) {
RHS = rcp(new Epetra_Vector(*Map));
RHS->PutScalar(0.0);
}
RHS = rcp(new Epetra_Vector(*Map));
RHS->PutScalar(0.0);
// get charge densities from IPPL field and store in Epetra vector (RHS)
IpplTimings::startTimer(FunctionTimer3_m);
int id = 0;
......@@ -216,8 +210,7 @@ void MGPoissonSolver::computePotential(Field_t &rho, Vector_t hr) {
// build discretization matrix
IpplTimings::startTimer(FunctionTimer4_m);
if (A == Teuchos::null)
A = rcp(new Epetra_CrsMatrix(Copy, *Map, 7, true));
A = rcp(new Epetra_CrsMatrix(Copy, *Map, 7, true));
ComputeStencil(hr, RHS);
IpplTimings::stopTimer(FunctionTimer4_m);
......@@ -226,24 +219,24 @@ void MGPoissonSolver::computePotential(Field_t &rho, Vector_t hr) {
#endif
IpplTimings::startTimer(FunctionTimer5_m);
if (MLPrec == Teuchos::null)
MLPrec = rcp(new ML_Epetra::MultiLevelPreconditioner(*A, MLList_m));
switch(precmode_m) {
case REUSE_PREC:
if (MLPrec == Teuchos::null)
MLPrec = rcp(new ML_Epetra::MultiLevelPreconditioner(*A, MLList_m));
break;
case REUSE_HIERARCHY:
MLPrec->ReComputePreconditioner();
if (MLPrec == Teuchos::null)
MLPrec = rcp(new ML_Epetra::MultiLevelPreconditioner(*A, MLList_m));
else
MLPrec->ReComputePreconditioner();
break;
case STD_PREC:
MLPrec = rcp(new ML_Epetra::MultiLevelPreconditioner(*A, MLList_m));
break;
}
}
IpplTimings::stopTimer(FunctionTimer5_m);
// setup preconditioned iterative solver
// use old LHS solution as initial guess
IpplTimings::startTimer(FunctionTimer6_m);
......@@ -256,11 +249,10 @@ void MGPoissonSolver::computePotential(Field_t &rho, Vector_t hr) {
solver->setProblem(rcp(&problem,false));
if(!problem.isProblemSet()){
if (problem.setProblem() == false) {
std::cout << std::endl << "ERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
ERRORMSG("Belos::LinearProblem failed to set up correctly!" << endl);
}
}
IpplTimings::stopTimer(FunctionTimer6_m);
std::ofstream timings;
char filename[50];
sprintf(filename, "timing_MX%d_MY%d_MZ%d_nProc%d_recB%d_numB%d_nLHS%d", orig_nr_m[0], orig_nr_m[1], orig_nr_m[2], Comm.NumProc(), recycleBlocks_m, numBlocks_m, nLHS);
......@@ -298,7 +290,6 @@ void MGPoissonSolver::computePotential(Field_t &rho, Vector_t hr) {
IpplTimings::startTimer(FunctionTimer8_m);
id = 0;
rho = 0.0;
for (int idz = localId[2].first(); idz <= localId[2].last(); idz++) {
for (int idy = localId[1].first(); idy <= localId[1].last(); idy++) {
for (int idx = localId[0].first(); idx <= localId[0].last(); idx++) {
......@@ -371,14 +362,13 @@ void MGPoissonSolver::redistributeWithRCB(NDIndex<3> localId) {
}
void MGPoissonSolver::IPPLToMap3D(NDIndex<3> localId) {
int NumMyElements = 0;
vector<int> MyGlobalElements;
for (int idz = localId[2].first(); idz <= localId[2].last(); idz++) {
for (int idy = localId[1].first(); idy <= localId[1].last(); idy++) {
for (int idx = localId[0].first(); idx <= localId[0].last(); idx++) {
if (bp->isInside(idx, idy, idz)) {
if (bp->isInside(idx, idy, idz)) {
MyGlobalElements.push_back(bp->getIdx(idx, idy, idz));
NumMyElements++;
}
......@@ -388,6 +378,21 @@ void MGPoissonSolver::IPPLToMap3D(NDIndex<3> localId) {
Map = new Epetra_Map(-1, NumMyElements, &MyGlobalElements[0], 0, Comm);
}
void MGPoissonSolver::IPPLToMap3DGeo(NDIndex<3> localId) {
int NumMyElements = 0;
vector<int> MyGlobalElements;
for (int idz = localId[2].first(); idz <= localId[2].last(); idz++) {
for (int idy = localId[1].first(); idy <= localId[1].last(); idy++) {
for (int idx = localId[0].first(); idx <= localId[0].last(); idx++) {
MyGlobalElements.push_back(bp->getIdx(idx, idy, idz));
NumMyElements++;
}
}
}
Map = new Epetra_Map(-1, NumMyElements, &MyGlobalElements[0], 0, Comm);
}
void MGPoissonSolver::ComputeStencil(Vector_t hr, Teuchos::RCP<Epetra_Vector> RHS) {
A->PutScalar(0.0);
......
......@@ -234,6 +234,7 @@ private:
/// converts IPPL grid to a 3D Epetra_Map
/// \param localId local IPPL grid node indices
void IPPLToMap3D(NDIndex<3> localId);
void IPPLToMap3DGeo(NDIndex<3> localId);
/** returns a discretized stencil that has Neumann BC in z direction and
* Dirichlet BC on the surface of a specified geometry
......
......@@ -371,8 +371,6 @@ void TrackRun::execute() {
prob_lo_in[i] = prob_lo[i];
}
std::cout << "#TK ncell" << ncell[0] << " " << ncell[1] << " " << ncell[2] << std::endl;
// We set this to -1 so that we can now control max_lev from the inputs file
int maxLevel = -1;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment