DataSink.cpp 50.6 KB
Newer Older
gsell's avatar
gsell committed
1
//
gsell's avatar
gsell committed
2
//  Copyright & License: See Copyright.readme in src directory
gsell's avatar
gsell committed
3 4
//

kraus's avatar
kraus committed
5
#include "Structure/DataSink.h"
gsell's avatar
gsell committed
6

7
#include "OPALconfig.h"
8
#include "Algorithms/bet/EnvelopeBunch.h"
gsell's avatar
gsell committed
9
#include "AbstractObjects/OpalData.h"
10
#include "Utilities/Options.h"
11
#include "Utilities/Util.h"
12
#include "Fields/Fieldmap.h"
13
#include "Structure/BoundaryGeometry.h"
14 15
#include "Structure/H5PartWrapper.h"
#include "Structure/H5PartWrapperForPS.h"
16
#include "Utilities/Timer.h"
17

18 19
#include "H5hut.h"

20
#include <boost/filesystem.hpp>
21
#include <boost/regex.hpp>
22

23
#include <queue>
24
#include <sstream>
25

gsell's avatar
gsell committed
26 27 28 29 30 31 32 33 34 35 36 37
extern Inform *gmsg;

using namespace std;

//using Physics::m_e;

// backward compatibility with 1.99.5
#if defined(H5_O_FLUSHSTEP)
#define H5_FLUSH_STEP H5_O_FLUSHSTEP
#endif

DataSink::DataSink() :
38
    H5call_m(0),
adelmann's avatar
adelmann committed
39
    lossWrCounter_m(0),
40
    doHDF5_m(true),
41
    h5wrapper_m(NULL)
42
{ }
gsell's avatar
gsell committed
43

44
DataSink::DataSink(H5PartWrapper *h5wrapper, int restartStep):
45
    H5call_m(0),
46
    lossWrCounter_m(0),
47
    h5wrapper_m(h5wrapper)
kraus's avatar
kraus committed
48
{
49 50
    namespace fs = boost::filesystem;

adelmann's avatar
adelmann committed
51
    doHDF5_m = Options::enableHDF5;
kraus's avatar
kraus committed
52
    if (!doHDF5_m) {
53 54
        throw OpalException("DataSink::DataSink(int)",
                            "Can not restart when HDF5 is disabled");
adelmann's avatar
cleanup  
adelmann committed
55
    }
56

57 58
    H5PartTimer_m = IpplTimings::getTimer("Write H5-File");
    StatMarkerTimer_m = IpplTimings::getTimer("Write Stat");
gsell's avatar
gsell committed
59

60
    /// Set flags whether SDDS file exists or have to write header first.
gsell's avatar
gsell committed
61
    firstWriteToStat_m = true;
62

63
    string fn = OpalData::getInstance()->getInputBasename();
gsell's avatar
gsell committed
64 65 66 67

    statFileName_m = fn + string(".stat");
    lBalFileName_m = fn + string(".lbal");

68 69
    unsigned int linesToRewind = 0;
    if (fs::exists(statFileName_m)) {
70
        firstWriteToStat_m = false;
71 72 73
        INFOMSG("Appending statistical data to existing data file: " << statFileName_m << endl);
        double spos = h5wrapper->getLastPosition();
        linesToRewind = rewindSDDStoSPos(spos);
gsell's avatar
gsell committed
74
    } else {
75
        INFOMSG("Creating new file for statistical data: " << statFileName_m << endl);
gsell's avatar
gsell committed
76 77
    }

78 79 80
    if (fs::exists(lBalFileName_m)) {
        INFOMSG("Appending load balance data to existing data file: " << lBalFileName_m << endl);
        rewindLinesLBal(linesToRewind);
gsell's avatar
gsell committed
81
    } else {
82
        INFOMSG("Creating new file for load balance data: " << lBalFileName_m << endl);
gsell's avatar
gsell committed
83 84 85
    }
}

86
DataSink::DataSink(H5PartWrapper *h5wrapper):
87
    H5call_m(0),
88
    lossWrCounter_m(0),
89
    h5wrapper_m(h5wrapper)
90 91 92
{
    /// Constructor steps:
    /// Get timers from IPPL.
93 94
    H5PartTimer_m = IpplTimings::getTimer("Write H5-File");
    StatMarkerTimer_m = IpplTimings::getTimer("Write Stat");
95

96 97 98 99 100 101 102 103 104
    /// Set file write flags to true. These will be set to false after first
    /// write operation.
    firstWriteToStat_m = true;
    firstWriteH5Surface_m = true;
    /// Define file names.
    string fn = OpalData::getInstance()->getInputBasename();
    surfaceLossFileName_m = fn + string(".SurfaceLoss.h5");
    statFileName_m = fn + string(".stat");
    lBalFileName_m = fn + string(".lbal");
gsell's avatar
gsell committed
105

106
    doHDF5_m = Options::enableHDF5;
gsell's avatar
gsell committed
107

108
    h5wrapper_m->writeHeader();
gsell's avatar
gsell committed
109 110
}

111 112 113
DataSink::~DataSink() {
    h5wrapper_m = NULL;
}
adelmann's avatar
adelmann committed
114

115
void DataSink::storeCavityInformation() {
116
    if (!doHDF5_m) return;
kraus's avatar
kraus committed
117

118 119
    h5wrapper_m->storeCavityInformation();
}
gsell's avatar
gsell committed
120

121
void DataSink::writePhaseSpace(PartBunch &beam, Vector_t FDext[]) {
gsell's avatar
gsell committed
122

123
    if (!doHDF5_m) return;
gsell's avatar
gsell committed
124

125 126
    IpplTimings::startTimer(H5PartTimer_m);
    std::map<std::string, double> additionalAttributes = {
127 128 129 130 131 132
        std::make_pair("B-ref_x", FDext[0](0)),
        std::make_pair("B-ref_z", FDext[0](1)),
        std::make_pair("B-ref_y", FDext[0](2)),
        std::make_pair("E-ref_x", FDext[1](0)),
        std::make_pair("E-ref_z", FDext[1](1)),
        std::make_pair("E-ref_y", FDext[1](2))};
133 134 135

    h5wrapper_m->writeStep(beam, additionalAttributes);
    IpplTimings::stopTimer(H5PartTimer_m);
gsell's avatar
gsell committed
136

137 138
    return;
}
gsell's avatar
gsell committed
139 140 141



142 143 144 145
int DataSink::writePhaseSpace_cycl(PartBunch &beam, Vector_t FDext[], double meanEnergy,
                                   double refPr, double refPt, double refPz,
                                   double refR, double refTheta, double refZ,
                                   double azimuth, double elevation, bool local) {
gsell's avatar
gsell committed
146

147
    if (!doHDF5_m) return -1;
148
    if (beam.getLocalNum() < 3) return -1; // in single particle mode and tune calculation (2 particles) we do not need h5 data
gsell's avatar
gsell committed
149

150 151 152 153 154 155 156 157 158 159
    IpplTimings::startTimer(H5PartTimer_m);
    std::map<std::string, double> additionalAttributes = {
        std::make_pair("REFPR", refPr),
        std::make_pair("REFPT", refPt),
        std::make_pair("REFPZ", refPz),
        std::make_pair("REFR", refR),
        std::make_pair("REFTHETA", refTheta),
        std::make_pair("REFZ", refZ),
        std::make_pair("AZIMUTH", azimuth),
        std::make_pair("ELEVATION", elevation),
160 161 162 163 164 165 166 167 168 169 170 171
        std::make_pair("B-head_x", FDext[0](0)),
        std::make_pair("B-head_z", FDext[0](1)),
        std::make_pair("B-head_y", FDext[0](2)),
        std::make_pair("E-head_x", FDext[1](0)),
        std::make_pair("E-head_z", FDext[1](1)),
        std::make_pair("E-head_y", FDext[1](2)),
        std::make_pair("B-ref_x", FDext[2](0)),
        std::make_pair("B-ref_z", FDext[2](1)),
        std::make_pair("B-ref_y", FDext[2](2)),
        std::make_pair("E-ref_x", FDext[3](0)),
        std::make_pair("E-ref_z", FDext[3](1)),
        std::make_pair("E-ref_y", FDext[3](2)),
172 173 174 175 176 177 178 179 180
        std::make_pair("B-tail_x", FDext[4](0)),
        std::make_pair("B-tail_z", FDext[4](1)),
        std::make_pair("B-tail_y", FDext[4](2)),
        std::make_pair("E-tail_x", FDext[5](0)),
        std::make_pair("E-tail_z", FDext[5](1)),
        std::make_pair("E-tail_y", FDext[5](2))};

    h5wrapper_m->writeStep(beam, additionalAttributes);
    IpplTimings::stopTimer(H5PartTimer_m);
gsell's avatar
gsell committed
181

182 183 184
    ++ H5call_m;
    return H5call_m - 1;
}
gsell's avatar
gsell committed
185

186
void DataSink::writePhaseSpaceEnvelope(EnvelopeBunch &beam, Vector_t FDext[], double sposHead, double sposRef, double sposTail) {
gsell's avatar
gsell committed
187

188
    if (!doHDF5_m) return;
gsell's avatar
gsell committed
189

190 191 192 193 194
    IpplTimings::startTimer(H5PartTimer_m);
    std::map<std::string, double> additionalAttributes = {
        std::make_pair("sposHead", sposHead),
        std::make_pair("sposRef", sposRef),
        std::make_pair("sposTail", sposTail),
195 196 197 198 199 200 201 202 203 204 205 206
        std::make_pair("B-head_x", FDext[0](0)),
        std::make_pair("B-head_z", FDext[0](1)),
        std::make_pair("B-head_y", FDext[0](2)),
        std::make_pair("E-head_x", FDext[1](0)),
        std::make_pair("E-head_z", FDext[1](1)),
        std::make_pair("E-head_y", FDext[1](2)),
        std::make_pair("B-ref_x", FDext[2](0)),
        std::make_pair("B-ref_z", FDext[2](1)),
        std::make_pair("B-ref_y", FDext[2](2)),
        std::make_pair("E-ref_x", FDext[3](0)),
        std::make_pair("E-ref_z", FDext[3](1)),
        std::make_pair("E-ref_y", FDext[3](2)),
207 208 209 210 211 212 213 214 215 216
        std::make_pair("B-tail_x", FDext[4](0)),
        std::make_pair("B-tail_z", FDext[4](1)),
        std::make_pair("B-tail_y", FDext[4](2)),
        std::make_pair("E-tail_x", FDext[5](0)),
        std::make_pair("E-tail_z", FDext[5](1)),
        std::make_pair("E-tail_y", FDext[5](2))};

    h5wrapper_m->writeStep(beam, additionalAttributes);
    IpplTimings::stopTimer(H5PartTimer_m);
}
gsell's avatar
gsell committed
217

218
void DataSink::stashPhaseSpaceEnvelope(EnvelopeBunch &beam, Vector_t FDext[], double sposHead, double sposRef, double sposTail) {
gsell's avatar
gsell committed
219

220
    if (!doHDF5_m) return;
gsell's avatar
gsell committed
221

222 223
    /// Start timer.
    IpplTimings::startTimer(H5PartTimer_m);
gsell's avatar
gsell committed
224

225 226 227 228 229
    static_cast<H5PartWrapperForPS*>(h5wrapper_m)->stashPhaseSpaceEnvelope(beam,
                                                                           FDext,
                                                                           sposHead,
                                                                           sposRef,
                                                                           sposTail);
gsell's avatar
gsell committed
230 231 232 233 234 235 236
    H5call_m++;

    /// %Stop timer.
    IpplTimings::stopTimer(H5PartTimer_m);
}

void DataSink::dumpStashedPhaseSpaceEnvelope() {
kraus's avatar
kraus committed
237

238
    if (!doHDF5_m) return;
kraus's avatar
kraus committed
239

240
    static_cast<H5PartWrapperForPS*>(h5wrapper_m)->dumpStashedPhaseSpaceEnvelope();
gsell's avatar
gsell committed
241 242 243 244 245

    /// %Stop timer.
    IpplTimings::stopTimer(H5PartTimer_m);
}

246 247
void DataSink::writeStatData(PartBunch &beam, Vector_t FDext[], double E) {
    doWriteStatData(beam, FDext, E, std::vector<std::pair<std::string, unsigned int> >());
248 249
}

250 251
void DataSink::writeStatData(PartBunch &beam, Vector_t FDext[], const std::vector<std::pair<std::string, unsigned int> >& losses) {
    doWriteStatData(beam, FDext, beam.get_meanKineticEnergy(), losses);
252 253 254
}


255
void DataSink::doWriteStatData(PartBunch &beam, Vector_t FDext[], double Ekin, const std::vector<std::pair<std::string, unsigned int> > &losses) {
gsell's avatar
gsell committed
256 257 258 259 260 261 262 263

    /// Start timer.
    IpplTimings::startTimer(StatMarkerTimer_m);

    /// Set width of write fields in output files.
    unsigned int pwi = 10;

    /// Calculate beam statistics and gather load balance statistics.
264
    beam.calcBeamParameters();
gsell's avatar
gsell committed
265
    beam.gatherLoadBalanceStatistics();
kraus's avatar
kraus committed
266

267 268 269 270 271
    size_t npOutside = 0;
    if (Options::beamHaloBoundary>0)
        npOutside = beam.calcNumPartsOutside(Options::beamHaloBoundary*beam.get_rrms());
    // *gmsg << "npOutside 1 = " << npOutside << " beamHaloBoundary= " << Options::beamHaloBoundary << " rrms= " << beam.get_rrms() << endl;

272 273
    double  pathLength = 0.0;
    if (OpalData::getInstance()->isInOPALCyclMode())
274
        pathLength = beam.getLPath();
275
    else
276
        pathLength = beam.get_sPos();
kraus's avatar
kraus committed
277

gsell's avatar
gsell committed
278 279 280 281
    /// Write data to files. If this is the first write to the beam statistics file, write SDDS
    /// header information.
    ofstream os_statData;
    ofstream os_lBalData;
282
    double Q = beam.getCharge();
gsell's avatar
gsell committed
283 284 285 286 287 288

    if(Ippl::myNode() == 0) {
        if(firstWriteToStat_m) {
            os_statData.open(statFileName_m.c_str(), ios::out);
            os_statData.precision(15);
            os_statData.setf(ios::scientific, ios::floatfield);
289
            writeSDDSHeader(os_statData, losses);
gsell's avatar
gsell committed
290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306

            os_lBalData.open(lBalFileName_m.c_str(), ios::out);
            os_lBalData.precision(15);
            os_lBalData.setf(ios::scientific, ios::floatfield);
            os_lBalData << "# " << Ippl::getNodes() << endl;

            firstWriteToStat_m = false;
        } else {
            os_statData.open(statFileName_m.c_str(), ios::app);
            os_statData.precision(15);
            os_statData.setf(ios::scientific, ios::floatfield);

            os_lBalData.open(lBalFileName_m.c_str(), ios::app);
            os_lBalData.precision(15);
            os_lBalData.setf(ios::scientific, ios::floatfield);
        }

307
        os_statData << beam.getT() * 1e9 << setw(pwi) << "\t"                                 // 1
308
                    << pathLength << setw(pwi) << "\t"                                        // 2
gsell's avatar
gsell committed
309 310

                    << beam.getTotalNum() << setw(pwi) << "\t"                                // 3
311
                    << Q << setw(pwi) << "\t"                                                 // 4
gsell's avatar
gsell committed
312

313
                    << Ekin << setw(pwi) << "\t"                                              // 5
gsell's avatar
gsell committed
314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330

                    << beam.get_rrms()(0) << setw(pwi) << "\t"                                // 6
                    << beam.get_rrms()(1) << setw(pwi) << "\t"                                // 7
                    << beam.get_rrms()(2) << setw(pwi) << "\t"                                // 8

                    << beam.get_prms()(0) << setw(pwi) << "\t"                                // 9
                    << beam.get_prms()(1) << setw(pwi) << "\t"                                // 10
                    << beam.get_prms()(2) << setw(pwi) << "\t"                                // 11

                    << beam.get_norm_emit()(0) << setw(pwi) << "\t"                           // 12
                    << beam.get_norm_emit()(1) << setw(pwi) << "\t"                           // 13
                    << beam.get_norm_emit()(2) << setw(pwi) << "\t"                           // 14

                    << beam.get_rmean()(0)  << setw(pwi) << "\t"                              // 15
                    << beam.get_rmean()(1)  << setw(pwi) << "\t"                              // 16
                    << beam.get_rmean()(2)  << setw(pwi) << "\t"                              // 17

331 332 333
                    << beam.RefPartR_m(0) << setw(pwi) << "\t"                                // 18
                    << beam.RefPartR_m(1) << setw(pwi) << "\t"                                // 19
                    << beam.RefPartR_m(2) << setw(pwi) << "\t"                                // 20
gsell's avatar
gsell committed
334

335 336 337
                    << beam.RefPartP_m(0) << setw(pwi) << "\t"                                // 21
                    << beam.RefPartP_m(1) << setw(pwi) << "\t"                                // 22
                    << beam.RefPartP_m(2) << setw(pwi) << "\t"                                // 23
gsell's avatar
gsell committed
338

339 340 341 342 343 344 345 346
                    << beam.get_maxExtent()(0) << setw(pwi) << "\t"                           // 24
                    << beam.get_maxExtent()(1) << setw(pwi) << "\t"                           // 25
                    << beam.get_maxExtent()(2) << setw(pwi) << "\t"                           // 26

            // Write out Courant Snyder parameters.
                    << beam.get_rprms()(0) << setw(pwi) << "\t"                               // 27
                    << beam.get_rprms()(1) << setw(pwi) << "\t"                               // 28
                    << beam.get_rprms()(2) << setw(pwi) << "\t"                               // 29
gsell's avatar
gsell committed
347

348
            // Write out dispersion.
349 350 351 352
                    << beam.get_Dx() << setw(pwi) << "\t"                                      // 30
                    << beam.get_DDx() << setw(pwi) << "\t"                                     // 31
                    << beam.get_Dy() << setw(pwi) << "\t"                                      // 32
                    << beam.get_DDy() << setw(pwi) << "\t"                                     // 33
gsell's avatar
gsell committed
353 354


355
            // Write head/reference particle/tail field information.
356 357 358
                    << FDext[0](0) << setw(pwi) << "\t"                                         // 34 B-ref x
                    << FDext[0](1) << setw(pwi) << "\t"                                         // 35 B-ref y
                    << FDext[0](2) << setw(pwi) << "\t"                                         // 36 B-ref z
gsell's avatar
gsell committed
359

360 361 362
                    << FDext[1](0) << setw(pwi) << "\t"                                         // 37 E-ref x
                    << FDext[1](1) << setw(pwi) << "\t"                                         // 38 E-ref y
                    << FDext[1](2) << setw(pwi) << "\t"                                         // 39 E-ref z
gsell's avatar
gsell committed
363

364 365 366
                    << beam.getdE() << setw(pwi) << "\t"                                        // 40 dE energy spread
                    << beam.getdT() * 1e9 << setw(pwi) << "\t"                                        // 41 dt time step size
                    << npOutside << setw(pwi) << "\t";                                          // 42 number of particles outside n*sigma
gsell's avatar
gsell committed
367

368
        if(Ippl::getNodes() == 1 && beam.getLocalNum() > 0) {
369 370 371 372 373 374
            os_statData << beam.R[0](0) << setw(pwi) << "\t";                                   // 43 R0_x
            os_statData << beam.R[0](1) << setw(pwi) << "\t";                                   // 44 R0_y
            os_statData << beam.R[0](2) << setw(pwi) << "\t";                                   // 45 R0_z
            os_statData << beam.P[0](0) << setw(pwi) << "\t";                                   // 46 P0_x
            os_statData << beam.P[0](1) << setw(pwi) << "\t";                                   // 47 P0_y
            os_statData << beam.P[0](2) << setw(pwi) << "\t";                                   // 48 P0_z
gsell's avatar
gsell committed
375 376
        }

377

378 379 380
        for(size_t i = 0; i < losses.size(); ++ i) {
            os_statData << losses[i].second << setw(pwi) << "\t";
        }
gsell's avatar
gsell committed
381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410
        os_statData   << endl;

        for(int p = 0; p < Ippl::getNodes(); p++)
            os_lBalData << beam.getLoadBalance(p)  << setw(pwi) << "\t";
        os_lBalData << endl;

        os_statData.close();
        os_lBalData.close();
    }

    /// %Stop timer.
    IpplTimings::stopTimer(StatMarkerTimer_m);
}

void DataSink::writeStatData(EnvelopeBunch &beam, Vector_t FDext[], double sposHead, double sposRef, double sposTail) {
    /// Function steps:
    /// Start timer.
    IpplTimings::startTimer(StatMarkerTimer_m);

    /// Set width of write fields in output files.
    unsigned int pwi = 10;

    /// Calculate beam statistics and gather load balance statistics.
    beam.calcBeamParameters();
    beam.gatherLoadBalanceStatistics();

    /// Write data to files. If this is the first write to the beam statistics file, write SDDS
    /// header information.
    ofstream os_statData;
    ofstream os_lBalData;
411
    double en = beam.get_meanKineticEnergy() * 1e-6;
gsell's avatar
gsell committed
412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459

    if(Ippl::myNode() == 0) {
        if(firstWriteToStat_m) {
            os_statData.open(statFileName_m.c_str(), ios::out);
            os_statData.precision(15);
            os_statData.setf(ios::scientific, ios::floatfield);
            writeSDDSHeader(os_statData);

            os_lBalData.open(lBalFileName_m.c_str(), ios::out);
            os_lBalData.precision(15);
            os_lBalData.setf(ios::scientific, ios::floatfield);
            os_lBalData << "# " << Ippl::getNodes() << endl;

            firstWriteToStat_m = false;
        } else {
            os_statData.open(statFileName_m.c_str(), ios::app);
            os_statData.precision(15);
            os_statData.setf(ios::scientific, ios::floatfield);

            os_lBalData.open(lBalFileName_m.c_str(), ios::app);
            os_lBalData.precision(15);
            os_lBalData.setf(ios::scientific, ios::floatfield);
        }


        os_statData << beam.getT() << setw(pwi) << "\t"                                       // 1
                    << sposRef << setw(pwi) << "\t"                                           // 2

                    << beam.getTotalNum() << setw(pwi) << "\t"                                // 3
                    << beam.getTotalNum() * beam.getChargePerParticle() << setw(pwi) << "\t"  // 4
                    << en << setw(pwi) << "\t"                                                // 5

                    << beam.get_rrms()(0) << setw(pwi) << "\t"                                // 6
                    << beam.get_rrms()(1) << setw(pwi) << "\t"                                // 7
                    << beam.get_rrms()(2) << setw(pwi) << "\t"                                // 8

                    << beam.get_prms()(0) << setw(pwi) << "\t"                                // 9
                    << beam.get_prms()(1) << setw(pwi) << "\t"                                // 10
                    << beam.get_prms()(2) << setw(pwi) << "\t"                                // 11

                    << beam.get_norm_emit()(0) << setw(pwi) << "\t"                           // 12
                    << beam.get_norm_emit()(1) << setw(pwi) << "\t"                           // 13
                    << beam.get_norm_emit()(2) << setw(pwi) << "\t"                           // 14

                    << beam.get_rmean()(0)  << setw(pwi) << "\t"                              // 15
                    << beam.get_rmean()(1)  << setw(pwi) << "\t"                              // 16
                    << beam.get_rmean()(2)  << setw(pwi) << "\t"                              // 17

460 461 462
                    << beam.get_maxExtent()(0) << setw(pwi) << "\t"                           // 18
                    << beam.get_maxExtent()(1) << setw(pwi) << "\t"                           // 19
                    << beam.get_maxExtent()(2) << setw(pwi) << "\t"                           // 20
gsell's avatar
gsell committed
463

464
            // Write out Courant Snyder parameters.
gsell's avatar
gsell committed
465 466 467 468 469 470
                    << 0.0  << setw(pwi) << "\t"                                              // 21
                    << 0.0  << setw(pwi) << "\t"                                              // 22

                    << 0.0 << setw(pwi) << "\t"                                               // 23
                    << 0.0 << setw(pwi) << "\t"                                               // 24

471
            // Write out dispersion.
gsell's avatar
gsell committed
472 473 474 475 476 477
                    << beam.get_Dx() << setw(pwi) << "\t"                                     // 25
                    << beam.get_DDx() << setw(pwi) << "\t"                                    // 26
                    << beam.get_Dy() << setw(pwi) << "\t"                                     // 27
                    << beam.get_DDy() << setw(pwi) << "\t"                                    // 28


478
            // Write head/reference particle/tail field information.
479 480 481
                    << FDext[2](0) << setw(pwi) << "\t"                                       // 29 B-ref x
                    << FDext[2](1) << setw(pwi) << "\t"                                       // 30 B-ref y
                    << FDext[2](2) << setw(pwi) << "\t"                                       // 31 B-ref z
gsell's avatar
gsell committed
482

483 484 485
                    << FDext[3](0) << setw(pwi) << "\t"                                       // 32 E-ref x
                    << FDext[3](1) << setw(pwi) << "\t"                                       // 33 E-ref y
                    << FDext[3](2) << setw(pwi) << "\t"                                       // 34 E-ref z
gsell's avatar
gsell committed
486

487
                    << beam.get_dEdt() << setw(pwi) << "\t"                                   // 35 dE energy spread
488

gsell's avatar
gsell committed
489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504
                    << endl;


        for(int p = 0; p < Ippl::getNodes(); p++)
            os_lBalData << beam.getLoadBalance(p)  << setw(pwi) << "\t";
        os_lBalData << endl;

        os_statData.close();
        os_lBalData.close();
    }

    /// %Stop timer.
    IpplTimings::stopTimer(StatMarkerTimer_m);
}

void DataSink::writeSDDSHeader(ofstream &outputFile) {
505 506 507 508 509
    writeSDDSHeader(outputFile,
                    std::vector<std::pair<std::string, unsigned int> >());
}
void DataSink::writeSDDSHeader(ofstream &outputFile,
                               const std::vector<std::pair<std::string, unsigned int> > &losses) {
510 511 512 513
    OPALTimer::Timer simtimer;

    string dateStr(simtimer.date());
    string timeStr(simtimer.time());
514
    string indent("        ");
515

gsell's avatar
gsell committed
516
    outputFile << "SDDS1" << endl;
517 518 519 520 521 522 523 524 525 526 527 528
    outputFile << "&description\n"
               << indent << "text=\"Statistics data '" << OpalData::getInstance()->getInputFn() << "' " << dateStr << "" << timeStr << "\",\n"
               << indent << "contents=\"stat parameters\"\n"
               << "&end\n";
    outputFile << "&parameter\n"
               << indent << "name=processors,\n"
               << indent << "type=long,\n"
               << indent << "description=\"Number of Cores used\"\n"
               << "&end\n";
    outputFile << "&parameter\n"
               << indent << "name=revision,\n"
               << indent << "type=string,\n"
529 530 531 532 533 534 535
               << indent << "description=\"git revision of opal\"\n"
               << "&end\n";
    outputFile << "&parameter\n"
               << indent << "name=flavor,\n"
               << indent << "type=string,\n"
               << indent << "description=\"OPAL flavor that wrote file\""
               << "&end\n";
536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789
    outputFile << "&column\n"
               << indent << "name=t,\n"
               << indent << "type=double,\n"
               << indent << "units=ns,\n"
               << indent << "description=\"1 Time\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=s,\n"
               << indent << "type=double,\n"
               << indent << "units=m,\n"
               << indent << "description=\"2 Average Longitudinal Position\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=numParticles,\n"
               << indent << "type=long,\n"
               << indent << "units=1,\n"
               << indent << "description=\"3 Number of Macro Particles\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=charge,\n"
               << indent << "type=double,\n"
               << indent << "units=1,\n"
               << indent << "description=\"4 Bunch Charge\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=energy,\n"
               << indent << "type=double,\n"
               << indent << "units=MeV,\n"
               << indent << "description=\"5 Mean Energy\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=rms_x,\n"
               << indent << "type=double,\n"
               << indent << "units=m,\n"
               << indent << "description=\"6 RMS Beamsize in x\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=rms_y,\n"
               << indent << "type=double,\n"
               << indent << "units=m,\n"
               << indent << "description=\"7 RMS Beamsize in y\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=rms_s,\n"
               << indent << "type=double,\n"
               << indent << "units=m,\n"
               << indent << "description=\"8 RMS Beamsize in s\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=rms_px,\n"
               << indent << "type=double,\n"
               << indent << "units=1,\n"
               << indent << "description=\"9 RMS Momenta in x\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=rms_py,\n"
               << indent << "type=double,\n"
               << indent << "units=1,\n"
               << indent << "description=\"10 RMS Momenta in y\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=rms_ps,\n"
               << indent << "type=double,\n"
               << indent << "units=1,\n"
               << indent << "description=\"11 RMS Momenta in s\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=emit_x,\n"
               << indent << "type=double,\n"
               << indent << "units=m,\n"
               << indent << "description=\"12 Normalized Emittance x\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=emit_y,\n"
               << indent << "type=double,\n"
               << indent << "units=m,\n"
               << indent << "description=\"13 Normalized Emittance y\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=emit_s,\n"
               << indent << "type=double,\n"
               << indent << "units=m,\n"
               << indent << "description=\"14 Normalized Emittance s\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=mean_x,\n"
               << indent << "type=double,\n"
               << indent << "units=m,\n"
               << indent << "description=\"15 Mean Beam Position in x\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=mean_y,\n"
               << indent << "type=double,\n"
               << indent << "units=m,\n"
               << indent << "description=\"16 Mean Beam Position in y\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=mean_s,\n"
               << indent << "type=double,\n"
               << indent << "units=m,\n"
               << indent << "description=\"17 Mean Beam Position in s\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=ref_x,\n"
               << indent << "type=double,\n"
               << indent << "units=m,\n"
               << indent << "description=\"18 x coordinate of reference particle in lab cs\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=ref_y,\n"
               << indent << "type=double,\n"
               << indent << "units=m,\n"
               << indent << "description=\"19 y coordinate of reference particle in lab cs\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=ref_z,\n"
               << indent << "type=double,\n"
               << indent << "units=m,\n"
               << indent << "description=\"20 z coordinate of reference particle in lab cs\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=ref_px,\n"
               << indent << "type=double,\n"
               << indent << "units=1,\n"
               << indent << "description=\"21 x momentum of reference particle in lab cs\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=ref_py,\n"
               << indent << "type=double,\n"
               << indent << "units=1,\n"
               << indent << "description=\"22 y momentum of reference particle in lab cs\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=ref_pz,\n"
               << indent << "type=double,\n"
               << indent << "units=1,\n"
               << indent << "description=\"23 z momentum of reference particle in lab cs\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=max_x,\n"
               << indent << "type=double,\n"
               << indent << "units=m,\n"
               << indent << "description=\"24 Max Beamsize in x\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=max_y,\n"
               << indent << "type=double,\n"
               << indent << "units=m,\n"
               << indent << "description=\"25 Max Beamsize in y\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=max_s,\n"
               << indent << "type=double,\n"
               << indent << "units=m,\n"
               << indent << "description=\"26 Max Beamsize in s\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=xpx,\n"
               << indent << "type=double,\n"
               << indent << "units=1,\n"
               << indent << "description=\"27 Correlation xpx\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=ypy,\n"
               << indent << "type=double,\n"
               << indent << "units=1,\n"
               << indent << "description=\"28 Correlation ypyy\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=zpz,\n"
               << indent << "type=double,\n"
               << indent << "units=1,\n"
               << indent << "description=\"29 Correlation zpz\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=Dx,\n"
               << indent << "type=double,\n"
               << indent << "units=m,\n"
               << indent << "description=\"30 Dispersion in x\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=DDx,\n"
               << indent << "type=double,\n"
               << indent << "units=1,\n"
               << indent << "description=\"31 Derivative of dispersion in x\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=Dy,\n"
               << indent << "type=double,\n"
               << indent << "units=m,\n"
               << indent << "description=\"32 Dispersion in y\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=DDy,\n"
               << indent << "type=double,\n"
               << indent << "units=1,\n"
               << indent << "description=\"33 Derivative of dispersion in y\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=Bx_ref,\n"
               << indent << "type=double,\n"
               << indent << "units=T,\n"
               << indent << "description=\"34 Bx-Field component of ref particle\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=By_ref,\n"
               << indent << "type=double,\n"
               << indent << "units=T,\n"
               << indent << "description=\"35 By-Field component of ref particle\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=Bz_ref,\n"
               << indent << "type=double,\n"
               << indent << "units=T,\n"
               << indent << "description=\"36 Bz-Field component of ref particle\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=Ex_ref,\n"
               << indent << "type=double,\n"
               << indent << "units=MV/m,\n"
               << indent << "description=\"37 Ex-Field component of ref particle\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=Ey_ref,\n"
               << indent << "type=double,\n"
               << indent << "units=MV/m,\n"
               << indent << "description=\"38 Ey-Field component of ref particle\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=Ez_ref,\n"
               << indent << "type=double,\n"
               << indent << "units=MV/m,\n"
               << indent << "description=\"39 Ez-Field component of ref particle\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=dE,\n"
               << indent << "type=double,\n"
               << indent << "units=MeV,\n"
               << indent << "description=\"40 energy spread of the beam\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=dt,\n"
               << indent << "type=double,\n"
               << indent << "units=ns,\n"
               << indent << "description=\"41 time step size\"\n"
               << "&end\n";
    outputFile << "&column\n"
               << indent << "name=partsOutside,\n"
               << indent << "type=double,\n"
               << indent << "units=1,\n"
               << indent << "description=\"42 outside n*sigma of the beam\"\n"
               << "&end\n";

    unsigned int columnStart = 43;
gsell's avatar
gsell committed
790
    if(Ippl::getNodes() == 1) {
791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827
        outputFile << "&column\n"
                   << indent << "name=R0_x,\n"
                   << indent << "type=double,\n"
                   << indent << "units=m,\n"
                   << indent << "description=\"43 R0 Particle position in x\""
                   << "&end\n";
        outputFile << "&column\n"
                   << indent << "name=R0_y,\n"
                   << indent << "type=double,\n"
                   << indent << "units=m,\n"
                   << indent << "description=\"44 R0 Particle position in y\"\n"
                   << "&end\n";
        outputFile << "&column\n"
                   << indent << "name=R0_s,\n"
                   << indent << "type=double,\n"
                   << indent << "units=m,\n"
                   << indent << "description=\"45 R0 Particle position in s\"\n"
                   << "&end\n";
        outputFile << "&column\n"
                   << indent << "name=P0_x,\n"
                   << indent << "type=double,\n"
                   << indent << "units=1,\n"
                   << indent << "description=\"46 R0 Particle momentum in x\"\n"
                   << "&end\n";
        outputFile << "&column\n"
                   << indent << "name=P0_y,\n"
                   << indent << "type=double,\n"
                   << indent << "units=1,\n"
                   << indent << "description=\"47 R0 Particle momentum in y\"\n"
                   << "&end\n";
        outputFile << "&column\n"
                   << indent << "name=P0_s,\n"
                   << indent << "type=double,\n"
                   << indent << "units=1,\n"
                   << indent << "description=\"48 R0 Particle momentum in s\"\n"
                   << "&end\n";
        columnStart = 49;
gsell's avatar
gsell committed
828 829
    }

830
    for (size_t i = 0; i < losses.size(); ++ i) {
831 832 833 834 835 836
        outputFile << "&column\n"
                   << indent << "name=" << losses[i].first << ",\n"
                   << indent << "type=long,\n"
                   << indent << "units=1,\n"
                   << indent << "description=\"" << columnStart ++ << "Number of lost particles in element\"\n"
                   << "&end\n";
837
    }
838 839 840 841
    outputFile << "&data\n"
               << indent << "mode=ascii,\n"
               << indent << "no_row_counts=1\n"
               << "&end\n";
gsell's avatar
gsell committed
842

843
    outputFile << Ippl::getNodes() << endl;
844
    outputFile << PACKAGE_NAME << " " << PACKAGE_VERSION_STR << " git rev. " << Util::getGitRevision() << endl;
845 846
    outputFile << (OpalData::getInstance()->isInOPALTMode()? "opal-t":
                   (OpalData::getInstance()->isInOPALCyclMode()? "opal-cycl": "opal-env")) << endl;
gsell's avatar
gsell committed
847 848 849 850 851 852 853
}


void DataSink::writePartlossZASCII(PartBunch &beam, BoundaryGeometry &bg, string fn) {

    size_t temp = lossWrCounter_m ;

854
    string ffn = fn + convertToString(temp) + string("Z.dat");
855
    std::unique_ptr<Inform> ofp(new Inform(NULL, ffn.c_str(), Inform::OVERWRITE, 0));
gsell's avatar
gsell committed
856 857 858 859
    Inform &fid = *ofp;
    setInform(fid);
    fid.precision(6);

860
    string ftrn =  fn + string("triangle") + convertToString(temp) + string(".dat");
861
    std::unique_ptr<Inform> oftr(new Inform(NULL, ftrn.c_str(), Inform::OVERWRITE, 0));
gsell's avatar
gsell committed
862 863 864 865 866 867 868 869 870
    Inform &fidtr = *oftr;
    setInform(fidtr);
    fidtr.precision(6);

    Vector_t Geo_nr = bg.getnr();
    Vector_t Geo_hr = bg.gethr();
    Vector_t Geo_mincoords = bg.getmincoords();
    double t = beam.getT();
    double t_step = t * 1.0e9;
871 872 873
    double* prPartLossZ = new double[bg.getnr() (2)];
    double* sePartLossZ = new double[bg.getnr() (2)];
    double* fePartLossZ = new double[bg.getnr() (2)];
gsell's avatar
gsell committed
874
    fidtr << "# Time/ns" << std::setw(18) << "Triangle_ID" << std::setw(18)
adelmann's avatar
adelmann committed
875 876 877 878 879 880
          << "Xcoordinates (m)" << std::setw(18)
          << "Ycoordinates (m)" << std::setw(18)
          << "Zcoordinates (m)" << std::setw(18)
          << "Primary part. charge (C)" << std::setw(40)
          << "Field emit. part. charge (C)" << std::setw(40)
          << "Secondary emit. part. charge (C)" << std::setw(40) << endl ;
gsell's avatar
gsell committed
881
    for(int i = 0; i < Geo_nr(2) ; i++) {
882 883 884
        prPartLossZ[i] = 0;
        sePartLossZ[i] = 0;
        fePartLossZ[i] = 0;
885
        for(int j = 0; j < bg.getNumBFaces(); j++) {
gsell's avatar
gsell committed
886 887
            if(((Geo_mincoords[2] + Geo_hr(2)*i) < bg.TriBarycenters_m[j](2))
               && (bg.TriBarycenters_m[j](2) < (Geo_hr(2)*i + Geo_hr(2) + Geo_mincoords[2]))) {
888 889 890
                prPartLossZ[i] += bg.TriPrPartloss_m[j];
                sePartLossZ[i] += bg.TriSePartloss_m[j];
                fePartLossZ[i] += bg.TriFEPartloss_m[j];
gsell's avatar
gsell committed
891 892 893 894
            }

        }
    }
895
    for(int j = 0; j < bg.getNumBFaces(); j++) {
gsell's avatar
gsell committed
896
        fidtr << t_step << std::setw(18) << j << std::setw(18)// fixme: maybe gether particle loss data, i.e., do a reduce() for each triangle in each node befor write to file.
gsell's avatar
gsell committed
897 898 899
              << bg.TriBarycenters_m[j](0) << std::setw(18)
              << bg.TriBarycenters_m[j](1) << std::setw(18)
              << bg.TriBarycenters_m[j](2) <<  std::setw(40)
adelmann's avatar
adelmann committed
900 901
              << -bg.TriPrPartloss_m[j] << std::setw(40)
              << -bg.TriFEPartloss_m[j] <<  std::setw(40)
gsell's avatar
gsell committed
902 903 904
              << -bg.TriSePartloss_m[j] << endl;
    }
    fid << "# Delta_Z/m" << std::setw(18)
adelmann's avatar
adelmann committed
905 906 907 908
        << "Zcoordinates (m)" << std::setw(18)
        << "Primary part. charge (C)" << std::setw(40)
        << "Field emit. part. charge (C)" << std::setw(40)
        << "Secondary emit. part. charge (C)" << std::setw(40) << "t" << endl ;
gsell's avatar
gsell committed
909 910 911


    for(int i = 0; i < Geo_nr(2) ; i++) {
912 913 914
        double primaryPLoss = -prPartLossZ[i];
        double secondaryPLoss = -sePartLossZ[i];
        double fieldemissionPLoss = -fePartLossZ[i];
gsell's avatar
gsell committed
915 916 917 918 919
        reduce(primaryPLoss, primaryPLoss, OpAddAssign());
        reduce(secondaryPLoss, secondaryPLoss, OpAddAssign());
        reduce(fieldemissionPLoss, fieldemissionPLoss, OpAddAssign());
        fid << Geo_hr(2) << std::setw(18)
            << Geo_mincoords[2] + Geo_hr(2)*i << std::setw(18)
adelmann's avatar
adelmann committed
920 921 922
            << primaryPLoss << std::setw(40)
            << fieldemissionPLoss << std::setw(40)
            << secondaryPLoss << std::setw(40) << t << endl;
gsell's avatar
gsell committed
923 924
    }
    lossWrCounter_m++;
925 926 927
    delete[] prPartLossZ;
    delete[] sePartLossZ;
    delete[] fePartLossZ;
gsell's avatar
gsell committed
928 929 930
}

void DataSink::writeSurfaceInteraction(PartBunch &beam, long long &step, BoundaryGeometry &bg, string fn) {
kraus's avatar
kraus committed
931

932
    if (!doHDF5_m) return;
kraus's avatar
kraus committed
933

934
    h5_int64_t rc;
gsell's avatar
gsell committed
935 936 937 938 939
    /// Start timer.
    IpplTimings::startTimer(H5PartTimer_m);
    if(firstWriteH5Surface_m) {
        firstWriteH5Surface_m = false;

940 941 942 943 944
#if defined (USE_H5HUT2)
	h5_prop_t props = H5CreateFileProp ();
	MPI_Comm comm = Ippl::getComm();
	H5SetPropFileMPIOCollective (props, &comm);
	H5fileS_m = H5OpenFile (surfaceLossFileName_m.c_str(), H5_O_WRONLY, props);
gsell's avatar
gsell committed
945
        if(H5fileS_m == (h5_file_t)H5_ERR) {
946 947 948 949 950
            throw OpalException("DataSink::writeSurfaceInteraction",
                                "failed to open h5 file '" + surfaceLossFileName_m + "' for surface loss");
        }
#else
        H5fileS_m = H5OpenFile(surfaceLossFileName_m.c_str(), H5_FLUSH_STEP | H5_O_WRONLY, Ippl::getComm());
951
        if(H5fileS_m == (void*)H5_ERR) {
952 953
            throw OpalException("DataSink::writeSurfaceInteraction",
                                "failed to open h5 file '" + surfaceLossFileName_m + "' for surface loss");
gsell's avatar
gsell committed
954
        }
955
#endif
gsell's avatar
gsell committed
956 957

    }
958
    int nTot = bg.getNumBFaces();
gsell's avatar
gsell committed
959 960 961 962 963 964 965 966

    int N_mean = static_cast<int>(floor(nTot / Ippl::getNodes()));
    int N_extra = static_cast<int>(nTot - N_mean * Ippl::getNodes());
    int pc = 0;
    int count = 0;
    if(Ippl::myNode() == 0) {
        N_mean += N_extra;
    }
967 968 969
    std::unique_ptr<char[]> varray(new char[(N_mean)*sizeof(double)]);
    double *farray = reinterpret_cast<double *>(varray.get());
    h5_int64_t *larray = reinterpret_cast<h5_int64_t *>(varray.get());
gsell's avatar
gsell committed
970 971 972 973 974 975 976 977 978 979 980


    rc = H5SetStep(H5fileS_m, step);
    if(rc != H5_SUCCESS)
        ERRORMSG("H5 rc= " << rc << " in " << __FILE__ << " @ line " << __LINE__ << endl);
    rc = H5PartSetNumParticles(H5fileS_m, N_mean);
    if(rc != H5_SUCCESS)
        ERRORMSG("H5 rc= " << rc << " in " << __FILE__ << " @ line " << __LINE__ << endl);
    double    qi = beam.getChargePerParticle();
    rc = H5WriteStepAttribFloat64(H5fileS_m, "qi", &qi, 1);

981
    std::unique_ptr<double[]> tmploss(new double[nTot]);
gsell's avatar
gsell committed
982 983 984
    for(int i = 0; i < nTot; i++)
        tmploss[i] = 0.0;
    //memset( tmploss, 0.0, nTot * sizeof(double));
985
    reduce(bg.TriPrPartloss_m, bg.TriPrPartloss_m + nTot, tmploss.get(), OpAddAssign()); // may be removed if we have parallelized the geometry .
gsell's avatar
gsell committed
986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017

    for(int i = 0; i < nTot; i++) {
        if(pc == Ippl::myNode()) {
            if(count < N_mean) {
                if(pc != 0) {
                    //farray[count] =  bg.TriPrPartloss_m[Ippl::myNode()*N_mean+count];
                    if(((bg.TriBGphysicstag_m[pc * N_mean + count + N_extra] & (BGphysics::Absorption)) == (BGphysics::Absorption)) && (((bg.TriBGphysicstag_m[pc * N_mean + count + N_extra] & (BGphysics::FNEmission)) != (BGphysics::FNEmission)) && ((bg.TriBGphysicstag_m[pc * N_mean + count + N_extra] & (BGphysics::SecondaryEmission)) != (BGphysics::SecondaryEmission)))) {
                        farray[count] = 0.0;
                    } else {
                        farray[count] =  tmploss[pc * N_mean + count + N_extra];
                    }
                    count ++;
                } else {
                    if(((bg.TriBGphysicstag_m[count] & (BGphysics::Absorption)) == (BGphysics::Absorption)) && (((bg.TriBGphysicstag_m[count] & (BGphysics::FNEmission)) != (BGphysics::FNEmission)) && ((bg.TriBGphysicstag_m[count] & (BGphysics::SecondaryEmission)) != (BGphysics::SecondaryEmission)))) {
                        farray[count] = 0.0;
                    } else {
                        farray[count] =  tmploss[count];
                    }
                    count ++;

                }
            }
        }
        pc++;
        if(pc == Ippl::getNodes())
            pc = 0;
    }
    rc = H5PartWriteDataFloat64(H5fileS_m, "PrimaryLoss", farray);
    if(rc != H5_SUCCESS)
        ERRORMSG("H5 rc= " << rc << " in " << __FILE__ << " @ line " << __LINE__ << endl);
    for(int i = 0; i < nTot; i++)
        tmploss[i] = 0.0;
1018
    reduce(bg.TriSePartloss_m, bg.TriSePartloss_m + nTot, tmploss.get(), OpAddAssign()); // may be removed if we parallelize the geometry as well.
gsell's avatar
gsell committed
1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053
    count = 0;
    pc = 0;
    for(int i = 0; i < nTot; i++) {
        if(pc == Ippl::myNode()) {
            if(count < N_mean) {

                if(pc != 0) {

                    if(((bg.TriBGphysicstag_m[pc * N_mean + count + N_extra] & (BGphysics::Absorption)) == (BGphysics::Absorption)) && (((bg.TriBGphysicstag_m[pc * N_mean + count + N_extra] & (BGphysics::FNEmission)) != (BGphysics::FNEmission)) && ((bg.TriBGphysicstag_m[pc * N_mean + count + N_extra] & (BGphysics::SecondaryEmission)) != (BGphysics::SecondaryEmission)))) {
                        farray[count] = 0.0;
                    } else {
                        farray[count] =  tmploss[pc * N_mean + count + N_extra];
                    }
                    count ++;
                } else {
                    if(((bg.TriBGphysicstag_m[count] & (BGphysics::Absorption)) == (BGphysics::Absorption)) && (((bg.TriBGphysicstag_m[count] & (BGphysics::FNEmission)) != (BGphysics::FNEmission)) && ((bg.TriBGphysicstag_m[count] & (BGphysics::SecondaryEmission)) != (BGphysics::SecondaryEmission)))) {
                        farray[count] = 0.0;
                    } else {
                        farray[count] =  tmploss[count];
                    }
                    count ++;

                }
            }
        }
        pc++;
        if(pc == Ippl::getNodes())
            pc = 0;
    }
    rc = H5PartWriteDataFloat64(H5fileS_m, "SecondaryLoss", farray);
    if(rc != H5_SUCCESS)
        ERRORMSG("H5 rc= " << rc << " in " << __FILE__ << " @ line " << __LINE__ << endl);
    for(int i = 0; i < nTot; i++)
        tmploss[i] = 0.0;

1054
    reduce(bg.TriFEPartloss_m, bg.TriFEPartloss_m + nTot, tmploss.get(), OpAddAssign()); // may be removed if we parallelize the geometry as well.
gsell's avatar
gsell committed
1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128
    count = 0;
    pc = 0;
    for(int i = 0; i < nTot; i++) {
        if(pc == Ippl::myNode()) {
            if(count < N_mean) {

                if(pc != 0) {

                    if(((bg.TriBGphysicstag_m[pc * N_mean + count + N_extra] & (BGphysics::Absorption)) == (BGphysics::Absorption)) && (((bg.TriBGphysicstag_m[pc * N_mean + count + N_extra] & (BGphysics::FNEmission)) != (BGphysics::FNEmission)) && ((bg.TriBGphysicstag_m[pc * N_mean + count + N_extra] & (BGphysics::SecondaryEmission)) != (BGphysics::SecondaryEmission)))) {
                        farray[count] = 0.0;
                    } else {
                        farray[count] =  tmploss[pc * N_mean + count + N_extra];
                    }
                    count ++;
                } else {
                    if(((bg.TriBGphysicstag_m[count] & (BGphysics::Absorption)) == (BGphysics::Absorption)) && (((bg.TriBGphysicstag_m[count] & (BGphysics::FNEmission)) != (BGphysics::FNEmission)) && ((bg.TriBGphysicstag_m[count] & (BGphysics::SecondaryEmission)) != (BGphysics::SecondaryEmission)))) {
                        farray[count] = 0.0;
                    } else {
                        farray[count] =  tmploss[count];
                    }
                    count ++;

                }

            }
        }
        pc++;
        if(pc == Ippl::getNodes())
            pc = 0;
    }
    rc = H5PartWriteDataFloat64(H5fileS_m, "FNEmissionLoss", farray);
    if(rc != H5_SUCCESS)
        ERRORMSG("H5 rc= " << rc << " in " << __FILE__ << " @ line " << __LINE__ << endl);
    count = 0;
    pc = 0;
    for(int i = 0; i < nTot; i++) {
        if(pc == Ippl::myNode()) {
            if(count < N_mean) {
                if(pc != 0)
                    larray[count] =  Ippl::myNode() * N_mean + count + N_extra; // node 0 will be 0*N_mean+count+N_extra also correct.
                else
                    larray[count] = count;
                count ++;
            }
        }
        pc++;
        if(pc == Ippl::getNodes())
            pc = 0;
    }
    rc = H5PartWriteDataInt64(H5fileS_m, "TriangleID", larray);
    if(rc != H5_SUCCESS)
        ERRORMSG("H5 rc= " << rc << " in " << __FILE__ << " @ line " << __LINE__ << endl);

    /// %Stop timer.
    IpplTimings::stopTimer(H5PartTimer_m);

}

void DataSink::writeImpactStatistics(PartBunch &beam, long long &step, size_t &impact, double &sey_num,
                                     size_t numberOfFieldEmittedParticles, bool nEmissionMode, string fn) {

    double charge = 0.0;
    size_t Npart = 0;
    double Npart_d = 0.0;
    if(!nEmissionMode) {
        charge = -1.0 * beam.getCharge();
        //reduce(charge, charge, OpAddAssign());
        Npart_d = -1.0 * charge / beam.getChargePerParticle();
    } else {
        Npart = beam.getTotalNum();
    }
    if(Ippl::myNode() == 0) {
        string ffn = fn + string(".dat");

1129
        std::unique_ptr<Inform> ofp(new Inform(NULL, ffn.c_str(), Inform::APPEND, 0));
gsell's avatar
gsell committed
1130 1131 1132 1133 1134 1135 1136 1137
        Inform &fid = *ofp;
        setInform(fid);

        fid.precision(6);
        fid << setiosflags(ios::scientific);
        double t = beam.getT() * 1.0e9;
        if(!nEmissionMode) {

1138
            if(step == 0) {
1139
                fid << "#Time/ns"  << std::setw(18) << "#Geometry impacts" << std::setw(18) << "tot_sey" << std::setw(18)
gsell's avatar
gsell committed
1140 1141 1142 1143 1144 1145
                    << "TotalCharge" << std::setw(18) << "PartNum" << " numberOfFieldEmittedParticles " << endl ;
            }
            fid << t << std::setw(18) << impact << std::setw(18) << sey_num << std::setw(18) << charge
                << std::setw(18) << Npart_d << std::setw(18) << numberOfFieldEmittedParticles << endl ;
        } else {

1146
            if(step == 0) {
1147
                fid << "#Time/ns"  << std::setw(18) << "#Geometry impacts" << std::setw(18) << "tot_sey" << std::setw(18)
gsell's avatar
gsell committed
1148 1149 1150 1151 1152 1153 1154 1155 1156 1157
                    << "ParticleNumber" << " numberOfFieldEmittedParticles " << endl;
            }
            fid << t << std::setw(18) << impact << std::setw(18) << sey_num
                << std::setw(18) << double(Npart) << std::setw(18) << numberOfFieldEmittedParticles << endl ;
        }
    }
}

void DataSink::writeGeomToVtk(BoundaryGeometry &bg, string fn) {
    if(Ippl::myNode() == 0) {