IpplInfo.cpp 41 KB
Newer Older
gsell's avatar
gsell committed
1 2 3 4
// -*- C++ -*-
/***************************************************************************
 *
 * The IPPL Framework
5 6
 *
 * This program was prepared by PSI.
gsell's avatar
gsell committed
7 8 9 10 11 12 13 14 15 16 17 18 19
 * All rights in the program are reserved by PSI.
 * Neither PSI nor the author(s)
 * makes any warranty, express or implied, or assumes any liability or
 * responsibility for the use of this software
 *
 * Visit www.amas.web.psi for more details
 *
 ***************************************************************************/

// -*- C++ -*-
/***************************************************************************
 *
 * The IPPL Framework
20
 *
gsell's avatar
gsell committed
21 22 23 24 25 26 27
 *
 * Visit http://people.web.psi.ch/adelmann/ for more details
 *
 ***************************************************************************/

// include files
#include "Utility/IpplInfo.h"
28
#include "Utility/Inform.h"
gsell's avatar
gsell committed
29 30 31 32 33 34 35
#include "Utility/IpplStats.h"
#include "Utility/PAssert.h"
#include "Utility/RandomNumberGen.h"
#include "Utility/vmap.h"
#include "DataSource/DataConnectCreator.h"
#include "Message/CommCreator.h"
#include "Message/Communicate.h"
uldis_l's avatar
uldis_l committed
36

gsell's avatar
gsell committed
37 38 39 40 41 42 43 44
#include "IpplVersions.h"

#include <unistd.h>
#include <cstdio>
#include <csignal>

/////////////////////////////////////////////////////////////////////
// public static members of IpplInfo, initialized to default values
45 46 47 48 49 50 51
Communicate *IpplInfo::Comm = 0;
IpplStats  *IpplInfo::Stats = 0;
Inform *IpplInfo::Info = 0;
Inform *IpplInfo::Warn = 0;
Inform *IpplInfo::Error = 0;
Inform *IpplInfo::Debug = 0;

kraus's avatar
kraus committed
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66
void IpplInfo::instantiateGlobals() {
    if (Comm == 0)
        Comm = new Communicate();
    if (Stats == 0)
        Stats = new IpplStats();
    if (Info == 0)
        Info = new Inform("Ippl");
    if (Warn == 0)
        Warn = new Inform("Warning", std::cerr);
    if (Error == 0)
        Error = new Inform("Error", std::cerr, INFORM_ALL_NODES);
    if (Debug == 0)
        Debug = new Inform("**DEBUG**", std::cerr, INFORM_ALL_NODES);
}

kraus's avatar
kraus committed
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82
void IpplInfo::deleteGlobals() {
    delete Comm;
    delete Stats;
    delete Info;
    delete Warn;
    delete Error;
    delete Debug;

    Comm = 0;
    Stats = 0;
    Info = 0;
    Warn = 0;
    Error = 0;
    Debug = 0;
}

83
std::stack<StaticIpplInfo> IpplInfo::stashedStaticMembers;
gsell's avatar
gsell committed
84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171

// should we use the optimization of deferring guard cell fills until
// absolutely needed?  Can be changed to true by specifying the
// flag --defergcfill
bool IpplInfo::deferGuardCellFills = false;

// should we use the compression capabilities in {[Bare]Field,LField}? Can be
// changed to false by specifying the flag --nofieldcompression
bool IpplInfo::noFieldCompression = false;

// private static members of IpplInfo, initialized to default values
MPI_Comm IpplInfo::communicator_m = MPI_COMM_WORLD;
int  IpplInfo::NumCreated = 0;
bool IpplInfo::CommInitialized = false;
bool IpplInfo::PrintStats = false;
bool IpplInfo::NeedDeleteComm = false;
int  IpplInfo::MyArgc = 0;
char **IpplInfo::MyArgv = 0;
int  IpplInfo::MyNode = 0;
int  IpplInfo::TotalNodes = 1;
int  IpplInfo::NumSMPs = 1;
int* IpplInfo::SMPIDList = 0;
int* IpplInfo::SMPNodeList = 0;
bool IpplInfo::UseChecksums = false;
bool IpplInfo::Retransmit = false;
int  IpplInfo::MaxFFTNodes = 0;
int  IpplInfo::ChunkSize = 512*1024; // 512K == 64K doubles
bool IpplInfo::PerSMPParallelIO = false;
bool IpplInfo::offsetStorage = false;
bool IpplInfo::extraCompressChecks = false;

/////////////////////////////////////////////////////////////////////
// print out current state to the given output stream
std::ostream& operator<<(std::ostream& o, const IpplInfo&) {
    o << "------------------------------------------\n";
    o << "IPPL Framework Application Summary:\n";
    o << "  Running on node " << IpplInfo::myNode();
    o << ", out of " << IpplInfo::getNodes() << " total.\n";
    o << "  Number of SMPs: " << IpplInfo::getSMPs() << "\n";
    o << "  Relative SMP node: " << IpplInfo::mySMPNode();
    o << ", out of " << IpplInfo::getSMPNodes(IpplInfo::mySMP());
    o << " nodes.\n";
    o << "  Communication method: " << IpplInfo::Comm->name() << "\n";
    o << "  Disc read chunk size: " << IpplInfo::chunkSize() << " bytes.\n";
    o << "  Deferring guard cell fills? ";
    o << IpplInfo::deferGuardCellFills << "\n";
    o << "  Turning off Field compression? ";
    o << IpplInfo::noFieldCompression << "\n";
    o << "  Offsetting storage? ";
    o << IpplInfo::offsetStorage << "\n";
    o << "  Using extra compression checks in expressions? ";
    o << IpplInfo::extraCompressChecks << "\n";
    o << "  Use per-SMP parallel IO? ";
    o << IpplInfo::perSMPParallelIO() << "\n";
    o << "  Computing message CRC checksums? ";
    o << IpplInfo::useChecksums() << "\n";
    o << "  Retransmit messages on error (only if checkums on)? ";
    o << IpplInfo::retransmit() << "\n";

    o << "  Elapsed wall-clock time (in seconds): ";
    o << IpplInfo::Stats->getTime().clock_time() << "\n";
    o << "  Elapsed CPU-clock time (in seconds) : ";
    o << IpplInfo::Stats->getTime().cpu_time() << "\n";
    o << "------------------------------------------\n";
    return o;
}


/////////////////////////////////////////////////////////////////////
// Constructor 1: parse argc, argv, and create proper Communicate object
// The second argument controls whether the IPPL-specific command line
// arguments are stripped out (the default) or left in (if the setting
// is IpplInfo::KEEP).
IpplInfo::IpplInfo(int& argc, char**& argv, int removeargs, MPI_Comm mpicomm) {

    int i;			// loop variables
    int connectoption = (-1);     // for connection method option
    int retargc;			// number of args to return to caller
    char **retargv;		// arguments to return
    bool printsummary = false;	// print summary at end of constructor

    //Inform dbgmsg("IpplInfo(argc,argv)", INFORM_ALL_NODES);

    // determine whether we should strip out ippl-specific arguments, or keep
    bool stripargs = (removeargs != KEEP);

    communicator_m = mpicomm;

172 173 174 175 176 177 178 179
    if (NumCreated == 0) {
        Comm = new Communicate();
        Stats = new IpplStats();
        Info = new Inform("Ippl");
        Warn = new Inform("Warning", std::cerr);
        Error = new Inform("Error", std::cerr, INFORM_ALL_NODES);
        Debug = new Inform("**DEBUG**", std::cerr, INFORM_ALL_NODES);
    }
gsell's avatar
gsell committed
180 181 182
    // You can only specify argc, argv once; if it is done again, print a warning
    // and continue as if we had not given argc, argv.
    if ( CommInitialized ) {
adelmann's avatar
adelmann committed
183 184
      // ADA WARNMSG("Attempt to create IpplInfo with argc, argv again." << endl);
      //WARNMSG("Using previous argc,argv settings." << endl);
gsell's avatar
gsell committed
185 186 187 188 189 190 191 192 193 194 195 196
    } else {
        // dbgmsg << "Starting initialization: argc = " << argc << ", " << endl;
        // for (unsigned int dbgi=0; dbgi < argc; ++dbgi)
        //   dbgmsg << "  argv[" << dbgi << "] = '" << argv[dbgi] << "'" << endl;

        // first make a pass through the arguments, figure out whether we should
        // run in parallel, and start up the parallel environment.  After this,
        // process all the other cmdline args
        std::string commtype;
        bool startcomm = false;
        bool comminit = true;         // do comm. system's init call
        int nprocs = (-1);		// num of processes to start; -1 means default
197

gsell's avatar
gsell committed
198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232

        /*
          if no argument is given, we assume mpi as
          communication method
         */
        commtype = std::string("mpi");
        startcomm = true;

        for (i=1; i < argc; ++i) {
            if ( ( strcmp(argv[i], "--processes") == 0 ) ||
                    ( strcmp(argv[i], "-procs") == 0 ) ) {
                // The user specified how many processes to use. This may not be useful
                // for all communication methods.
                if ( (i + 1) < argc && argv[i+1][0] != '-' && atoi(argv[i+1]) > 0 )
                    nprocs = atoi(argv[++i]);
                else
                    param_error(argv[i],
                            "Please specify a positive number of processes", 0);
            } else if ( ( strcmp(argv[i], "--commlib") == 0 ) ||
                    ( strcmp(argv[i], "-comm") == 0 ) ) {
                // The user specified what kind of comm library to use
                if ( (i + 1) < argc && argv[i+1][0] != '-' ) {
                    commtype = argv[++i];
                    startcomm = true;
                } else {
                    param_error(argv[i], "Please use one of: ",
                                CommCreator::getAllLibraryNames(), 0);
                    startcomm = false;
                }

            } else if ( strcmp(argv[i], "--nocomminit") == 0 ) {
                // The user requested that we do not let the run-time system call
                // whatever initialization routine it might have (like MPI_Init).
                // This is in case another agency has already done the initialization.
                comminit = false;
233
            }
gsell's avatar
gsell committed
234 235 236 237 238 239 240 241 242 243
        }

        // create Communicate object now.
        // dbgmsg << "Setting up parallel environment ..." << endl;
        if (startcomm && nprocs != 0 && nprocs != 1) {
            // dbgmsg << "  commlibarg=" << commtype << endl;
            // dbgmsg << ", nprocs=" << nprocs << endl;
            Communicate *newcomm = CommCreator::create(commtype.c_str(),
                    argc, argv,
                    nprocs, comminit, mpicomm);
244

gsell's avatar
gsell committed
245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
            if (newcomm == 0) {
                if (CommCreator::supported(commtype.c_str()))
                    param_error("--commlib", "Could not initialize this ",
                            "communication library.", commtype.c_str());
                else if (CommCreator::known(commtype.c_str()))
                    param_error("--commlib", "This communication library is not ",
                            "available.", commtype.c_str());
                else
                    param_error("--commlib", "Please use one of: ",
                            CommCreator::getAllLibraryNames(), 0);
            } else {
                // success, we have a new comm object
                NeedDeleteComm = true;
                delete Comm;
                Comm = newcomm;

                // cache our node number and node count
                MyNode = Comm->myNode();
                TotalNodes = Comm->getNodes();
                find_smp_nodes();

                // advance the default random number generator
                IpplRandom.AdvanceSeed(Comm->myNode());

                // dbgmsg << "  Comm creation successful." << endl;
                // dbgmsg << *this << endl;
            }
        }

        // dbgmsg << "After comm init: argc = " << argc << ", " << endl;
        // for (unsigned int dbgi=0; dbgi < argc; ++dbgi)
        //   dbgmsg << "  argv[" << dbgi << "] = '" << argv[dbgi] << "'" << endl;

        // keep track of which arguments we do no use; these are returned
        retargc = 1;
        retargv = new char*[argc];
        retargv[0] = argv[0];	// we always return arg 0 (the exec. name)

        // if we're not stripping out arguments, just save all the args
        if (!stripargs)
            for (i=1; i < argc; ++i)
                retargv[retargc++] = argv[i];

288

gsell's avatar
gsell committed
289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305
        // Parse command-line options, looking for ippl options.  When found,
        // save their suggested values and use them at the end to create data, etc.
        for (i=1; i < argc; ++i) {
            if ( ( strcmp(argv[i], "--processes") == 0 ) ||
                    ( strcmp(argv[i], "-procs") == 0 ) ) {
                // handled above
                if ( (i + 1) < argc && argv[i+1][0] != '-' && atoi(argv[i+1]) > 0 )
                    ++i;

            } else if ( ( strcmp(argv[i], "--nocomminit") == 0 ) ) {
                // handled above, nothing to do here but skip the arg

            } else if ( ( strcmp(argv[i], "--summary") == 0 ) ) {
                // set flag to print out summary of Ippl library settings at the
                // end of this constructor
                printsummary = true;

306
            } else if ( ( strcmp(argv[i], "--ipplversion") == 0 ) ) {
307
                printVersion();
308 309 310 311 312 313 314 315 316 317 318 319
                std::string options = compileOptions();
                std::string header("Compile-time options: ");
                while (options.length() > 58) {
                    std::string line = options.substr(0, 58);
                    size_t n = line.find_last_of(' ');
                    INFOMSG(header << line.substr(0, n) << "\n");

                    header = std::string(22, ' ');
                    options = options.substr(n + 1);
                }
                INFOMSG(header << options << endl);
                exit(0);
gsell's avatar
gsell committed
320 321 322 323 324 325 326 327

            } else if ( ( strcmp(argv[i], "--checksums") == 0 ) ||
                    ( strcmp(argv[i], "--checksum") == 0 ) ) {
                UseChecksums = true;

            } else if ( ( strcmp(argv[i], "--retransmit") == 0 ) ) {
                Retransmit = true;

328 329
            } else if ( ( strcmp(argv[i], "--ipplversionall") == 0 ) ||
                        ( strcmp(argv[i], "-vall") == 0 ) ) {
330
                printVersion();
331 332 333 334 335 336 337 338 339 340 341 342
                std::string options = compileOptions();
                std::string header("Compile-time options: ");
                while (options.length() > 58) {
                    std::string line = options.substr(0, 58);
                    size_t n = line.find_last_of(' ');
                    INFOMSG(header << line.substr(0, n) << "\n");

                    header = std::string(22, ' ');
                    options = options.substr(n + 1);
                }
                INFOMSG(header << options << endl);
                exit(0);
gsell's avatar
gsell committed
343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407

            } else if ( ( strcmp(argv[i], "--time") == 0 ) ||
                    ( strcmp(argv[i], "-time") == 0 ) ||
                    ( strcmp(argv[i], "--statistics") == 0 ) ||
                    ( strcmp(argv[i], "-stats") == 0 ) ) {
                // The user specified that the program stats be printed at
                // the end of the program.
                PrintStats = true;

            } else if ( ( strcmp(argv[i], "--info") == 0 ) ) {
                // Set the output level for informative messages.
                if ( (i + 1) < argc && argv[i+1][0] != '-' && atoi(argv[i+1]) >= 0 )
                    Info->setOutputLevel(atoi(argv[++i]));
                else
                    param_error(argv[i],
                            "Please specify an output level from 0 to 5", 0);

            } else if ( ( strcmp(argv[i], "--warn") == 0 ) ) {
                // Set the output level for warning messages.
                if ( (i + 1) < argc && argv[i+1][0] != '-' && atoi(argv[i+1]) >= 0 )
                    Warn->setOutputLevel(atoi(argv[++i]));
                else
                    param_error(argv[i],
                            "Please specify an output level from 0 to 5", 0);

            } else if ( ( strcmp(argv[i], "--error") == 0 ) ) {
                // Set the output level for error messages.
                if ( (i + 1) < argc && argv[i+1][0] != '-' && atoi(argv[i+1]) >= 0 )
                    Error->setOutputLevel(atoi(argv[++i]));
                else
                    param_error(argv[i],
                            "Please specify an output level from 0 to 5", 0);

            } else if ( ( strcmp(argv[i], "--debug") == 0 ) ) {
                // Set the output level for debug messages.
                if ( (i + 1) < argc && argv[i+1][0] != '-' && atoi(argv[i+1]) >= 0 )
                    Debug->setOutputLevel(atoi(argv[++i]));
                else
                    param_error(argv[i],
                            "Please specify an output level from 0 to 5", 0);

            } else if ( ( strcmp(argv[i], "--connect") == 0 ) ) {
                // Set the default external connection method
                if ( (i + 1) < argc && argv[i+1][0] != '-' )
                    connectoption = ++i;
                else
                    param_error(argv[i], "Please use one of: ",
                            DataConnectCreator::getAllMethodNames(), 0);

            } else if ( ( strcmp(argv[i], "--connectnodes") == 0 ) ) {
                // Set the number of nodes that are used in connections, by default
                if ( (i + 1) < argc && argv[i+1][0] != '-' && atoi(argv[i+1]) > 0 )
                    DataConnectCreator::setDefaultNodes(atoi(argv[++i]));
                else
                    param_error(argv[i],
                            "Please specify a number of nodes for connections > 0",
                            0);

            } else if ( ( strcmp(argv[i], "--commlib") == 0 ) ||
                    ( strcmp(argv[i], "-comm") == 0 ) ) {
                // handled above
                if ( (i + 1) < argc && argv[i+1][0] != '-' )
                    ++i;

            } else if   ( strcmp(argv[i], "--profile") == 0 )  {
408
                // handled above in
gsell's avatar
gsell committed
409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424
                if ( (i + 1) < argc && argv[i+1][0] != '-' )
                    ++i;

            } else if ( ( strcmp(argv[i], "--persmppario") == 0 ) ) {
                // Turn on the ability to use per-smp parallel IO
                PerSMPParallelIO = true;

            } else if ( ( strcmp(argv[i], "--nopersmppario") == 0 ) ) {
                // Turn off the ability to use per-smp parallel IO
                PerSMPParallelIO = false;

            } else if ( ( strcmp(argv[i], "--chunksize") == 0 ) ) {
                // Set the I/O chunk size, used to limit how many items
                // are read in or written to disk at one time.
                if ( (i + 1) < argc && argv[i+1][0] != '-' && atoi(argv[i+1]) >= 0 ) {
                    ChunkSize = atoi(argv[++i]);
425
                    char units = static_cast<char>(toupper(argv[i][strlen(argv[i])-1]));
gsell's avatar
gsell committed
426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463
                    if (units == 'K')
                        ChunkSize *= 1024;
                    else if (units == 'M')
                        ChunkSize *= 1024*1024;
                    else if (units == 'G')
                        ChunkSize *= 1024*1024*1024;
                } else {
                    param_error(argv[i],
                            "Please specify a timeout value (in seconds)", 0);
                }
            } else if ( ( strcmp(argv[i], "--defergcfill") == 0 ) ) {
                // Turn on the defer guard cell fill optimization
                deferGuardCellFills = true;

            } else if ( ( strcmp(argv[i], "--offsetstorage") == 0 ) ) {
                // Turn on the offset-storage modification to LFields
                offsetStorage = true;

            } else if ( ( strcmp(argv[i], "--extracompcheck") == 0 ) ) {
                // Turn on the extra compression checks in expressions
                extraCompressChecks = true;

            } else if ( ( strcmp(argv[i], "--nofieldcompression") == 0 ) ) {
                // Turn off compression in the Field classes
                noFieldCompression = true;

            } else if ( ( strcmp(argv[i], "--directio") == 0 ) ) {
                // Turn on the use of Direct-IO, if possible
                param_error(argv[i],
                        "Direct-IO is not available in this build of IPPL", 0);
            } else if ( ( strcmp(argv[i], "--maxfftnodes") == 0 ) ) {
                // Limit the number of nodes that can participate in FFT operations
                if ( (i + 1) < argc && argv[i+1][0] != '-' && atoi(argv[i+1]) > 0 )
                    MaxFFTNodes = atoi(argv[++i]);
                else
                    param_error(argv[i],
                            "Please specify a maximum number of FFT nodes > 0", 0);

464
            } else if ( ( strcmp(argv[i], "--ipplhelp") == 0 ) ||
gsell's avatar
gsell committed
465 466 467
                    ( strcmp(argv[i], "-h") == 0 ) ||
                    ( strcmp(argv[i], "-?") == 0 ) ) {
                // print out summary of command line switches and exit
468 469 470 471
                printHelp(argv);
                INFOMSG("   --ipplversion       : Print a brief version summary.\n");
                INFOMSG("   --ipplversionall    : Print a detailed version summary.\n");
                INFOMSG("   --ipplhelp          : Display this command-line summary.\n");
gsell's avatar
gsell committed
472
                INFOMSG(endl);
473
                exit(0);
gsell's avatar
gsell committed
474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529

            } else {
                // Unknown option; just ignore it.
                if (stripargs)
                    retargv[retargc++] = argv[i];
            }
        }

        // We can get on with creating and initializing all globally-used objects.

        // Select the default connection method
        if ( connectoption >= 0 ) {
            if ( ! DataConnectCreator::setDefaultMethod(argv[connectoption]) ) {
                if (DataConnectCreator::supported(argv[connectoption]))
                    param_error(argv[connectoption - 1], "Could not initialize this ",
                            "connection.", argv[connectoption]);
                else if (DataConnectCreator::known(argv[connectoption]))
                    param_error(argv[connectoption - 1],"This connection method is not ",
                            "available.", argv[connectoption]);
                else
                    param_error(argv[connectoption - 1], "Please use one of: ",
                            DataConnectCreator::getAllMethodNames(), 0);
            }
        }

        // indicate back to the caller which arguments are left
        MyArgc = retargc;
        MyArgv = retargv;
        if (stripargs) {
            argc = retargc;
            argv = retargv;
        }

        // Inform dbgmsg("IpplInfo::IpplInfo", INFORM_ALL_NODES);
        // dbgmsg << "Created IpplInfo.  node = " << MyNode << " out of ";
        // dbgmsg << TotalNodes << ", commlib = " << Comm->name() << endl;

        // now, at end, start the timer running, and print out a summary if asked
        Stats->getTime().stop();
        Stats->getTime().clear();
        Stats->getTime().start();
    }

    // indicate we've created one more Ippl object
    CommInitialized = true;
    NumCreated++;

    // At the very end, print out a summary if requested
    if (printsummary)
        INFOMSG(*this << endl);
}


/////////////////////////////////////////////////////////////////////
// Constructor 2: default constructor.
IpplInfo::IpplInfo() {
530 531 532 533 534 535 536 537 538
    if (NumCreated == 0) {
        Comm = new Communicate();
        Stats = new IpplStats();
        Info = new Inform("Ippl");
        Warn = new Inform("Warning", std::cerr);
        Error = new Inform("Error", std::cerr, INFORM_ALL_NODES);
        Debug = new Inform("**DEBUG**", std::cerr, INFORM_ALL_NODES);
    }

gsell's avatar
gsell committed
539 540 541 542 543 544 545 546
    // just indicate we've also been created
    NumCreated++;
}


/////////////////////////////////////////////////////////////////////
// Constructor 3: copy constructor.
IpplInfo::IpplInfo(const IpplInfo&) {
547 548 549 550 551 552 553 554 555
    if (NumCreated == 0) {
        Comm = new Communicate();
        Stats = new IpplStats();
        Info = new Inform("Ippl");
        Warn = new Inform("Warning", std::cerr);
        Error = new Inform("Error", std::cerr, INFORM_ALL_NODES);
        Debug = new Inform("**DEBUG**", std::cerr, INFORM_ALL_NODES);
    }

gsell's avatar
gsell committed
556 557 558 559 560 561 562 563 564 565 566 567
    // just indicate we've also been created
    NumCreated++;
}


/////////////////////////////////////////////////////////////////////
// Destructor: need to delete comm library if this is the last IpplInfo
IpplInfo::~IpplInfo() {
    // indicate we have one less instance; if this is the last one,
    // close communication and clean up
    // Inform dbgmsg("IpplInfo::~IpplInfo", INFORM_ALL_NODES);
    // dbgmsg << "In destructor: Current NumCreated = " << NumCreated << endl;
568

gsell's avatar
gsell committed
569 570 571 572 573 574 575 576 577 578 579 580 581 582
    if ((--NumCreated) == 0) {
        // at end of program, print statistics if requested to do so
        if (PrintStats) {
            Inform statsmsg("Stats", INFORM_ALL_NODES);
            statsmsg << *this;
            printStatistics(statsmsg);
        }

        // Delete the communications object, if necessary, to shut down parallel
        // environment
        if (NeedDeleteComm) {
             // dbgmsg << "  Deleting comm object, since now NumCreated = ";
             // dbgmsg << NumCreated << endl;
             delete Comm;
kraus's avatar
kraus committed
583
             Comm = 0;
gsell's avatar
gsell committed
584 585 586 587 588 589
             NeedDeleteComm = false;
        }
        CommInitialized = false;

        // delete other dynamically-allocated static objects
        delete [] MyArgv;
590
        if (SMPIDList != 0) {
gsell's avatar
gsell committed
591
            delete [] SMPIDList;
592 593
        }
        if (SMPNodeList != 0) {
gsell's avatar
gsell committed
594
            delete [] SMPNodeList;
595
        }
gsell's avatar
gsell committed
596
        delete Stats;
597 598 599 600 601 602

        MyArgv = 0;
        SMPIDList = 0;
        SMPNodeList = 0;
        Stats = 0;
    }
gsell's avatar
gsell committed
603 604 605 606 607 608 609 610 611 612
}


/////////////////////////////////////////////////////////////////////
// equal operator
IpplInfo& IpplInfo::operator=(const IpplInfo&) {
    // nothing to do, we don't even need to indicate we've made another
    return *this;
}

613
void IpplInfo::abort(const char *msg) {
gsell's avatar
gsell committed
614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632
    // print out message, if one was provided
    if (msg != 0) {
        ERRORMSG(msg << endl);
    }

    // print out final stats, if necessary
    if (PrintStats) {
        Inform statsmsg("Stats", INFORM_ALL_NODES);
        statsmsg << IpplInfo();
        printStatistics(statsmsg);
    }

    // delete communication object, if necessary
    if (NeedDeleteComm) {
        NeedDeleteComm = false;
        delete Comm;
        Comm = 0;
    }

adelmann's avatar
adelmann committed
633
    // that's it, folks this error will be propperly catched in the main
634
    throw std::runtime_error("Error form IpplInfo::abort");
gsell's avatar
gsell committed
635 636 637 638 639 640 641 642 643
}


/////////////////////////////////////////////////////////////////////
// Signal to ALL the nodes that we should exit or abort.  If we abort,
// a core file will be produced.  If we exit, no core file will be made.
// The node which calls abortAllNodes will print out the given message;
// the other nodes will print out that they are aborting due to a message
// from this node.
644
void IpplInfo::abortAllNodes(const char *msg) {
gsell's avatar
gsell committed
645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660
    // print out message, if one was provided
    if (msg != 0) {
        ERRORMSG(msg << endl);
    }

    // print out final stats, if necessary
    if (PrintStats) {
        Inform statsmsg("Stats", INFORM_ALL_NODES);
        statsmsg << IpplInfo();
        printStatistics(statsmsg);
    }

    // broadcast out the kill message, if necessary
    if (getNodes() > 1)
        Comm->broadcast_others(new Message, IPPL_ABORT_TAG);

661
    throw std::runtime_error("Error form IpplInfo::abortAllNodes");
Adelmann Andreas's avatar
Adelmann Andreas committed
662

gsell's avatar
gsell committed
663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732
}

/////////////////////////////////////////////////////////////////////
// getNodes: return the number of 'Nodes' in use for the computation
int IpplInfo::getNodes() {
    return TotalNodes;
}


/////////////////////////////////////////////////////////////////////
// getContexts: return the number of 'Contexts' for the given node
int IpplInfo::getContexts(const int n) {
    return Comm->getContexts(n);
}


/////////////////////////////////////////////////////////////////////
// getProcesses: return the number of 'Processes' for the given Node and Context
int IpplInfo::getProcesses(const int n, const int c) {
    return Comm->getProcesses(n, c);
}


/////////////////////////////////////////////////////////////////////
// myNode: return which Node we are running on right now
int IpplInfo::myNode() {
    return MyNode;
}


/////////////////////////////////////////////////////////////////////
// getSMPs: return number of SMP's (each of which may be running
// several processes)
int IpplInfo::getSMPs() {
    return NumSMPs;
}


/////////////////////////////////////////////////////////////////////
// getSMPNodes: return number of nodes on the SMP with the given index
int IpplInfo::getSMPNodes(int smpindx) {
    int num = 0;
    if (SMPIDList == 0) {
        num = 1;
    } else {
        for (int i=0; i < TotalNodes; ++i)
            if (SMPIDList[i] == smpindx)
                num++;
    }
    return num;
}


/////////////////////////////////////////////////////////////////////
// mySMP: return ID of my SMP (numbered 0 ... getSMPs() - 1)
int IpplInfo::mySMP() {
    return (SMPIDList != 0 ? SMPIDList[MyNode] : 0);
}


/////////////////////////////////////////////////////////////////////
// mySMPNode: return relative node number within the nodes on our SMP
int IpplInfo::mySMPNode() {
    return (SMPNodeList != 0 ? SMPNodeList[MyNode] : 0);
}


/////////////////////////////////////////////////////////////////////
// printVersion: print out a version summary.  If the argument is true,
// print out a detailed listing, otherwise a summary.
733
void IpplInfo::printVersion(void) {
gsell's avatar
gsell committed
734 735 736 737 738 739 740
    INFOMSG("IPPL Framework version " << version() << endl);
    INFOMSG("Last build date: " << compileDate() << " by user ");
    INFOMSG(compileUser() << endl);
    INFOMSG("Built for machine: " << compileMachine() << endl);
}


741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779
void IpplInfo::printHelp(char** argv) {
    INFOMSG("Usage: " << argv[0] << " [<option> <option> ...]\n");
    INFOMSG("       The possible values for <option> are:\n");
    INFOMSG("   --summary           : Print IPPL lib summary at start.\n");
    INFOMSG("   --processes <n>     : Number of parallel nodes to use.\n");
    INFOMSG("   --commlib <x>       : Selects a parallel comm. library.\n");
    INFOMSG("                         <x> = ");
    INFOMSG(CommCreator::getAllLibraryNames() << "\n");
    INFOMSG("   --nocomminit        : IPPL does not do communication\n");
    INFOMSG("                         initialization, assume already done.\n");
    INFOMSG("   --connect <x>       : Select external connection method.\n");
    INFOMSG("                         <x> = ");
    INFOMSG(DataConnectCreator::getAllMethodNames() << "\n");
    INFOMSG("   --time              : Show total time used in execution.\n");
    INFOMSG("   --notime            : Do not show timing info (default).\n");
    INFOMSG("   --info <n>          : Set info message level.  0 = off.\n");
    INFOMSG("   --warn <n>          : Set warning message level.  0 = off.\n");
    INFOMSG("   --error <n>         : Set error message level.  0 = off.\n");
    INFOMSG("   --debug <n>         : Set debug message level.  0 = off.\n");
    /*#ifdef PROFILING_ON
      INFOMSG("   --profile <gr>  : Enable profiling for groups (e.g., M+P+io) \n");
      INFOMSG("             M - Message, P - Pete, V - Viz, A - Assign, I - IO\n");
      INFOMSG("             F - Field, L - Layout, S - Sparse, D - Domainmap \n");
      INFOMSG("             Ut - Utility, R - Region, Ff - FFT \n");
      INFOMSG("             U - User, 1 - User1, 2 - User2, 3 - User3, 4 - User4\n");

      #endif*/ //PROFILING_ON
    INFOMSG("   --defergcfill       : Turn on deferred guard cell fills.\n");
    INFOMSG("   --nofieldcompression: Turn off compression in the Field classes.\n");
    INFOMSG("   --offsetstorage     : Turn on random LField storage offsets.\n");
    INFOMSG("   --extracompcheck    : Turn on extra compression checks in evaluator.\n");
    INFOMSG("   --checksums         : Turn on CRC checksums for messages.\n");
    INFOMSG("   --retransmit        : Resent messages if a CRC error occurs.\n");
    INFOMSG("   --maxfftnodes <n>   : Limit the nodes that work on FFT's.\n");
    INFOMSG("   --chunksize <n>     : Set I/O chunk size.  Can end w/K,M,G.\n");
    INFOMSG("   --persmppario       : Enable on-SMP parallel IO option.\n");
    INFOMSG("   --nopersmppario     : Disable on-SMP parallel IO option (default).\n");
}

gsell's avatar
gsell committed
780 781
/////////////////////////////////////////////////////////////////////
// here: as in stop in IpplInfo::here (in the debugger)
782
void IpplInfo::here()
gsell's avatar
gsell committed
783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860
{
}

/////////////////////////////////////////////////////////////////////
// print out statistics to the given Inform stream
void IpplInfo::printStatistics(Inform &o) { Stats->print(o); }


/////////////////////////////////////////////////////////////////////
// version: return the name of this version of Ippl, as a string
// (from Versions.h)
const char *IpplInfo::version() {
    return ippl_version_name;
}



/////////////////////////////////////////////////////////////////////
// compileArch: return the architecture on which this library was built
// (from IpplVersions.h)
const char *IpplInfo::compileArch() {
    return ippl_compile_arch;
}


/////////////////////////////////////////////////////////////////////
// compileDate: return the date on which this library was prepared for
// compilation (from IpplVersions.h)
const char *IpplInfo::compileDate() {
    return ippl_compile_date;
}


/////////////////////////////////////////////////////////////////////
// compileLine: return the compiler command used to compile each source file
// (from IpplVersions.h)
const char *IpplInfo::compileLine() {
    return ippl_compile_line;
}


/////////////////////////////////////////////////////////////////////
// compileMachine: return the machine on which this library was
// compiled (from IpplVersions.h)
const char *IpplInfo::compileMachine() {
    return ippl_compile_machine;
}


/////////////////////////////////////////////////////////////////////
// compileOptions: return the option list used to compile this library
// (from IpplVersions.h)
const char *IpplInfo::compileOptions() {
    return ippl_compile_options;
}


/////////////////////////////////////////////////////////////////////
// compileUser: return the username of the user who compiled this
// library (from IpplVersions.h)
const char *IpplInfo::compileUser() {
    return ippl_compile_user;
}


/////////////////////////////////////////////////////////////////////
// param_error: print out an error message when an illegal cmd-line
// parameter is encountered.
// Arguments are: parameter, error message, bad value (if any)
void IpplInfo::param_error(const char *param, const char *msg,
        const char *bad) {
    if ( param != 0 )
        ERRORMSG(param << " ");
    if ( bad != 0 )
        ERRORMSG(bad << " ");
    if ( msg != 0 )
        ERRORMSG(": " << msg);
    ERRORMSG(endl);
861
    IpplInfo::abort(0);
gsell's avatar
gsell committed
862 863 864 865 866 867 868 869 870 871 872 873 874
}

void IpplInfo::param_error(const char *param, const char *msg1,
        const char *msg2, const char *bad) {
    if ( param != 0 )
        ERRORMSG(param << " ");
    if ( bad != 0 )
        ERRORMSG(bad << " ");
    if ( msg1 != 0 )
        ERRORMSG(": " << msg1);
    if ( msg2 != 0 )
        ERRORMSG(msg2);
    ERRORMSG(endl);
875
    IpplInfo::abort(0);
gsell's avatar
gsell committed
876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986
}


/////////////////////////////////////////////////////////////////////
// find out how many SMP's there are, and which processor we are on
// our local SMP (e.g., if there are two SMP's with 4 nodes each,
// the process will have a node number from 0 ... 7, and an SMP node
// number from 0 ... 3
void IpplInfo::find_smp_nodes() {
    // Inform dbgmsg("IpplInfo::find_smp_nodes", INFORM_ALL_NODES);

    // create a tag for use in sending info to/from other nodes
    int tag = Comm->next_tag(IPPL_MAKE_HOST_MAP_TAG, IPPL_TAG_CYCLE);

    // create arrays to store the Node -> SMP mapping, and the relative
    // SMP node number
    if (SMPIDList != 0)
        delete [] SMPIDList;
    if (SMPNodeList != 0)
        delete [] SMPNodeList;
    SMPIDList   = new int[TotalNodes];
    SMPNodeList = new int[TotalNodes];

    // obtain the hostname and processor ID to send out
    char name[1024];
    if (gethostname(name, 1023) != 0) {
        ERRORMSG("Could not get hostname ... using localhost." << endl);
        strcpy(name, "localhost");
    }
    std::string NodeName(name,strlen(name));
    // dbgmsg << "My hostname is " << NodeName << endl;

    // all other nodes send their hostname to node 0; node 0 gets the names,
    // maps pnode ID's -> SMP ID's, then broadcasts all the necessary info to
    // all other nodes
    if (MyNode != 0) {
        // other nodes send their node name to node 0
        Message *msg = new Message;
        ::putMessage(*msg,NodeName);
        // dbgmsg << "Sending my name to node 0." << endl;
        Comm->send(msg, 0, tag);

        // receive back the SMPIDList mapping
        int node = 0;
        msg = Comm->receive_block(node, tag);
        PInsist(msg != 0 && node == 0,
                "SPMDList map not received from master in IpplInfo::find_smp_nodes!!");
        ::getMessage_iter(*msg, SMPIDList);
        ::getMessage_iter(*msg, SMPNodeList);
        delete msg;
    }
    else {
        // collect node names from everyone else, and then retransmit the collected
        // list.
        SMPIDList[0] = 0;
        vmap<std::string,int> smpMap;
        vmap<std::string,int>::iterator smpiter;
        smpMap.insert(vmap<std::string,int>::value_type(NodeName, 0));
        unsigned int unreceived = TotalNodes - 1;
        while (unreceived-- > 0) {
            // get the hostname from the remote node
            int node = COMM_ANY_NODE;
            Message *msg = Comm->receive_block(node, tag);
            PInsist(msg != 0,
                    "Hostname not received by master in IpplInfo::find_smp_nodes!!");
            std::string nodename;
            ::getMessage(*msg,nodename);
            delete msg;
            // dbgmsg <<"Received name '"<< nodename <<"' from node "<< node<<endl;

            // put it in the mapping from hostname -> SMP ID, if necessary
            smpiter = smpMap.find(nodename);
            if (smpiter == smpMap.end())
                smpMap.insert(vmap<std::string,int>::value_type(nodename,smpMap.size()));

            // from the hostname, get the SMP ID number and store it in SMPIDList
            SMPIDList[node] = smpMap[nodename];
        }

        // convert from SMPID mapping -> relative node number
        for (int smpindx = 0; (unsigned int) smpindx < smpMap.size(); ++smpindx) {
            int smpnodes = 0;
            for (int n=0; n < TotalNodes; ++n) {
                if (SMPIDList[n] == smpindx)
                    SMPNodeList[n] = smpnodes++;
            }
        }

        // broadcast SMP info to other nodes
        if (TotalNodes > 1) {
            Message *msg = new Message;
            ::putMessage(*msg, SMPIDList, SMPIDList + TotalNodes);
            ::putMessage(*msg, SMPNodeList, SMPNodeList + TotalNodes);
            Comm->broadcast_others(msg, tag);
        }
    }

    // compute number of SMP's ... necessary for all but node 0, but we'll do
    // it for all
    NumSMPs = 0;
    for (int ns=0; ns < TotalNodes; ++ns)
        if (SMPNodeList[ns] == 0)
            NumSMPs++;

    // dbgmsg << "Results of SMP mapping: NumSMPs = " << NumSMPs << endl;
    // for (unsigned int n=0; n < TotalNodes; ++n) {
    //   dbgmsg << "  n=" << n << ", SMPID=" << SMPIDList[n] << ", SMPNode=";
    //   dbgmsg << SMPNodeList[n] << endl;
    // }
}

987
void IpplInfo::stash() {
988
    PAssert_EQ(stashedStaticMembers.size(), 0);
989 990 991 992 993

    StaticIpplInfo obj;

    obj.Comm =                Comm;
    obj.Stats =               Stats;
kraus's avatar
kraus committed
994 995 996 997
    obj.Info =                Info;
    obj.Warn =                Warn;
    obj.Error =               Error;
    obj.Debug =               Debug;
998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
    obj.deferGuardCellFills = deferGuardCellFills;
    obj.noFieldCompression =  noFieldCompression;
    obj.offsetStorage =       offsetStorage;
    obj.extraCompressChecks = extraCompressChecks;
    obj.communicator_m =      communicator_m;
    obj.NumCreated =          NumCreated;
    obj.CommInitialized =     CommInitialized;
    obj.PrintStats =          PrintStats;
    obj.NeedDeleteComm =      NeedDeleteComm;
    obj.UseChecksums =        UseChecksums;
    obj.Retransmit =          Retransmit;
    obj.MyArgc =              MyArgc;
    obj.MyArgv =              MyArgv;
    obj.MyNode =              MyNode;
    obj.TotalNodes =          TotalNodes;
    obj.NumSMPs =             NumSMPs;
    obj.SMPIDList =           SMPIDList;
    obj.SMPNodeList =         SMPNodeList;
    obj.MaxFFTNodes =         MaxFFTNodes;
    obj.ChunkSize =           ChunkSize;
    obj.PerSMPParallelIO =    PerSMPParallelIO;

    stashedStaticMembers.push(obj);

kraus's avatar
kraus committed
1022 1023
    Comm = 0;
    Stats = 0;
kraus's avatar
kraus committed
1024 1025
    Info = 0;
    Warn = 0;
kraus's avatar
kraus committed
1026 1027 1028
    Error = 0;
    Debug = 0;

1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
    deferGuardCellFills = false;
    noFieldCompression = false;
    offsetStorage = false;
    extraCompressChecks = false;
    communicator_m = MPI_COMM_WORLD;
    NumCreated = 0;
    CommInitialized = false;
    PrintStats = false;
    NeedDeleteComm = false;
    UseChecksums = false;
    Retransmit = false;
    MyArgc = 0;
    MyArgv = 0;
    MyNode = 0;
    TotalNodes = 1;
    NumSMPs = 1;
    SMPIDList = 0;
    SMPNodeList = 0;
    MaxFFTNodes = 0;
    ChunkSize = 512*1024; // 512K == 64K doubles
    PerSMPParallelIO = false;
}

void IpplInfo::pop() {
1053
    PAssert_EQ(stashedStaticMembers.size(), 1);
1054 1055 1056 1057 1058

    StaticIpplInfo obj = stashedStaticMembers.top();
    stashedStaticMembers.pop();
    // Delete the communications object, if necessary, to shut down parallel
    // environment
kraus's avatar
kraus committed
1059
    // Comm is deleted in destructor
1060 1061 1062
    delete [] MyArgv;
    delete [] SMPIDList;
    delete [] SMPNodeList;
kraus's avatar
kraus committed
1063 1064 1065 1066
    delete Info;
    delete Warn;
    delete Error;
    delete Debug;
1067 1068 1069 1070
    delete Stats;

    Comm =                obj.Comm;
    Stats =               obj.Stats;
kraus's avatar
kraus committed
1071 1072 1073 1074
    Info =                obj.Info;
    Warn =                obj.Warn;
    Error =               obj.Error;
    Debug =               obj.Debug;
1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096
    deferGuardCellFills = obj.deferGuardCellFills;
    noFieldCompression =  obj.noFieldCompression;
    offsetStorage =       obj.offsetStorage;
    extraCompressChecks = obj.extraCompressChecks;
    communicator_m =      obj.communicator_m;
    NumCreated =          obj.NumCreated;
    CommInitialized =     obj.CommInitialized;
    PrintStats =          obj.PrintStats;
    NeedDeleteComm =      obj.NeedDeleteComm;
    UseChecksums =        obj.UseChecksums;
    Retransmit =          obj.Retransmit;
    MyArgc =              obj.MyArgc;
    MyArgv =              obj.MyArgv;
    MyNode =              obj.MyNode;
    TotalNodes =          obj.TotalNodes;
    NumSMPs =             obj.NumSMPs;
    SMPIDList =           obj.SMPIDList;
    SMPNodeList =         obj.SMPNodeList;
    MaxFFTNodes =         obj.MaxFFTNodes;
    ChunkSize =           obj.ChunkSize;
    PerSMPParallelIO =    obj.PerSMPParallelIO;
}