From 76d3d503e24ecfca8c4a62708d1c1961bef6df03 Mon Sep 17 00:00:00 2001
From: Achim Gsell <achim.gsell@psi.ch>
Date: Wed, 8 Jan 2020 18:25:47 +0100
Subject: [PATCH] cleanup: IPPL_PM and IPPL_SHMEMPI related code removed
---
CMakeLists.txt | 1 -
ippl/src/Message/CMakeLists.txt | 2 -
ippl/src/Message/CommCreator.cpp | 57 +---
ippl/src/Message/CommCreator.h | 2 +-
ippl/src/Message/CommPM.cpp | 483 -------------------------------
ippl/src/Message/CommPM.h | 86 ------
ippl/src/Message/CommSHMEMPI.cpp | 448 ----------------------------
ippl/src/Message/CommSHMEMPI.h | 92 ------
ippl/src/Message/Message.h | 9 -
tools/mslang/CMakeLists.txt | 1 -
10 files changed, 7 insertions(+), 1174 deletions(-)
delete mode 100644 ippl/src/Message/CommPM.cpp
delete mode 100644 ippl/src/Message/CommPM.h
delete mode 100644 ippl/src/Message/CommSHMEMPI.cpp
delete mode 100644 ippl/src/Message/CommSHMEMPI.h
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 39e3a936f..b20def11b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -41,7 +41,6 @@ set (CMAKE_CXX_STANDARD 11)
set (CMAKE_CXX_STANDARD_REQUIRED ON)
set (CMAKE_CXX_EXTENSIONS OFF)
-add_definitions (-DIPPL_MPI)
add_definitions (-DIPPL_USE_XDIV_RNG)
add_definitions (-DIPPL_USE_PARTIAL_SPECIALIZATION)
add_definitions (-DNOCTAssert)
diff --git a/ippl/src/Message/CMakeLists.txt b/ippl/src/Message/CMakeLists.txt
index 4af95458b..4c37ebe53 100644
--- a/ippl/src/Message/CMakeLists.txt
+++ b/ippl/src/Message/CMakeLists.txt
@@ -10,8 +10,6 @@ set (_SRCS
set (_HDRS
CommCreator.h
CommMPI.h
- CommPM.h
- CommSHMEMPI.h
Communicate.h
CRC.h
DataTypes.h
diff --git a/ippl/src/Message/CommCreator.cpp b/ippl/src/Message/CommCreator.cpp
index 8c9cd960f..3bd237164 100644
--- a/ippl/src/Message/CommCreator.cpp
+++ b/ippl/src/Message/CommCreator.cpp
@@ -23,33 +23,17 @@
*
***************************************************************************/
-// include files
#include "Message/CommCreator.h"
#include "Message/Communicate.h"
-
-
-// include files for specific communication libraries
-
-
-#if defined(IPPL_MPI)
#include "Message/CommMPI.h"
-#endif
-
-#if defined(IPPL_SHMEMPI)
-#include "Message/CommSHMEMPI.h"
-#endif
-
-#if defined(IPPL_PM)
-#include "Message/CommPM.h"
-#endif
#include <cstring>
// static data for this file
static const char *CommLibraryNames[CommCreator::COMMLIBRARIES] =
- { "pm", "mpi", "shmempi","serial" };
+ { "mpi", "serial" };
-static const char *CommLibraryList = "pm, mpi, shmempi, or serial";
+static const char *CommLibraryList = "mpi or serial";
/////////////////////////////////////////////////////////////////////////
@@ -77,24 +61,9 @@ const char *CommCreator::getAllLibraryNames()
bool CommCreator::supported(int cm)
{
- if (cm == PM)
- {
-#ifdef IPPL_PM
- return true;
-#endif
- }
- else if (cm == MPI)
- {
-#ifdef IPPL_MPI
- return true;
-#endif
- }
- else if (cm == SHMEMPI)
- {
-#ifdef IPPL_SHMEMPI
- return true;
-#endif
- }
+ if (cm == MPI) {
+ return true;
+ }
else if (cm == SERIAL) {
return true;
}
@@ -132,23 +101,9 @@ Communicate *CommCreator::create(int cm, int& argc, char**& argv, int nodes,
// to avoid warning message
if (doinit) { }
- if (cm == PM)
+ if (cm == MPI)
{
-#ifdef IPPL_PM
- comm = new CommPM(argc, argv, nodes);
-#endif
- }
- else if (cm == MPI)
- {
-#ifdef IPPL_MPI
comm = new CommMPI(argc, argv, nodes, doinit, mpicomm);
-#endif
- }
- else if (cm == SHMEMPI)
- {
-#ifdef IPPL_SHMEMPI
- comm = new CommSHMEMPI(argc, argv, nodes);
-#endif
}
else if (cm == SERIAL)
{
diff --git a/ippl/src/Message/CommCreator.h b/ippl/src/Message/CommCreator.h
index f08cee1c5..af424d7ff 100644
--- a/ippl/src/Message/CommCreator.h
+++ b/ippl/src/Message/CommCreator.h
@@ -35,7 +35,7 @@ class CommCreator
public:
// enumeration of communication libraries
- enum { PM, MPI, SHMEMPI, SERIAL, COMMLIBRARIES };
+ enum { MPI, SERIAL, COMMLIBRARIES };
public:
// constructor and destructor
diff --git a/ippl/src/Message/CommPM.cpp b/ippl/src/Message/CommPM.cpp
deleted file mode 100644
index 1c611349c..000000000
--- a/ippl/src/Message/CommPM.cpp
+++ /dev/null
@@ -1,483 +0,0 @@
-// -*- C++ -*-
-/***************************************************************************
- *
- * The IPPL Framework
- *
- * This program was prepared by PSI.
- * All rights in the program are reserved by PSI.
- * Neither PSI nor the author(s)
- * makes any warranty, express or implied, or assumes any liability or
- * responsibility for the use of this software
- *
- * Visit www.amas.web.psi for more details
- *
- ***************************************************************************/
-
-// -*- C++ -*-
-/***************************************************************************
- *
- * The IPPL Framework
- *
- *
- * Visit http://people.web.psi.ch/adelmann/ for more details
- *
- ***************************************************************************/
-
-// include files
-#include "Message/CommPM.h"
-#include "Message/Message.h"
-#include "Utility/IpplInfo.h"
-
-
-// include mpi header file
-#include <mpi.h>
-
-// include PM header file. need to invoke PM library.
-// #include <Pm.h>
-// define interface directly, instead of using Pm.h.
-// (because KCC couldn't parse Pm.h)
-// The following definitions are written in Pm.h
-typedef void* pmCtx;
-extern "C"
-{
- int _pmGetSendBuf(pmCtx pmc, caddr_t *bufp, size_t length);
- int _pmSend(pmCtx pmc, int dst_node);
- int _pmSendDone(pmCtx pmc);
- int _pmReceive(pmCtx pmc, caddr_t *bufp, size_t *length);
- int _pmPutReceiveBuf(pmCtx pmc);
- extern pmCtx _pm_subnet[];
- extern int _pm_subnet_count;
-};
-#define PM_MTU (8192 + 32)
-
-// include score header file. need to invoke sub-network functions.
-#include <score.h>
-
-#include <cstring>
-#include <cstdlib>
-#include <unistd.h>
-
-// if an error occurs during myreceive more times than this, CommPM
-// will just exit. Make it negative to totally disable checking for a
-// maximum number of errors
-#define MAX_MPI_ERRS 500
-
-
-// static data to keep track of errors
-static int size_of_MPI_INT; /* needed for tracing */
-
-// sub-network for message passing using PM directly.
-static pmCtx pm_network;
-
-////////////////////////////////////////////////////////////////////////////
-// constructor. arguments: command-line args, and number of processes
-// to start (if < 0, start the 'default' number, i.e. the number of
-// hosts in a MPI virtual machine, the number of nodes in an O2K, etc)
-// Note: The base-class constructor does not need the argument info or
-// the number of nodes, it just by default sets the number of nodes=1
-CommPM::CommPM(int& argc , char**& argv, int procs)
- : Communicate(argc, argv, procs)
-{
-
- int i, reported, rep_host, ierror, result_len;
- MPI_Status stat;
- char *currtok, *nexttok, *execname;
-
- // a little "string magic" to strip the absolute pathname off the executable
- currtok = strstr(argv[0],"/");
- if (!currtok)
- {
- execname = strdup(argv[0]);
- }
- else
- {
- currtok++;
- nexttok = strstr(currtok,"/");
- while (nexttok)
- {
- currtok = nexttok+1;
- nexttok = strstr(currtok,"/");
- }
- execname = strdup(currtok);
- }
-
- // allocate a sub-network for message passing using PM directly.
- if (_score_alloc_subnet())
- {
- ERRORMSG("CommPM: Error with allocating sub-network.");
- }
- pm_network = _pm_subnet[_pm_subnet_count - 1];
-
- // initialize mpi
- MPI_Init(&argc, &argv);
-
- // restore original executable name without absolute path
- strcpy(argv[0],execname);
-
- // determine the number of nodes running and my node number
- MPI_Comm_size(MPI_COMM_WORLD,&TotalNodes);
- MPI_Comm_rank(MPI_COMM_WORLD,&myHost);
-
- // make sure we do not have too many processes running
- if (procs > 0 && procs < TotalNodes)
- {
- // if this is a process that is beyond what we had requested, just exit
- if (myHost >= procs)
- Ippl::abort();
- TotalNodes = procs;
- }
-
- MPI_Type_size ( MPI_INT, (MPI_Aint*)&size_of_MPI_INT );
- if (myHost == 0) // this code is run by the master process
- {
- // send a messages to each child node
- for (i = 1; i < TotalNodes; i++)
- {
- MPI_Send(&myHost, 1, MPI_INT, i, COMM_HOSTS_TAG, MPI_COMM_WORLD);
-
- }
-
- // wait for the spawned processes to report back that they're ready
- //~ int *child_ready = new int[TotalNodes];
- std::vector<int> child_ready(TotalNodes);
- for (i = 0; i < TotalNodes; child_ready[i++] = 0);
- INFOMSG("CommPM: Parent process waiting for children ..." << endl);
- reported = 1; // since the parent is already ready
- while (reported < TotalNodes)
- {
- ierror = MPI_Recv(&rep_host, 1, MPI_INT, MPI_ANY_SOURCE,
- COMM_HOSTS_TAG, MPI_COMM_WORLD, &stat);
-
- if (rep_host >= 0 && rep_host < TotalNodes && !(child_ready[rep_host]))
- {
- child_ready[rep_host] = 1;
- reported++;
- INFOMSG("CommPM: Child " << rep_host << " ready." << endl);
- }
- else
- {
- ERRORMSG("CommPM: Error with child reporting to parent. ");
- ERRORMSG("rep_host = " << rep_host);
- ERRORMSG(", child_ready[] = " << child_ready[rep_host] << endl);
- }
- }
-
- //~ delete [] child_ready;
- INFOMSG("CommPM: Initialization complete." << endl);
-
- }
- else // this is a child process; get data from pops
- {
- char host_name[MPI_MAX_PROCESSOR_NAME];
- ierror = MPI_Get_processor_name(host_name, &result_len);
- if (ierror >= 0)
- {
- INFOMSG("CommPM: Started job " << myHost << " on host `");
- INFOMSG(host_name << "'." << endl);
- }
- else
- {
- ERRORMSG("CommPM: failed" << endl);
- }
-
- // receive message from the master node
- int checknode;
- MPI_Recv(&checknode, 1, MPI_INT, 0, COMM_HOSTS_TAG, MPI_COMM_WORLD,
- &stat);
-
- if (checknode != 0)
- WARNMSG("CommPM: Child received bad message during startup." << endl);
-
- // send back an acknowledgement
- MPI_Send(&myHost, 1, MPI_INT, 0, COMM_HOSTS_TAG, MPI_COMM_WORLD);
-
- }
-
- // set up the contexts and processes arrays properly
- if (TotalNodes > 1)
- {
- vector<int> proccount;
- proccount.push_back(1);
- for (i = 1; i < TotalNodes; i++)
- {
- Contexts.push_back(1);
- Processes.push_back(proccount);
- }
- }
-
-}
-
-
-////////////////////////////////////////////////////////////////////////////
-// class destructor
-CommPM::~CommPM(void)
-{
-
- int i, dieCode = 0;
- MPI_Status stat;
-
- // on all nodes, when running in parallel, get any extra messages not
- // yet received
- if (TotalNodes > 1)
- {
- int trial, node, tag;
- Message *msg;
- for (trial = 0; trial < 50000; ++trial)
- {
- do
- {
- node = COMM_ANY_NODE;
- tag = COMM_ANY_TAG;
- msg = myreceive(node, tag, COMM_SEND_TAG);
- if (msg != 0)
- {
- WARNMSG("CommPM: Found extra message from node " << node);
- WARNMSG(", tag " << tag << ": msg = " << *msg << endl);
- }
- }
- while (msg != 0);
- }
- }
-
- // broadcast a message to all other nodes to tell them to quit
- if (myNode() == 0)
- {
- // on master node, send out messages
- for (i = 1; i < TotalNodes; i++)
- {
- MPI_Send(&dieCode, 1, MPI_INT, i, COMM_DIE_TAG, MPI_COMM_WORLD);
-
- }
- }
- else
- {
- // on client nodes, receive message
- MPI_Recv(&dieCode, 1, MPI_INT, 0, COMM_DIE_TAG, MPI_COMM_WORLD, &stat);
-
- }
-
- // release sub-network
- _score_free_subnet();
-
- MPI_Finalize();
-}
-
-
-struct PM_Message
-{
- int node;
- int tag;
-};
-
-#if 0
-////////////////////////////////////////////////////////////////////////////
-// take the data from a Message object and pack it into the current send buf.
-// each message is packed in this order:
-// tag, sending node, number of items (3-int array)
-// type of item 1 (short)
-// size of item 1, in number of elements (int)
-// item 1 data (various)
-// ...
-// type of item N (short)
-// size of item N, in number of elements (int)
-// item N data (various)
-void *CommPM::pack_message(Message *msg, int tag, int &buffsize)
-{
- // calculate size of buffer
- buffsize = find_msg_length(*msg);
-
- // allocate storage for buffer
- void *pos = (buffsize > PACKSIZE) ? makebuffer(buffsize) : mpipackbuf;
-
- // pack message data and return the necessary pointer
- fill_msg_buffer(pos, *msg, tag);
- return pos;
-}
-#endif
-
-////////////////////////////////////////////////////////////////////////////
-// send a message ... arguments are the Message itself, the
-// destination node, the 'user' tag, and the 'encoding' tag.
-// Messages should be sent via the underlying mechanism by using the
-// encoding tag (one of the COMM_ tags),
-// and should embed the information about what the user
-// tag is in the data sent between nodes. Return success.
-bool CommPM::mysend(Message *msg, int node, int tag, int etag)
-{
-
- int nerr = 0;
-
- // printf("mysend\n");
-
- // calculate size of buffer
- int length;
- length = find_msg_length(*msg) + sizeof(PM_Message);
-
- // printf("length = %d\n", length);
-
- // allocate storage for buffer
- PM_Message* msgbuf;
- int timeout_counter = 0;
- while (_pmGetSendBuf(pm_network, (caddr_t*)&msgbuf, length) == ENOBUFS)
- {
- timeout_counter++;
- if (timeout_counter > 10000000)
- {
- ERRORMSG("CommPM: _pmGetSendBuf TIMEOUT");
- timeout_counter = 0;
- nerr++;
- }
- // receive a message when the buffer couldn't be allocated.
- nerr += pickup_message();
- }
-
- // printf("buff %x is allocated\n", msgbuf);
- // pack message
- msgbuf->node = myHost;
- msgbuf->tag = tag;
- // printf("invoke fill_msg_buffer\n");
- fill_msg_buffer((void*) (&(msgbuf->tag) + 1), *msg, tag);
- // printf("fill_msg_buffer done\n");
-
- // send message
- if (_pmSend(pm_network, node))
- {
- ERRORMSG("CommPM: _pmSend Error");
- nerr++;
- }
- // receive message waiting the sending done.
- while (_pmSendDone(pm_network) == EBUSY)
- {
- nerr += pickup_message();
- }
-
- // printf("mysend done\n");
- // return the succsess of the operation
- return (nerr == 0);
-}
-
-int CommPM::pickup_message(void)
-{
- int nerr = 0;
- int length;
- PM_Message* msgbuf;
- int error;
- Message* newmsg = 0;
- // int rec_tag;
- int src_node, rec_size, rec_utag;
- void* rec_buff;
-
- // pickup message
- if (error = _pmReceive(pm_network, (caddr_t*)&msgbuf, (size_t*)&length))
- {
- // no message is received
- if (error != ENOBUFS)
- {
- ERRORMSG("CommPM: _pmReceive Error (in pickup_message)");
- nerr++;
- }
- }
- else
- {
- // a message is received, unpack it
-
- src_node = msgbuf->node;
- // rec_tag = msgbuf->tag;
- rec_size = length - sizeof(PM_Message);
- rec_buff = makebuffer(rec_size);
- // bcopy((void*) (&(msgbuf->tag) + 1), rec_buff, rec_size);
- memcpy(rec_buff, (void*) (&(msgbuf->tag) + 1), rec_size);
- _pmPutReceiveBuf(pm_network);
- newmsg = unpack_message(src_node, rec_utag, rec_buff);
- newmsg->useCommunicate(this, rec_buff);
-
- // put message in my message queue
- if (add_msg(newmsg, src_node, rec_utag))
- {
- newmsg = NULL; // reset message pointer
- }
- }
-
- // return the number of errors;
- return nerr;
-}
-
-
-////////////////////////////////////////////////////////////////////////////
-// receive a message from the given node and user tag. Return a NEW
-// Message object if a message arrives, or NULL if no message available.
-// node will be set to the node from which the message was sent.
-// tag will be set to the 'user tag' for that message.
-// etag is the 'encoding' tag, and must be one of the COMM_ tags.
-// Only message sent via the underlying mechanism with the
-// given etag are checked. When one is found, the user tag and sending
-// node are extracted from the sent data.
-// If node = COMM_ANY_NODE, checks for messages from any node.
-// If tag = COMM_ANY_TAG, checks for messages with any user tag.
-Message *CommPM::myreceive(int& node, int& tag, int etag)
-{
-
- int error;
- PM_Message* msgbuf;
- int length;
- int rec_tag, src_node, rec_size, rec_utag;
- void* rec_buff;
- Message* newmsg = 0;
-
- // pickup message
- if (error = _pmReceive(pm_network, (caddr_t*)&msgbuf, (size_t*)&length))
- {
- // no message is received
- if (error != ENOBUFS)
- {
- ERRORMSG("CommPM: _pmReceive Error (in myreceive)");
- }
- else
- {
- // no message is available
- DEBUGMSG(level2<<"CommPM: No Message Received to Match Request"<<endl);
- }
- }
- else
- {
- // a message is received, unpack it
-
- src_node = msgbuf->node;
- rec_tag = msgbuf->tag;
- rec_size = length - sizeof(PM_Message);
- rec_buff = makebuffer(rec_size);
- // bcopy((void*) (&(msgbuf->tag) + 1), rec_buff, rec_size);
- memcpy(rec_buff, (void*) (&(msgbuf->tag) + 1), rec_size);
- _pmPutReceiveBuf(pm_network);
- newmsg = unpack_message(src_node, rec_utag, rec_buff);
- newmsg->useCommunicate(this, rec_buff);
- }
-
- // Retrun the new Message, or NULL if no message available
- node = src_node;
- tag = rec_tag;
- return newmsg;
-}
-
-
-////////////////////////////////////////////////////////////////////////////
-// Synchronize all processors (everybody waits for everybody
-// else to get here before returning to calling function).
-// Uses MPI barrier for all procs
-void CommPM::mybarrier(void)
-{
-
-
- MPI_Barrier(MPI_COMM_WORLD);
-}
-
-
-////////////////////////////////////////////////////////////////////////////
-// clean up after a Message has been used (called by Message).
-void CommPM::cleanupMessage(void *d)
-{
-
-
- // need to free the allocated storage
- freebuffer(d);
-}
diff --git a/ippl/src/Message/CommPM.h b/ippl/src/Message/CommPM.h
deleted file mode 100644
index 2a03e30d1..000000000
--- a/ippl/src/Message/CommPM.h
+++ /dev/null
@@ -1,86 +0,0 @@
-// -*- C++ -*-
-/***************************************************************************
- *
- * The IPPL Framework
- *
- *
- * Visit http://people.web.psi.ch/adelmann/ for more details
- *
- ***************************************************************************/
-
-#ifndef COMM_PM_H
-#define COMM_PM_H
-
-/***************************************************************************
- * CommPM.h - PM-specific communications object for use with the
- * Ippl framework.
- * Allows user to establish id's for available nodes, establish connections,
- * and send/receive data.
- ***************************************************************************/
-
-// include files
-#include "Message/Communicate.h"
-
-
-class CommPM : public Communicate
-{
-
-public:
- // constructor and destructor
- // constructor arguments: command-line args, and number of processes
- // to start (if < 0, start the 'default' number, i.e. the number of
- // hosts in a MPI virtual machine, the number of nodes in an O2K, etc)
- CommPM(int& argc, char**& argv, int procs = (-1));
- virtual ~CommPM(void);
-
- // return the name of this item
- virtual const char *name() const
- {
- return "PM";
- }
-
- //
- // virtual routines to deal with memory management
- //
-
- // clean up after a Message has been used (called by Message).
- virtual void cleanupMessage(void *);
-
-protected:
-
- //
- // implementation-specific routines (which begin with 'my')
- // these should be provided in a derived class, and contain the
- // comm-library-specific code
- //
-
- // send a message ... arguments are the Message itself, the
- // destination node, the 'user' tag, and the 'encoding' tag.
- // Messages should be sent via the underlying mechanism by using the
- // encoding tag (one of the COMM_ tags),
- // and should embed the information about what the user
- // tag is in the data sent between nodes. Return success.
- virtual bool mysend(Message *, int node, int utag, int etag);
-
- // receive a message from the given node and user tag. Return a NEW
- // Message object if a message arrives, or NULL if no message available.
- // node will be set to the node from which the message was sent.
- // tag will be set to the 'user tag' for that message.
- // etag is the 'encoding' tag, and must be one of the COMM_ tags.
- // Only message sent via the underlying mechanism with the
- // given etag are checked. When one is found, the user tag and sending
- // node are extracted from the sent data.
- // If node = COMM_ANY_NODE, checks for messages from any node.
- // If tag = COMM_ANY_TAG, checks for messages with any user tag.
- virtual Message *myreceive(int& node, int& tag, int etag);
-
- // Synchronize all processors (everybody waits for everybody
- // else to get here before returning to calling function).
- virtual void mybarrier(void);
-
-private:
- // pickup message from PM buffer
- int pickup_message(void);
-};
-
-#endif // COMM_PM_H
diff --git a/ippl/src/Message/CommSHMEMPI.cpp b/ippl/src/Message/CommSHMEMPI.cpp
deleted file mode 100644
index 9f4a1645e..000000000
--- a/ippl/src/Message/CommSHMEMPI.cpp
+++ /dev/null
@@ -1,448 +0,0 @@
-// -*- C++ -*-
-/***************************************************************************
- *
- * The IPPL Framework
- *
- * This program was prepared by PSI.
- * All rights in the program are reserved by PSI.
- * Neither PSI nor the author(s)
- * makes any warranty, express or implied, or assumes any liability or
- * responsibility for the use of this software
- *
- * Visit www.amas.web.psi for more details
- *
- ***************************************************************************/
-
-// -*- C++ -*-
-/***************************************************************************
- *
- * The IPPL Framework
- *
- *
- * Visit http://people.web.psi.ch/adelmann/ for more details
- *
- ***************************************************************************/
-
-// include files
-#include "Message/CommSHMEMPI.h"
-#include "Message/Message.h"
-#include "Utility/IpplInfo.h"
-
-
-// include mpi header file.
-#include <mpi.h>
-
-#include <cstring>
-#include <cstdlib>
-#include <unistd.h>
-
-
-// if an error occurs during myreceive more times than this, CommSHMEMPI
-// will just exit. Make it negative to totally disable checking for a
-// maximum number of errors
-#define MAX_SHMEMPI_ERRS 500
-
-
-// static data to keep track of errors
-static int numErrors = 0;
-static int size_of_SHMEMPI_INT; /* needed for tracing */
-
-// temporary buffer used for speed
-#define PSIZE 1024*16
-#define PACKSIZE ((PSIZE)*sizeof(long))
-static long shmempipackbuf[PSIZE];
-
-
-
-////////////////////////////////////////////////////////////////////////////
-// constructor. arguments: command-line args, and number of processes
-// to start (if < 0, start the 'default' number ... for MPI, this value
-// will be ignored, since the number of nodes is determined by the args
-// to mpirun.
-CommSHMEMPI::CommSHMEMPI(int& argc , char**& argv, int procs)
- : Communicate(argc, argv, procs)
-{
-
- int i, reported, rep_host, ierror, result_len;
- MPI_Status stat;
- char *currtok, *nexttok, *execname;
-
- // a little "string magic" to strip the absolute pathname off the executable
- currtok = strstr(argv[0],"/");
- if (!currtok)
- {
- execname = strdup(argv[0]);
- }
- else
- {
- currtok++;
- nexttok = strstr(currtok,"/");
- while (nexttok)
- {
- currtok = nexttok+1;
- nexttok = strstr(currtok,"/");
- }
- execname = strdup(currtok);
- }
-
- // initialize mpi
- MPI_Init(&argc, &argv);
-
- // restore original executable name without absolute path
- strcpy(argv[0],execname);
-
- // determine the number of nodes running and my node number
- MPI_Comm_size(MPI_COMM_WORLD,&TotalNodes);
- MPI_Comm_rank(MPI_COMM_WORLD,&myHost);
-
- // make sure we do not have too many processes running
- if (procs > 0 && procs < TotalNodes)
- {
- // if this is a process that is beyond what we had requested, just exit
- if (myHost >= procs)
- Ippl::abort();
- TotalNodes = procs;
- }
-
- MPI_Type_size ( MPI_INT, &size_of_SHMEMPI_INT );
- if (myHost == 0) // this code is run by the master process
- {
- // send a messages to each child node
- for (i = 1; i < TotalNodes; i++)
- {
- MPI_Send(&myHost, 1, MPI_INT, i, COMM_HOSTS_TAG, MPI_COMM_WORLD);
-
- }
-
- // wait for the spawned processes to report back that they're ready
- std::vector<int> child_ready(TotalNodes);
- for (i = 0; i < TotalNodes; child_ready[i++] = 0);
- INFOMSG("CommSHMEMPI: Parent process waiting for children ..." << endl);
- reported = 1; // since the parent is already ready
- while (reported < TotalNodes)
- {
- ierror = MPI_Recv(&rep_host, 1, MPI_INT, MPI_ANY_SOURCE,
- COMM_HOSTS_TAG, MPI_COMM_WORLD, &stat);
-
- if (rep_host >= 0 && rep_host < TotalNodes && !(child_ready[rep_host]))
- {
- child_ready[rep_host] = 1;
- reported++;
- INFOMSG("CommSHMEMPI: Child " << rep_host << " ready." << endl);
- }
- else
- {
- ERRORMSG("CommSHMEMPI: Error with child reporting to parent. ");
- ERRORMSG("rep_host = " << rep_host);
- ERRORMSG(", child_ready[] = " << child_ready[rep_host] << endl);
- }
- }
-
- INFOMSG("CommSHMEMPI: Initialization complete." << endl);
-
- }
- else // this is a child process; get data from pops
- {
- char host_name[MPI_MAX_PROCESSOR_NAME];
- ierror = MPI_Get_processor_name(host_name, &result_len);
- if (ierror >= 0)
- {
- INFOMSG("CommSHMEMPI: Started job " << myHost << " on host `");
- INFOMSG(host_name << "'." << endl);
- }
- else
- {
- ERRORMSG("CommSHMEMPI: failed" << endl);
- }
-
- // receive message from the master node
- int checknode;
- MPI_Recv(&checknode, 1, MPI_INT, 0, COMM_HOSTS_TAG, MPI_COMM_WORLD,
- &stat);
-
- if (checknode != 0)
- WARNMSG("CommSHMEMPI: Child received bad message during startup." << endl);
-
- // send back an acknowledgement
- MPI_Send(&myHost, 1, MPI_INT, 0, COMM_HOSTS_TAG, MPI_COMM_WORLD);
-
- }
-
- // set up the contexts and processes arrays properly
- if (TotalNodes > 1)
- {
- vector<int> proccount;
- proccount.push_back(1);
- for (i = 1; i < TotalNodes; i++)
- {
- Contexts.push_back(1);
- Processes.push_back(proccount);
- }
- }
-}
-
-
-////////////////////////////////////////////////////////////////////////////
-// class destructor
-CommSHMEMPI::~CommSHMEMPI(void)
-{
-
- int i, dieCode = 0;
- MPI_Status stat;
-
- // on all nodes, when running in parallel, get any extra messages not
- // yet received
- if (TotalNodes > 1)
- {
- int trial, node, tag;
- Message *msg;
- for (trial = 0; trial < 50000; ++trial)
- {
- do
- {
- node = COMM_ANY_NODE;
- tag = COMM_ANY_TAG;
- msg = myreceive(node, tag, COMM_SEND_TAG);
- if (msg != 0 && tag != IPPL_ABORT_TAG && tag != IPPL_EXIT_TAG)
- {
- WARNMSG("CommSHMEMPI: Found extra message from node " << node);
- WARNMSG(", tag " << tag << ": msg = " << *msg << endl);
- }
- }
- while (msg != 0);
- }
- }
-
- // broadcast a message to all other nodes to tell them to quit
- if (myNode() == 0)
- {
- // on master node, send out messages
- for (i = 1; i < TotalNodes; i++)
- {
- MPI_Send(&dieCode, 1, MPI_INT, i, COMM_DIE_TAG, MPI_COMM_WORLD);
-
- }
- }
- else
- {
- // on client nodes, receive message
- MPI_Recv(&dieCode, 1, MPI_INT, 0, COMM_DIE_TAG, MPI_COMM_WORLD, &stat);
-
- }
-
- MPI_Finalize();
-}
-
-
-////////////////////////////////////////////////////////////////////////////
-// take the data from a Message object and pack it into the current send buf.
-// each message is packed in this order:
-// tag, sending node, number of items (3-int array)
-// type of item 1 (short)
-// size of item 1, in number of elements (int)
-// item 1 data (various)
-// ...
-// type of item N (short)
-// size of item N, in number of elements (int)
-// item N data (various)
-void *CommSHMEMPI::pack_message(Message *msg, int tag, int &buffsize)
-{
- // calculate size of buffer
- buffsize = find_msg_length(*msg);
-
- // allocate storage for buffer
- void *pos = (buffsize > PACKSIZE) ? makebuffer(buffsize) : shmempipackbuf;
-
- // pack message data and return the necessary pointer
- fill_msg_buffer(pos, *msg, tag, buffsize);
- return pos;
-}
-
-
-////////////////////////////////////////////////////////////////////////////
-// send a message ... arguments are the Message itself, the
-// destination node, the 'user' tag, and the 'encoding' tag.
-// Messages should be sent via the underlying mechanism by using the
-// encoding tag (one of the COMM_ tags),
-// and should embed the information about what the user
-// tag is in the data sent between nodes. Return success.
-bool CommSHMEMPI::mysend(Message *msg, int node, int tag, int etag)
-{
-
- int nitems = msg->size();
- int errstat = (-1);
- int flag = false;
- MPI_Request request;
- MPI_Status status;
-
- MPI_Status rec_status;
- int src_node, rec_node, rec_tag, rec_size, rec_utag, bufid, rec_flag = 0;
- Message* newmsg = NULL;
-
- // pack the message data into the buffer
- int size;
- void *outbuffer = pack_message(msg, tag, size);
-
- // send the message (non-blocking)
- errstat = MPI_Isend(outbuffer, size, MPI_BYTE, node, etag,
- MPI_COMM_WORLD, &request);
-
-
- while (!flag)
- {
- // get info about messages to be received
- bufid = MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD,
- &rec_flag, &rec_status);
- if ( (bufid >= 0) && (rec_flag != 0) )
- {
- // a message is available to be received
- src_node = rec_status.MPI_SOURCE;
- rec_tag = rec_status.MPI_TAG;
- MPI_Get_count(&rec_status, MPI_BYTE, &rec_size);
- if ( (rec_size >= 0) && (rec_tag >= 0) && (src_node >= 0) )
- {
- // message is a valid one, so malloc the output buffer
- void *rec_buff = makebuffer(rec_size);
-
- // blocking receive, unpack message
- MPI_Recv(rec_buff, rec_size, MPI_BYTE, src_node, rec_tag,
- MPI_COMM_WORLD, &rec_status);
-
- newmsg = unpack_message(rec_node, rec_utag, rec_buff);
-
- // tell this new Message that we were the one that created its
- // storage buffer, so that when the Messageis deleted, we can
- // be told about it in order to free the storage.
- newmsg->useCommunicate(this, rec_buff);
-
- // put message in my message queue
- if (add_msg(newmsg,rec_node,rec_utag))
- {
- newmsg = NULL; // reset message pointer
- rec_flag = 0; // reset receive flag
- }
- }
- }
-
- // check for completion of send
- MPI_Test(&request, &flag, &status);
- }
-
- // free up the send buffer
- if (size > PACKSIZE)
- freebuffer(outbuffer);
-
- // return the success of the operation
- return (errstat == 0);
-}
-
-
-////////////////////////////////////////////////////////////////////////////
-// receive a message from the given node and user tag. Return a NEW
-// Message object if a message arrives, or NULL if no message available.
-// node will be set to the node from which the message was sent.
-// tag will be set to the 'user tag' for that message.
-// etag is the 'encoding' tag, and must be one of the COMM_ tags.
-// Only message sent via the underlying mechanism with the
-// given etag are checked. When one is found, the user tag and sending
-// node are extracted from the sent data.
-// If node = COMM_ANY_NODE, checks for messages from any node.
-// If tag = COMM_ANY_TAG, checks for messages with any user tag.
-Message *CommSHMEMPI::myreceive(int& node, int& tag, int etag)
-{
-
- int bufid, size, checknode, checktag, flag = false;
- Message *newmsg = 0;
- MPI_Status stat;
-
- checknode = (node < 0 || node >= TotalNodes ? MPI_ANY_SOURCE : node);
- checktag = etag;
-
- // get info about message
- bufid = MPI_Iprobe(checknode, checktag, MPI_COMM_WORLD, &flag, &stat);
- if (bufid < 0)
- {
- // an error has occurred
- ERRORMSG("CommSHMEMPI: cannot receive msg from node " << checknode);
- ERRORMSG(", tag " << checktag << endl);
-
- if (MAX_SHMEMPI_ERRS > 0 && ++numErrors > MAX_SHMEMPI_ERRS)
- {
- ERRORMSG("Maximum number of MPI receive errors (" << numErrors);
- ERRORMSG(") exceeded. MPI is hosed!!" << endl);
- Ippl::abort();
- }
- }
-
- // if the message is actually available, see if we can get it now
- if (flag == true)
- {
- MPI_Get_count(&stat,MPI_BYTE,&size);
- if (size < 0)
- {
- ERRORMSG("CommSHMEMPI: received message has size " << size << endl);
- }
- else if ((stat.MPI_TAG != checktag) || (stat.MPI_TAG < 0))
- {
- ERRORMSG("CommSHMEMPI: received message with invalid tag ");
- ERRORMSG(stat.MPI_TAG << endl);
- }
- else if (stat.MPI_SOURCE < 0)
- {
- ERRORMSG("CommSHMEMPI: received message from invalid source ");
- ERRORMSG(stat.MPI_SOURCE << endl);
- }
- else
- {
- checknode = stat.MPI_SOURCE;
- checktag = stat.MPI_TAG;
-
- // malloc the receive buffer
- void *outbuff = makebuffer(size);
-
- // blocking receive
- MPI_Recv(outbuff, size, MPI_BYTE, checknode, checktag,
- MPI_COMM_WORLD, &stat);
-
- newmsg = unpack_message(node, tag, outbuff);
-
- // tell this new Message that we were the one that created its
- // storage buffer, so that when the Messageis deleted, we can
- // be told about it in order to free the storage.
- newmsg->useCommunicate(this, outbuff);
- numErrors = 0;
- }
- }
- else
- {
- // no message is available
- DEBUGMSG(level2 << "CommSHMEMPI: No Message Received to Match Request");
- DEBUGMSG(endl);
- }
-
- // return the new Message, or NULL if no message available
- return newmsg;
-}
-
-
-////////////////////////////////////////////////////////////////////////////
-// Synchronize all processors (everybody waits for everybody
-// else to get here before returning to calling function).
-// Uses MPI barrier for all procs
-void CommSHMEMPI::mybarrier(void)
-{
-
-
- MPI_Barrier(MPI_COMM_WORLD);
-}
-
-
-////////////////////////////////////////////////////////////////////////////
-// clean up after a Message has been used (called by Message).
-void CommSHMEMPI::cleanupMessage(void *d)
-{
-
-
- // need to free the allocated storage
- freebuffer(d);
-}
diff --git a/ippl/src/Message/CommSHMEMPI.h b/ippl/src/Message/CommSHMEMPI.h
deleted file mode 100644
index a3d95afc2..000000000
--- a/ippl/src/Message/CommSHMEMPI.h
+++ /dev/null
@@ -1,92 +0,0 @@
-// -*- C++ -*-
-/***************************************************************************
- *
- * The IPPL Framework
- *
- *
- * Visit http://people.web.psi.ch/adelmann/ for more details
- *
- ***************************************************************************/
-
-#ifndef COMM_SHMEMPI_H
-#define COMM_SHMEMPI_H
-
-/***************************************************************************
- * CommSHMEMPI.h - MPI-specific communications object for use with the
- * Ippl framework. This differs from CommMPI in that it will, for on-box
- * communication on symmetric multiprocessors, use SHMEM instead of MPI
- * for greater efficiency. It is basically "MPI + SHMEM on-box". John
- * Hall is the person to praise or blame for the name SHMEMPI.
- *
- * Allows user to establish id's for available nodes, establish connections,
- * and send/receive data.
- *
- ***************************************************************************/
-
-// include files
-#include "Message/Communicate.h"
-
-
-class CommSHMEMPI : public Communicate
-{
-
-public:
- // constructor and destructor
- // constructor arguments: command-line args, and number of processes
- // to start (if < 0, start the 'default' number ... for MPI, this value
- // will be ignored, since the number of nodes is determined by the args
- // to mpirun.
- CommSHMEMPI(int& argc, char**& argv, int procs = (-1));
- virtual ~CommSHMEMPI(void);
-
- // return the name of this item
- virtual const char *name() const
- {
- return "SHMEMPI";
- }
-
- //
- // virtual routines to deal with memory management
- //
-
- // clean up after a Message has been used (called by Message).
- virtual void cleanupMessage(void *);
-
-protected:
-
- //
- // implementation-specific routines (which begin with 'my')
- // these should be provided in a derived class, and contain the
- // comm-library-specific code
- //
-
- // send a message ... arguments are the Message itself, the
- // destination node, the 'user' tag, and the 'encoding' tag.
- // Messages should be sent via the underlying mechanism by using the
- // encoding tag (one of the COMM_ tags),
- // and should embed the information about what the user
- // tag is in the data sent between nodes. Return success.
- virtual bool mysend(Message *, int node, int utag, int etag);
-
- // receive a message from the given node and user tag. Return a NEW
- // Message object if a message arrives, or NULL if no message available.
- // node will be set to the node from which the message was sent.
- // tag will be set to the 'user tag' for that message.
- // etag is the 'encoding' tag, and must be one of the COMM_ tags.
- // Only message sent via the underlying mechanism with the
- // given etag are checked. When one is found, the user tag and sending
- // node are extracted from the sent data.
- // If node = COMM_ANY_NODE, checks for messages from any node.
- // If tag = COMM_ANY_TAG, checks for messages with any user tag.
- virtual Message *myreceive(int& node, int& tag, int etag);
-
- // Synchronize all processors (everybody waits for everybody
- // else to get here before returning to calling function).
- virtual void mybarrier(void);
-
-private:
- // take data from the given Message, and pack it into the current send buf
- void *pack_message(Message *msg, int tag, int &buffsize);
-};
-
-#endif // COMM_SHMEMPI_H
diff --git a/ippl/src/Message/Message.h b/ippl/src/Message/Message.h
index 9e663bcd6..6633ecbc6 100644
--- a/ippl/src/Message/Message.h
+++ b/ippl/src/Message/Message.h
@@ -605,15 +605,6 @@ inline void getMessage_iter(Message &m, OutputIterator o)
m.get_iter(o);
}
-
-#if ( defined(IPPL_PM) )
-#define main mpi_main
-extern "C"
-{
- int mpi_main(int, char**);
-}
-#endif // IPPL_PM
-
#include "Message/Message.hpp"
#endif // MESSAGE_H
diff --git a/tools/mslang/CMakeLists.txt b/tools/mslang/CMakeLists.txt
index 1e8fdf8c7..7d2048fb7 100644
--- a/tools/mslang/CMakeLists.txt
+++ b/tools/mslang/CMakeLists.txt
@@ -4,7 +4,6 @@ project (MSLANG)
set (MSLANG_VERSION_MAJOR 0)
set (MSLANG_VERSION_MINOR 1)
-add_definitions (-DIPPL_MPI)
add_definitions (-DIPPL_USE_XDIV_RNG)
add_definitions (-DIPPL_USE_PARTIAL_SPECIALIZATION)
add_definitions (-DNOCTAssert)
--
GitLab