Nektar++
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Public Member Functions | Static Public Member Functions | Static Public Attributes | Protected Member Functions | Private Member Functions | Private Attributes | List of all members
Nektar::LibUtilities::CommMpi Class Reference

A global linear system. More...

#include <CommMpi.h>

Inheritance diagram for Nektar::LibUtilities::CommMpi:
Inheritance graph
[legend]
Collaboration diagram for Nektar::LibUtilities::CommMpi:
Collaboration graph
[legend]

Public Member Functions

 CommMpi (int narg, char *arg[])
 
virtual ~CommMpi ()
 
MPI_Comm GetComm ()
 
- Public Member Functions inherited from Nektar::LibUtilities::Comm
 Comm (int narg, char *arg[])
 
virtual ~Comm ()
 
void Finalise ()
 
int GetSize ()
 Returns number of processes. More...
 
int GetRank ()
 
const std::string & GetType () const
 
void Block ()
 Block execution until all processes reach this point. More...
 
NekDouble Wtime ()
 Return the time in seconds. More...
 
template<class T >
void Send (int pProc, T &pData)
 
template<class T >
void Recv (int pProc, T &pData)
 
template<class T >
void SendRecv (int pSendProc, T &pSendData, int pRecvProc, T &pRecvData)
 
template<class T >
void SendRecvReplace (int pSendProc, int pRecvProc, T &pData)
 
template<class T >
void AllReduce (T &pData, enum ReduceOperator pOp)
 
template<class T >
void AlltoAll (T &pSendData, T &pRecvData)
 
template<class T >
void AlltoAllv (Array< OneD, T > &pSendData, Array< OneD, int > &pSendDataSizeMap, Array< OneD, int > &pSendDataOffsetMap, Array< OneD, T > &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)
 
template<class T >
void Bcast (T &data, int rootProc)
 
template<class T >
void Exscan (T &pData, const enum ReduceOperator pOp, T &ans)
 
template<class T >
Gather (const int rootProc, T &val)
 
template<class T >
Scatter (const int rootProc, T &pData)
 
CommSharedPtr CommCreateIf (int flag)
 If the flag is non-zero create a new communicator. More...
 
void SplitComm (int pRows, int pColumns)
 Splits this communicator into a grid of size pRows*pColumns and creates row and column communicators. By default the communicator is a single row. More...
 
CommSharedPtr GetRowComm ()
 Retrieve the row communicator to which this process belongs. More...
 
CommSharedPtr GetColumnComm ()
 Retrieve the column communicator to which this process belongs. More...
 
bool TreatAsRankZero (void)
 
bool RemoveExistingFiles (void)
 

Static Public Member Functions

static CommSharedPtr create (int narg, char *arg[])
 Creates an instance of this class. More...
 

Static Public Attributes

static std::string className
 Name of class. More...
 

Protected Member Functions

virtual void v_Finalise ()
 
virtual int v_GetRank ()
 
virtual void v_Block ()
 
virtual double v_Wtime ()
 
virtual bool v_TreatAsRankZero (void)
 
virtual void v_Send (void *buf, int count, CommDataType dt, int dest)
 
virtual void v_Recv (void *buf, int count, CommDataType dt, int source)
 
virtual void v_SendRecv (void *sendbuf, int sendcount, CommDataType sendtype, int dest, void *recvbuf, int recvcount, CommDataType recvtype, int source)
 
virtual void v_SendRecvReplace (void *buf, int count, CommDataType dt, int pSendProc, int pRecvProc)
 
virtual void v_AllReduce (void *buf, int count, CommDataType dt, enum ReduceOperator pOp)
 
virtual void v_AlltoAll (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype)
 
virtual void v_AlltoAllv (void *sendbuf, int sendcounts[], int sensdispls[], CommDataType sendtype, void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype)
 
virtual void v_Bcast (void *buffer, int count, CommDataType dt, int root)
 
virtual void v_Exscan (Array< OneD, unsigned long long > &pData, const enum ReduceOperator pOp, Array< OneD, unsigned long long > &ans)
 
virtual void v_Gather (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype, int root)
 
virtual void v_Scatter (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype, int root)
 
virtual void v_SplitComm (int pRows, int pColumns)
 
virtual CommSharedPtr v_CommCreateIf (int flag)
 
- Protected Member Functions inherited from Nektar::LibUtilities::Comm
 Comm ()
 
virtual bool v_RemoveExistingFiles (void)
 

Private Member Functions

 CommMpi (MPI_Comm pComm)
 

Private Attributes

MPI_Comm m_comm
 
int m_rank
 

Additional Inherited Members

- Protected Attributes inherited from Nektar::LibUtilities::Comm
int m_size
 Number of processes. More...
 
std::string m_type
 Type of communication. More...
 
CommSharedPtr m_commRow
 Row communicator. More...
 
CommSharedPtr m_commColumn
 Column communicator. More...
 

Detailed Description

A global linear system.

Definition at line 61 of file CommMpi.h.

Constructor & Destructor Documentation

Nektar::LibUtilities::CommMpi::CommMpi ( int  narg,
char *  arg[] 
)

Definition at line 53 of file CommMpi.cpp.

References ASSERTL0, m_comm, m_rank, Nektar::LibUtilities::Comm::m_size, and Nektar::LibUtilities::Comm::m_type.

Referenced by v_CommCreateIf(), and v_SplitComm().

53  : Comm(narg, arg)
54 {
55  int init = 0;
56  MPI_Initialized(&init);
57  ASSERTL0(!init, "MPI has already been initialised.");
58 
59  int retval = MPI_Init(&narg, &arg);
60  if (retval != MPI_SUCCESS)
61  {
62  ASSERTL0(false, "Failed to initialise MPI");
63  }
64 
65  m_comm = MPI_COMM_WORLD;
66  MPI_Comm_size(m_comm, &m_size);
67  MPI_Comm_rank(m_comm, &m_rank);
68 
69 #ifdef NEKTAR_USING_PETSC
70  PetscInitializeNoArguments();
71 #endif
72 
73  m_type = "Parallel MPI";
74 }
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:198
std::string m_type
Type of communication.
Definition: Comm.h:131
int m_size
Number of processes.
Definition: Comm.h:130
Nektar::LibUtilities::CommMpi::~CommMpi ( )
virtual

Definition at line 91 of file CommMpi.cpp.

92 {
93 }
Nektar::LibUtilities::CommMpi::CommMpi ( MPI_Comm  pComm)
private

Definition at line 79 of file CommMpi.cpp.

References m_comm, m_rank, Nektar::LibUtilities::Comm::m_size, and Nektar::LibUtilities::Comm::m_type.

79  : Comm()
80 {
81  m_comm = pComm;
82  MPI_Comm_size(m_comm, &m_size);
83  MPI_Comm_rank(m_comm, &m_rank);
84 
85  m_type = "Parallel MPI";
86 }
std::string m_type
Type of communication.
Definition: Comm.h:131
int m_size
Number of processes.
Definition: Comm.h:130

Member Function Documentation

static CommSharedPtr Nektar::LibUtilities::CommMpi::create ( int  narg,
char *  arg[] 
)
inlinestatic

Creates an instance of this class.

Definition at line 65 of file CommMpi.h.

References Nektar::MemoryManager< DataType >::AllocateSharedPtr().

66  {
68  }
static boost::shared_ptr< DataType > AllocateSharedPtr()
Allocate a shared pointer from the memory pool.
MPI_Comm Nektar::LibUtilities::CommMpi::GetComm ( )

Definition at line 98 of file CommMpi.cpp.

References m_comm.

Referenced by Nektar::MultiRegions::GlobalLinSysPETSc::GlobalLinSysPETSc().

99 {
100  return m_comm;
101 }
void Nektar::LibUtilities::CommMpi::v_AllReduce ( void buf,
int  count,
CommDataType  dt,
enum ReduceOperator  pOp 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 216 of file CommMpi.cpp.

References ASSERTL0, Nektar::LibUtilities::Comm::GetSize(), m_comm, Nektar::LibUtilities::ReduceMax, Nektar::LibUtilities::ReduceMin, and Nektar::LibUtilities::ReduceSum.

218 {
219  if (GetSize() == 1)
220  {
221  return;
222  }
223 
224  MPI_Op vOp;
225  switch (pOp)
226  {
227  case ReduceMax:
228  vOp = MPI_MAX;
229  break;
230  case ReduceMin:
231  vOp = MPI_MIN;
232  break;
233  case ReduceSum:
234  default:
235  vOp = MPI_SUM;
236  break;
237  }
238  int retval = MPI_Allreduce(MPI_IN_PLACE, buf, count, dt, vOp, m_comm);
239 
240  ASSERTL0(retval == MPI_SUCCESS, "MPI error performing All-reduce.");
241 }
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:198
int GetSize()
Returns number of processes.
Definition: Comm.h:188
void Nektar::LibUtilities::CommMpi::v_AlltoAll ( void sendbuf,
int  sendcount,
CommDataType  sendtype,
void recvbuf,
int  recvcount,
CommDataType  recvtype 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 246 of file CommMpi.cpp.

References ASSERTL0, and m_comm.

248 {
249  int retval = MPI_Alltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount,
250  recvtype, m_comm);
251 
252  ASSERTL0(retval == MPI_SUCCESS, "MPI error performing All-to-All.");
253 }
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:198
void Nektar::LibUtilities::CommMpi::v_AlltoAllv ( void sendbuf,
int  sendcounts[],
int  sensdispls[],
CommDataType  sendtype,
void recvbuf,
int  recvcounts[],
int  rdispls[],
CommDataType  recvtype 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 258 of file CommMpi.cpp.

References ASSERTL0, and m_comm.

262 {
263  int retval = MPI_Alltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf,
264  recvcounts, rdispls, recvtype, m_comm);
265 
266  ASSERTL0(retval == MPI_SUCCESS, "MPI error performing All-to-All-v.");
267 }
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:198
void Nektar::LibUtilities::CommMpi::v_Bcast ( void buffer,
int  count,
CommDataType  dt,
int  root 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 269 of file CommMpi.cpp.

References ASSERTL0, and m_comm.

270 {
271  int retval = MPI_Bcast(buffer, count, dt, root, m_comm);
272  ASSERTL0(retval == MPI_SUCCESS, "MPI error performing Bcast-v.");
273 }
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:198
array buffer
Definition: GsLib.hpp:60
void Nektar::LibUtilities::CommMpi::v_Block ( )
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 146 of file CommMpi.cpp.

References m_comm.

147 {
148  MPI_Barrier(m_comm);
149 }
CommSharedPtr Nektar::LibUtilities::CommMpi::v_CommCreateIf ( int  flag)
protectedvirtual

Create a new communicator if the flag is non-zero.

Implements Nektar::LibUtilities::Comm.

Definition at line 354 of file CommMpi.cpp.

References CommMpi(), and m_comm.

355 {
356  MPI_Comm newComm;
357  // color == MPI_UNDEF => not in the new communicator
358  // key == 0 on all => use rank to order them. OpenMPI, at least,
359  // implies this is faster than ordering them ourselves.
360  MPI_Comm_split(m_comm, flag ? 0 : MPI_UNDEFINED, 0, &newComm);
361 
362  if (flag == 0)
363  {
364  // flag == 0 => get back MPI_COMM_NULL, return a null ptr instead.
365  return boost::shared_ptr<Comm>();
366  }
367  else
368  {
369  // Return a real communicator
370  return boost::shared_ptr<Comm>(new CommMpi(newComm));
371  }
372 }
CommMpi(int narg, char *arg[])
Definition: CommMpi.cpp:53
void Nektar::LibUtilities::CommMpi::v_Exscan ( Array< OneD, unsigned long long > &  pData,
const enum ReduceOperator  pOp,
Array< OneD, unsigned long long > &  ans 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 275 of file CommMpi.cpp.

References ASSERTL0, m_comm, Nektar::LibUtilities::MPI_UNSIGNED_LONG_LONG, Nektar::LibUtilities::ReduceMax, Nektar::LibUtilities::ReduceMin, and Nektar::LibUtilities::ReduceSum.

278 {
279  int n = pData.num_elements();
280  ASSERTL0(n == ans.num_elements(), "Array sizes differ in Exscan");
281 
282  MPI_Op vOp;
283  switch (pOp)
284  {
285  case ReduceMax:
286  vOp = MPI_MAX;
287  break;
288  case ReduceMin:
289  vOp = MPI_MIN;
290  break;
291  case ReduceSum:
292  default:
293  vOp = MPI_SUM;
294  break;
295  }
296 
297  int retval = MPI_Exscan(pData.get(), ans.get(), n, MPI_UNSIGNED_LONG_LONG,
298  vOp, m_comm);
299  ASSERTL0(retval == MPI_SUCCESS, "MPI error performing Exscan-v.");
300 }
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:198
void Nektar::LibUtilities::CommMpi::v_Finalise ( )
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 106 of file CommMpi.cpp.

107 {
108 #ifdef NEKTAR_USING_PETSC
109  PetscFinalize();
110 #endif
111  int flag;
112  MPI_Finalized(&flag);
113  if (!flag)
114  {
115  MPI_Finalize();
116  }
117 }
void Nektar::LibUtilities::CommMpi::v_Gather ( void sendbuf,
int  sendcount,
CommDataType  sendtype,
void recvbuf,
int  recvcount,
CommDataType  recvtype,
int  root 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 302 of file CommMpi.cpp.

References ASSERTL0, and m_comm.

305 {
306  int retval = MPI_Gather(sendbuf, sendcount, sendtype, recvbuf, recvcount,
307  recvtype, root, m_comm);
308 
309  ASSERTL0(retval == MPI_SUCCESS, "MPI error performing Gather.");
310 }
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:198
int Nektar::LibUtilities::CommMpi::v_GetRank ( void  )
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 122 of file CommMpi.cpp.

References m_rank.

123 {
124  return m_rank;
125 }
void Nektar::LibUtilities::CommMpi::v_Recv ( void buf,
int  count,
CommDataType  dt,
int  source 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 177 of file CommMpi.cpp.

References m_comm.

178 {
179  MPI_Recv(buf, count, dt, source, 0, m_comm, MPI_STATUS_IGNORE);
180  // ASSERTL0(status.MPI_ERROR == MPI_SUCCESS,
181  // "MPI error receiving data.");
182 }
void Nektar::LibUtilities::CommMpi::v_Scatter ( void sendbuf,
int  sendcount,
CommDataType  sendtype,
void recvbuf,
int  recvcount,
CommDataType  recvtype,
int  root 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 312 of file CommMpi.cpp.

References ASSERTL0, and m_comm.

315 {
316  int retval = MPI_Scatter(sendbuf, sendcount, sendtype, recvbuf, recvcount,
317  recvtype, root, m_comm);
318  ASSERTL0(retval == MPI_SUCCESS, "MPI error performing Scatter.");
319 }
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:198
void Nektar::LibUtilities::CommMpi::v_Send ( void buf,
int  count,
CommDataType  dt,
int  dest 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 162 of file CommMpi.cpp.

References m_comm, and MPISYNC.

163 {
164  if (MPISYNC)
165  {
166  MPI_Ssend(buf, count, dt, dest, 0, m_comm);
167  }
168  else
169  {
170  MPI_Send(buf, count, dt, dest, 0, m_comm);
171  }
172 }
#define MPISYNC
Definition: CommMpi.h:45
#define dest(otri, vertexptr)
void Nektar::LibUtilities::CommMpi::v_SendRecv ( void sendbuf,
int  sendcount,
CommDataType  sendtype,
int  dest,
void recvbuf,
int  recvcount,
CommDataType  recvtype,
int  source 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 187 of file CommMpi.cpp.

References ASSERTL0, and m_comm.

190 {
191  MPI_Status status;
192  int retval = MPI_Sendrecv(sendbuf, sendcount, sendtype, dest, 0, recvbuf,
193  recvcount, recvtype, source, 0, m_comm, &status);
194 
195  ASSERTL0(retval == MPI_SUCCESS,
196  "MPI error performing send-receive of data.");
197 }
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:198
#define dest(otri, vertexptr)
void Nektar::LibUtilities::CommMpi::v_SendRecvReplace ( void buf,
int  count,
CommDataType  dt,
int  pSendProc,
int  pRecvProc 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 202 of file CommMpi.cpp.

References ASSERTL0, and m_comm.

204 {
205  MPI_Status status;
206  int retval = MPI_Sendrecv_replace(buf, count, dt, pRecvProc, 0, pSendProc,
207  0, m_comm, &status);
208 
209  ASSERTL0(retval == MPI_SUCCESS,
210  "MPI error performing Send-Receive-Replace of data.");
211 }
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:198
void Nektar::LibUtilities::CommMpi::v_SplitComm ( int  pRows,
int  pColumns 
)
protectedvirtual

Processes are considered as a grid of size pRows*pColumns. Comm objects are created corresponding to the rows and columns of this grid. The row and column to which this process belongs is stored in m_commRow and m_commColumn.

Implements Nektar::LibUtilities::Comm.

Definition at line 327 of file CommMpi.cpp.

References ASSERTL0, CommMpi(), m_comm, Nektar::LibUtilities::Comm::m_commColumn, Nektar::LibUtilities::Comm::m_commRow, m_rank, and Nektar::LibUtilities::Comm::m_size.

328 {
329  ASSERTL0(pRows * pColumns == m_size,
330  "Rows/Columns do not match comm size.");
331 
332  MPI_Comm newComm;
333 
334  // Compute row and column in grid.
335  int myCol = m_rank % pColumns;
336  int myRow = (m_rank - myCol) / pColumns;
337 
338  // Split Comm into rows - all processes with same myRow are put in
339  // the same communicator. The rank within this communicator is the
340  // column index.
341  MPI_Comm_split(m_comm, myRow, myCol, &newComm);
342  m_commRow = boost::shared_ptr<Comm>(new CommMpi(newComm));
343 
344  // Split Comm into columns - all processes with same myCol are put
345  // in the same communicator. The rank within this communicator is
346  // the row index.
347  MPI_Comm_split(m_comm, myCol, myRow, &newComm);
348  m_commColumn = boost::shared_ptr<Comm>(new CommMpi(newComm));
349 }
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:198
CommSharedPtr m_commColumn
Column communicator.
Definition: Comm.h:133
CommSharedPtr m_commRow
Row communicator.
Definition: Comm.h:132
CommMpi(int narg, char *arg[])
Definition: CommMpi.cpp:53
int m_size
Number of processes.
Definition: Comm.h:130
bool Nektar::LibUtilities::CommMpi::v_TreatAsRankZero ( void  )
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 130 of file CommMpi.cpp.

References m_rank.

131 {
132  if (m_rank == 0)
133  {
134  return true;
135  }
136  else
137  {
138  return false;
139  }
140  return true;
141 }
double Nektar::LibUtilities::CommMpi::v_Wtime ( )
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 154 of file CommMpi.cpp.

155 {
156  return MPI_Wtime();
157 }

Member Data Documentation

std::string Nektar::LibUtilities::CommMpi::className
static
Initial value:
"ParallelMPI", CommMpi::create, "Parallel communication using MPI.")

Name of class.

Definition at line 71 of file CommMpi.h.

MPI_Comm Nektar::LibUtilities::CommMpi::m_comm
private
int Nektar::LibUtilities::CommMpi::m_rank
private

Definition at line 117 of file CommMpi.h.

Referenced by CommMpi(), v_GetRank(), v_SplitComm(), and v_TreatAsRankZero().