Nektar++
Public Member Functions | Static Public Member Functions | Static Public Attributes | Protected Member Functions | Private Member Functions | Private Attributes | List of all members
Nektar::LibUtilities::CommMpi Class Reference

A global linear system. More...

#include <CommMpi.h>

Inheritance diagram for Nektar::LibUtilities::CommMpi:
Inheritance graph
[legend]
Collaboration diagram for Nektar::LibUtilities::CommMpi:
Collaboration graph
[legend]

Public Member Functions

 CommMpi (int narg, char *arg[])
 
virtual ~CommMpi ()
 
MPI_Comm GetComm ()
 
- Public Member Functions inherited from Nektar::LibUtilities::Comm
 Comm (int narg, char *arg[])
 
virtual ~Comm ()
 
void Finalise ()
 
int GetSize ()
 Returns number of processes. More...
 
int GetRank ()
 
const std::string & GetType () const
 
void Block ()
 Block execution until all processes reach this point. More...
 
void Send (int pProc, Array< OneD, NekDouble > &pData)
 
void Send (int pProc, Array< OneD, int > &pData)
 
void Send (int pProc, std::vector< unsigned int > &pData)
 
void Recv (int pProc, Array< OneD, NekDouble > &pData)
 
void Recv (int pProc, Array< OneD, int > &pData)
 
void Recv (int pProc, std::vector< unsigned int > &pData)
 
void SendRecv (int pSendProc, Array< OneD, NekDouble > &pSendData, int pRecvProc, Array< OneD, NekDouble > &pRecvData)
 
void SendRecv (int pSendProc, Array< OneD, int > &pSendData, int pRecvProc, Array< OneD, int > &pRecvData)
 
void SendRecvReplace (int pSendProc, int pRecvProc, Array< OneD, NekDouble > &pSendData)
 
void SendRecvReplace (int pSendProc, int pRecvProc, Array< OneD, int > &pSendData)
 
void AllReduce (NekDouble &pData, enum ReduceOperator pOp)
 
void AllReduce (int &pData, enum ReduceOperator pOp)
 
void AllReduce (Array< OneD, NekDouble > &pData, enum ReduceOperator pOp)
 
void AllReduce (Array< OneD, int > &pData, enum ReduceOperator pOp)
 
void AllReduce (std::vector< unsigned int > &pData, enum ReduceOperator pOp)
 
void AlltoAll (Array< OneD, NekDouble > &pSendData, Array< OneD, NekDouble > &pRecvData)
 
void AlltoAll (Array< OneD, int > &pSendData, Array< OneD, int > &pRecvData)
 
void AlltoAllv (Array< OneD, NekDouble > &pSendData, Array< OneD, int > &pSendDataSizeMap, Array< OneD, int > &pSendDataOffsetMap, Array< OneD, NekDouble > &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)
 
void AlltoAllv (Array< OneD, int > &pSendData, Array< OneD, int > &pSendDataSizeMap, Array< OneD, int > &pSendDataOffsetMap, Array< OneD, int > &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)
 
void SplitComm (int pRows, int pColumns)
 Splits this communicator into a grid of size pRows*pColumns and creates row and column communicators. By default the communicator is a single row. More...
 
CommSharedPtr GetRowComm ()
 Retrieve the row communicator to which this process belongs. More...
 
CommSharedPtr GetColumnComm ()
 Retrieve the column communicator to which this process belongs. More...
 
bool TreatAsRankZero (void)
 
bool RemoveExistingFiles (void)
 

Static Public Member Functions

static CommSharedPtr create (int narg, char *arg[])
 Creates an instance of this class. More...
 

Static Public Attributes

static std::string className
 Name of class. More...
 

Protected Member Functions

virtual void v_Finalise ()
 
virtual int v_GetRank ()
 
virtual void v_Block ()
 
virtual bool v_TreatAsRankZero (void)
 
virtual void v_Send (int pProc, Array< OneD, NekDouble > &pData)
 
virtual void v_Send (int pProc, Array< OneD, int > &pData)
 
virtual void v_Send (int pProc, std::vector< unsigned int > &pData)
 
virtual void v_Recv (int pProc, Array< OneD, NekDouble > &pData)
 
virtual void v_Recv (int pProc, Array< OneD, int > &pData)
 
virtual void v_Recv (int pProc, std::vector< unsigned int > &pData)
 
virtual void v_SendRecv (int pSendProc, Array< OneD, NekDouble > &pSendData, int pRecvProc, Array< OneD, NekDouble > &pRecvData)
 
virtual void v_SendRecv (int pSendProc, Array< OneD, int > &pSendData, int pRecvProc, Array< OneD, int > &pRecvData)
 
virtual void v_SendRecvReplace (int pSendProc, int pRecvProc, Array< OneD, NekDouble > &pSendData)
 
virtual void v_SendRecvReplace (int pSendProc, int pRecvProc, Array< OneD, int > &pSendData)
 
virtual void v_AllReduce (NekDouble &pData, enum ReduceOperator pOp)
 
virtual void v_AllReduce (int &pData, enum ReduceOperator pOp)
 
virtual void v_AllReduce (Array< OneD, NekDouble > &pData, enum ReduceOperator pOp)
 
virtual void v_AllReduce (Array< OneD, int > &pData, enum ReduceOperator pOp)
 
virtual void v_AllReduce (std::vector< unsigned int > &pData, enum ReduceOperator pOp)
 
virtual void v_AlltoAll (Array< OneD, NekDouble > &pSendData, Array< OneD, NekDouble > &pRecvData)
 
virtual void v_AlltoAll (Array< OneD, int > &pSendData, Array< OneD, int > &pRecvData)
 
virtual void v_AlltoAllv (Array< OneD, NekDouble > &pSendData, Array< OneD, int > &pSendDataSizeMap, Array< OneD, int > &pSendDataOffsetMap, Array< OneD, NekDouble > &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)
 
virtual void v_AlltoAllv (Array< OneD, int > &pSendData, Array< OneD, int > &pSendDataSizeMap, Array< OneD, int > &pSendDataOffsetMap, Array< OneD, int > &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)
 
virtual void v_SplitComm (int pRows, int pColumns)
 
- Protected Member Functions inherited from Nektar::LibUtilities::Comm
 Comm ()
 
virtual bool v_RemoveExistingFiles (void)
 

Private Member Functions

 CommMpi (MPI_Comm pComm)
 

Private Attributes

MPI_Comm m_comm
 
int m_rank
 

Additional Inherited Members

- Protected Attributes inherited from Nektar::LibUtilities::Comm
int m_size
 Number of processes. More...
 
std::string m_type
 Type of communication. More...
 
CommSharedPtr m_commRow
 Row communicator. More...
 
CommSharedPtr m_commColumn
 Column communicator. More...
 

Detailed Description

A global linear system.

Definition at line 61 of file CommMpi.h.

Constructor & Destructor Documentation

Nektar::LibUtilities::CommMpi::CommMpi ( int  narg,
char *  arg[] 
)

Definition at line 56 of file CommMpi.cpp.

References ASSERTL0, m_comm, m_rank, Nektar::LibUtilities::Comm::m_size, and Nektar::LibUtilities::Comm::m_type.

Referenced by v_SplitComm().

57  : Comm(narg,arg)
58  {
59  int init = 0;
60  MPI_Initialized(&init);
61  ASSERTL0(!init, "MPI has already been initialised.");
62 
63  int retval = MPI_Init(&narg, &arg);
64  if (retval != MPI_SUCCESS)
65  {
66  ASSERTL0(false, "Failed to initialise MPI");
67  }
68 
69  m_comm = MPI_COMM_WORLD;
70  MPI_Comm_size( m_comm, &m_size );
71  MPI_Comm_rank( m_comm, &m_rank );
72 
73 #ifdef NEKTAR_USING_PETSC
74  PetscInitializeNoArguments();
75 #endif
76 
77  m_type = "Parallel MPI";
78  }
ASSERTL0
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:161
Nektar::LibUtilities::Comm::m_type
std::string m_type
Type of communication.
Definition: Comm.h:143
Nektar::LibUtilities::Comm::m_size
int m_size
Number of processes.
Definition: Comm.h:142
Nektar::LibUtilities::CommMpi::~CommMpi ( )
virtual

Definition at line 98 of file CommMpi.cpp.

99  {
100 
101  }
Nektar::LibUtilities::CommMpi::CommMpi ( MPI_Comm  pComm)
private

Definition at line 84 of file CommMpi.cpp.

References m_comm, m_rank, Nektar::LibUtilities::Comm::m_size, and Nektar::LibUtilities::Comm::m_type.

85  : Comm()
86  {
87  m_comm = pComm;
88  MPI_Comm_size( m_comm, &m_size );
89  MPI_Comm_rank( m_comm, &m_rank );
90 
91  m_type = "Parallel MPI";
92  }
Nektar::LibUtilities::Comm::m_type
std::string m_type
Type of communication.
Definition: Comm.h:143
Nektar::LibUtilities::Comm::m_size
int m_size
Number of processes.
Definition: Comm.h:142

Member Function Documentation

static CommSharedPtr Nektar::LibUtilities::CommMpi::create ( int  narg,
char *  arg[] 
)
inlinestatic

Creates an instance of this class.

Definition at line 65 of file CommMpi.h.

References Nektar::MemoryManager< DataType >::AllocateSharedPtr().

66  {
67  return MemoryManager<CommMpi>::AllocateSharedPtr(narg,arg);
68  }
Nektar::MemoryManager::AllocateSharedPtr
static boost::shared_ptr< DataType > AllocateSharedPtr()
Allocate a shared pointer from the memory pool.
Definition: NekMemoryManager.hpp:235
MPI_Comm Nektar::LibUtilities::CommMpi::GetComm ( )

Definition at line 107 of file CommMpi.cpp.

References m_comm.

108  {
109  return m_comm;
110  }
void Nektar::LibUtilities::CommMpi::v_AllReduce ( NekDouble &  pData,
enum ReduceOperator  pOp 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 396 of file CommMpi.cpp.

References ASSERTL0, Nektar::LibUtilities::Comm::GetSize(), m_comm, Nektar::LibUtilities::ReduceMax, Nektar::LibUtilities::ReduceMin, and Nektar::LibUtilities::ReduceSum.

397  {
398  if (GetSize() == 1)
399  {
400  return;
401  }
402 
403  MPI_Op vOp;
404  switch (pOp)
405  {
406  case ReduceMax: vOp = MPI_MAX; break;
407  case ReduceMin: vOp = MPI_MIN; break;
408  case ReduceSum:
409  default: vOp = MPI_SUM; break;
410  }
411  int retval = MPI_Allreduce( MPI_IN_PLACE,
412  &pData,
413  1,
414  MPI_DOUBLE,
415  vOp,
416  m_comm);
417 
418  ASSERTL0(retval == MPI_SUCCESS,
419  "MPI error performing All-reduce.");
420  }
ASSERTL0
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:161
Nektar::LibUtilities::Comm::GetSize
int GetSize()
Returns number of processes.
Definition: Comm.h:215
void Nektar::LibUtilities::CommMpi::v_AllReduce ( int &  pData,
enum ReduceOperator  pOp 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 426 of file CommMpi.cpp.

References ASSERTL0, Nektar::LibUtilities::Comm::GetSize(), m_comm, Nektar::LibUtilities::ReduceMax, Nektar::LibUtilities::ReduceMin, and Nektar::LibUtilities::ReduceSum.

427  {
428  if (GetSize() == 1)
429  {
430  return;
431  }
432 
433  MPI_Op vOp;
434  switch (pOp)
435  {
436  case ReduceMax: vOp = MPI_MAX; break;
437  case ReduceMin: vOp = MPI_MIN; break;
438  case ReduceSum:
439  default: vOp = MPI_SUM; break;
440  }
441  int retval = MPI_Allreduce( MPI_IN_PLACE,
442  &pData,
443  1,
444  MPI_INT,
445  vOp,
446  m_comm);
447 
448  ASSERTL0(retval == MPI_SUCCESS,
449  "MPI error performing All-reduce.");
450  }
ASSERTL0
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:161
Nektar::LibUtilities::Comm::GetSize
int GetSize()
Returns number of processes.
Definition: Comm.h:215
void Nektar::LibUtilities::CommMpi::v_AllReduce ( Array< OneD, NekDouble > &  pData,
enum ReduceOperator  pOp 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 456 of file CommMpi.cpp.

References ASSERTL0, Nektar::LibUtilities::Comm::GetSize(), m_comm, Nektar::LibUtilities::ReduceMax, Nektar::LibUtilities::ReduceMin, and Nektar::LibUtilities::ReduceSum.

457  {
458  if (GetSize() == 1)
459  {
460  return;
461  }
462 
463  MPI_Op vOp;
464  switch (pOp)
465  {
466  case ReduceMax: vOp = MPI_MAX; break;
467  case ReduceMin: vOp = MPI_MIN; break;
468  case ReduceSum:
469  default: vOp = MPI_SUM; break;
470  }
471  int retval = MPI_Allreduce( MPI_IN_PLACE,
472  pData.get(),
473  (int) pData.num_elements(),
474  MPI_DOUBLE,
475  vOp,
476  m_comm);
477 
478  ASSERTL0(retval == MPI_SUCCESS,
479  "MPI error performing All-reduce.");
480  }
ASSERTL0
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:161
Nektar::LibUtilities::Comm::GetSize
int GetSize()
Returns number of processes.
Definition: Comm.h:215
void Nektar::LibUtilities::CommMpi::v_AllReduce ( Array< OneD, int > &  pData,
enum ReduceOperator  pOp 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 486 of file CommMpi.cpp.

References ASSERTL0, Nektar::LibUtilities::Comm::GetSize(), m_comm, Nektar::LibUtilities::ReduceMax, Nektar::LibUtilities::ReduceMin, and Nektar::LibUtilities::ReduceSum.

487  {
488  if (GetSize() == 1)
489  {
490  return;
491  }
492 
493  MPI_Op vOp;
494  switch (pOp)
495  {
496  case ReduceMax: vOp = MPI_MAX; break;
497  case ReduceMin: vOp = MPI_MIN; break;
498  case ReduceSum:
499  default: vOp = MPI_SUM; break;
500  }
501  int retval = MPI_Allreduce( MPI_IN_PLACE,
502  pData.get(),
503  (int) pData.num_elements(),
504  MPI_INT,
505  vOp,
506  m_comm);
507 
508  ASSERTL0(retval == MPI_SUCCESS,
509  "MPI error performing All-reduce.");
510  }
ASSERTL0
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:161
Nektar::LibUtilities::Comm::GetSize
int GetSize()
Returns number of processes.
Definition: Comm.h:215
void Nektar::LibUtilities::CommMpi::v_AllReduce ( std::vector< unsigned int > &  pData,
enum ReduceOperator  pOp 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 516 of file CommMpi.cpp.

References ASSERTL0, Nektar::LibUtilities::Comm::GetSize(), m_comm, Nektar::LibUtilities::ReduceMax, Nektar::LibUtilities::ReduceMin, and Nektar::LibUtilities::ReduceSum.

517  {
518  if (GetSize() == 1)
519  {
520  return;
521  }
522 
523  MPI_Op vOp;
524  switch (pOp)
525  {
526  case ReduceMax: vOp = MPI_MAX; break;
527  case ReduceMin: vOp = MPI_MIN; break;
528  case ReduceSum:
529  default: vOp = MPI_SUM; break;
530  }
531  int retval = MPI_Allreduce( MPI_IN_PLACE,
532  &pData[0],
533  (int) pData.size(),
534  MPI_INT,
535  vOp,
536  m_comm);
537 
538  ASSERTL0(retval == MPI_SUCCESS,
539  "MPI error performing All-reduce.");
540  }
ASSERTL0
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:161
Nektar::LibUtilities::Comm::GetSize
int GetSize()
Returns number of processes.
Definition: Comm.h:215
void Nektar::LibUtilities::CommMpi::v_AlltoAll ( Array< OneD, NekDouble > &  pSendData,
Array< OneD, NekDouble > &  pRecvData 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 546 of file CommMpi.cpp.

References ASSERTL0, Nektar::LibUtilities::Comm::GetSize(), and m_comm.

547  {
548  int retval = MPI_Alltoall(pSendData.get(),
549  (int) pSendData.num_elements()/GetSize(),
550  MPI_DOUBLE,
551  pRecvData.get(),
552  (int) pRecvData.num_elements()/GetSize(),
553  MPI_DOUBLE,
554  m_comm);
555 
556  ASSERTL0(retval == MPI_SUCCESS,
557  "MPI error performing All-to-All.");
558  }
ASSERTL0
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:161
Nektar::LibUtilities::Comm::GetSize
int GetSize()
Returns number of processes.
Definition: Comm.h:215
void Nektar::LibUtilities::CommMpi::v_AlltoAll ( Array< OneD, int > &  pSendData,
Array< OneD, int > &  pRecvData 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 564 of file CommMpi.cpp.

References ASSERTL0, Nektar::LibUtilities::Comm::GetSize(), and m_comm.

565  {
566  int retval = MPI_Alltoall(pSendData.get(),
567  (int) pSendData.num_elements()/GetSize(),
568  MPI_INT,
569  pRecvData.get(),
570  (int) pRecvData.num_elements()/GetSize(),
571  MPI_INT,
572  m_comm);
573 
574  ASSERTL0(retval == MPI_SUCCESS,
575  "MPI error performing All-to-All.");
576  }
ASSERTL0
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:161
Nektar::LibUtilities::Comm::GetSize
int GetSize()
Returns number of processes.
Definition: Comm.h:215
void Nektar::LibUtilities::CommMpi::v_AlltoAllv ( Array< OneD, NekDouble > &  pSendData,
Array< OneD, int > &  pSendDataSizeMap,
Array< OneD, int > &  pSendDataOffsetMap,
Array< OneD, NekDouble > &  pRecvData,
Array< OneD, int > &  pRecvDataSizeMap,
Array< OneD, int > &  pRecvDataOffsetMap 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 582 of file CommMpi.cpp.

References ASSERTL0, and m_comm.

588  {
589  int retval = MPI_Alltoallv(pSendData.get(),
590  pSendDataSizeMap.get(),
591  pSendDataOffsetMap.get(),
592  MPI_DOUBLE,
593  pRecvData.get(),
594  pRecvDataSizeMap.get(),
595  pRecvDataOffsetMap.get(),
596  MPI_DOUBLE,
597  m_comm);
598 
599  ASSERTL0(retval == MPI_SUCCESS,
600  "MPI error performing All-to-All-v.");
601  }
ASSERTL0
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:161
void Nektar::LibUtilities::CommMpi::v_AlltoAllv ( Array< OneD, int > &  pSendData,
Array< OneD, int > &  pSendDataSizeMap,
Array< OneD, int > &  pSendDataOffsetMap,
Array< OneD, int > &  pRecvData,
Array< OneD, int > &  pRecvDataSizeMap,
Array< OneD, int > &  pRecvDataOffsetMap 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 606 of file CommMpi.cpp.

References ASSERTL0, and m_comm.

612  {
613  int retval = MPI_Alltoallv(pSendData.get(),
614  pSendDataSizeMap.get(),
615  pSendDataOffsetMap.get(),
616  MPI_INT,
617  pRecvData.get(),
618  pRecvDataSizeMap.get(),
619  pRecvDataOffsetMap.get(),
620  MPI_INT,
621  m_comm);
622 
623  ASSERTL0(retval == MPI_SUCCESS,
624  "MPI error performing All-to-All-v.");
625  }
ASSERTL0
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:161
void Nektar::LibUtilities::CommMpi::v_Block ( )
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 152 of file CommMpi.cpp.

References m_comm.

153  {
154  MPI_Barrier(m_comm);
155  }
void Nektar::LibUtilities::CommMpi::v_Finalise ( )
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 116 of file CommMpi.cpp.

117  {
118 #ifdef NEKTAR_USING_PETSC
119  PetscFinalize();
120 #endif
121  MPI_Finalize();
122  }
int Nektar::LibUtilities::CommMpi::v_GetRank ( void  )
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 128 of file CommMpi.cpp.

References m_rank.

129  {
130  return m_rank;
131  }
void Nektar::LibUtilities::CommMpi::v_Recv ( int  pProc,
Array< OneD, NekDouble > &  pData 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 187 of file CommMpi.cpp.

References m_comm.

188  {
189  MPI_Status status;
190  MPI_Recv( pData.get(),
191  (int) pData.num_elements(),
192  MPI_DOUBLE,
193  pProc,
194  0,
195  m_comm,
196  &status);
197 
198  //ASSERTL0(status.MPI_ERROR == MPI_SUCCESS,
199  // "MPI error receiving data.");
200  }
void Nektar::LibUtilities::CommMpi::v_Recv ( int  pProc,
Array< OneD, int > &  pData 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 232 of file CommMpi.cpp.

References m_comm.

233  {
234  MPI_Status status;
235  MPI_Recv( pData.get(),
236  (int) pData.num_elements(),
237  MPI_INT,
238  pProc,
239  0,
240  m_comm,
241  &status);
242 
243  //ASSERTL0(status.MPI_ERROR == MPI_SUCCESS,
244  // "MPI error receiving data.");
245  }
void Nektar::LibUtilities::CommMpi::v_Recv ( int  pProc,
std::vector< unsigned int > &  pData 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 277 of file CommMpi.cpp.

References m_comm.

278  {
279  MPI_Status status;
280  MPI_Recv( &pData[0],
281  (int) pData.size(),
282  MPI_UNSIGNED,
283  pProc,
284  0,
285  m_comm,
286  &status);
287 
288  //ASSERTL0(status.MPI_ERROR == MPI_SUCCESS,
289  // "MPI error receiving data.");
290  }
void Nektar::LibUtilities::CommMpi::v_Send ( int  pProc,
Array< OneD, NekDouble > &  pData 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 161 of file CommMpi.cpp.

References m_comm, and MPISYNC.

162  {
163  if (MPISYNC)
164  {
165  MPI_Ssend( pData.get(),
166  (int) pData.num_elements(),
167  MPI_DOUBLE,
168  pProc,
169  0,
170  m_comm);
171  }
172  else
173  {
174  MPI_Send( pData.get(),
175  (int) pData.num_elements(),
176  MPI_DOUBLE,
177  pProc,
178  0,
179  m_comm);
180  }
181  }
MPISYNC
#define MPISYNC
Definition: CommMpi.h:45
void Nektar::LibUtilities::CommMpi::v_Send ( int  pProc,
Array< OneD, int > &  pData 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 206 of file CommMpi.cpp.

References m_comm, and MPISYNC.

207  {
208  if (MPISYNC)
209  {
210  MPI_Ssend( pData.get(),
211  (int) pData.num_elements(),
212  MPI_INT,
213  pProc,
214  0,
215  m_comm);
216  }
217  else
218  {
219  MPI_Send( pData.get(),
220  (int) pData.num_elements(),
221  MPI_INT,
222  pProc,
223  0,
224  m_comm);
225  }
226  }
MPISYNC
#define MPISYNC
Definition: CommMpi.h:45
void Nektar::LibUtilities::CommMpi::v_Send ( int  pProc,
std::vector< unsigned int > &  pData 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 251 of file CommMpi.cpp.

References m_comm, and MPISYNC.

252  {
253  if (MPISYNC)
254  {
255  MPI_Ssend( &pData[0],
256  (int) pData.size(),
257  MPI_UNSIGNED,
258  pProc,
259  0,
260  m_comm);
261  }
262  else
263  {
264  MPI_Send( &pData[0],
265  (int) pData.size(),
266  MPI_UNSIGNED,
267  pProc,
268  0,
269  m_comm);
270  }
271  }
MPISYNC
#define MPISYNC
Definition: CommMpi.h:45
void Nektar::LibUtilities::CommMpi::v_SendRecv ( int  pSendProc,
Array< OneD, NekDouble > &  pSendData,
int  pRecvProc,
Array< OneD, NekDouble > &  pRecvData 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 296 of file CommMpi.cpp.

References ASSERTL0, and m_comm.

300  {
301  MPI_Status status;
302  int retval = MPI_Sendrecv(pSendData.get(),
303  (int) pSendData.num_elements(),
304  MPI_DOUBLE,
305  pRecvProc,
306  0,
307  pRecvData.get(),
308  (int) pRecvData.num_elements(),
309  MPI_DOUBLE,
310  pSendProc,
311  0,
312  m_comm,
313  &status);
314 
315  ASSERTL0(retval == MPI_SUCCESS,
316  "MPI error performing send-receive of data.");
317  }
ASSERTL0
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:161
void Nektar::LibUtilities::CommMpi::v_SendRecv ( int  pSendProc,
Array< OneD, int > &  pSendData,
int  pRecvProc,
Array< OneD, int > &  pRecvData 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 323 of file CommMpi.cpp.

References ASSERTL0, and m_comm.

327  {
328  MPI_Status status;
329  int retval = MPI_Sendrecv(pSendData.get(),
330  (int) pSendData.num_elements(),
331  MPI_INT,
332  pRecvProc,
333  0,
334  pRecvData.get(),
335  (int) pRecvData.num_elements(),
336  MPI_INT,
337  pSendProc,
338  0,
339  m_comm,
340  &status);
341 
342  ASSERTL0(retval == MPI_SUCCESS,
343  "MPI error performing send-receive of data.");
344  }
ASSERTL0
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:161
void Nektar::LibUtilities::CommMpi::v_SendRecvReplace ( int  pSendProc,
int  pRecvProc,
Array< OneD, NekDouble > &  pSendData 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 349 of file CommMpi.cpp.

References ASSERTL0, and m_comm.

352  {
353  MPI_Status status;
354  int retval = MPI_Sendrecv_replace(pSendData.get(),
355  (int) pSendData.num_elements(),
356  MPI_DOUBLE,
357  pRecvProc,
358  0,
359  pSendProc,
360  0,
361  m_comm,
362  &status);
363 
364  ASSERTL0(retval == MPI_SUCCESS,
365  "MPI error performing Send-Receive-Replace of data.");
366  }
ASSERTL0
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:161
void Nektar::LibUtilities::CommMpi::v_SendRecvReplace ( int  pSendProc,
int  pRecvProc,
Array< OneD, int > &  pSendData 
)
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 372 of file CommMpi.cpp.

References ASSERTL0, and m_comm.

376  {
377  MPI_Status status;
378  int retval = MPI_Sendrecv_replace(pSendData.get(),
379  (int) pSendData.num_elements(),
380  MPI_INT,
381  pRecvProc,
382  0,
383  pSendProc,
384  0,
385  m_comm,
386  &status);
387 
388  ASSERTL0(retval == MPI_SUCCESS,
389  "MPI error performing Send-Receive-Replace of data.");
390  }
ASSERTL0
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:161
void Nektar::LibUtilities::CommMpi::v_SplitComm ( int  pRows,
int  pColumns 
)
protectedvirtual

Processes are considered as a grid of size pRows*pColumns. Comm objects are created corresponding to the rows and columns of this grid. The row and column to which this process belongs is stored in m_commRow and m_commColumn.

Implements Nektar::LibUtilities::Comm.

Definition at line 634 of file CommMpi.cpp.

References ASSERTL0, CommMpi(), m_comm, Nektar::LibUtilities::Comm::m_commColumn, Nektar::LibUtilities::Comm::m_commRow, m_rank, and Nektar::LibUtilities::Comm::m_size.

635  {
636  ASSERTL0(pRows*pColumns == m_size,
637  "Rows/Columns do not match comm size.");
638 
639  MPI_Comm newComm;
640 
641  // Compute row and column in grid.
642  int myCol = m_rank % pColumns;
643  int myRow = (m_rank - myCol) / pColumns;
644 
645  // Split Comm into rows - all processes with same myRow are put in
646  // the same communicator. The rank within this communicator is the
647  // column index.
648  MPI_Comm_split(m_comm, myRow, myCol, &newComm);
649  m_commRow = boost::shared_ptr<Comm>(new CommMpi(newComm));
650 
651  // Split Comm into columns - all processes with same myCol are put
652  // in the same communicator. The rank within this communicator is
653  // the row index.
654  MPI_Comm_split(m_comm, myCol, myRow, &newComm);
655  m_commColumn = boost::shared_ptr<Comm>(new CommMpi(newComm));
656  }
ASSERTL0
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:161
Nektar::LibUtilities::Comm::m_commColumn
CommSharedPtr m_commColumn
Column communicator.
Definition: Comm.h:145
Nektar::LibUtilities::Comm::m_commRow
CommSharedPtr m_commRow
Row communicator.
Definition: Comm.h:144
Nektar::LibUtilities::CommMpi::CommMpi
CommMpi(int narg, char *arg[])
Definition: CommMpi.cpp:56
Nektar::LibUtilities::Comm::m_size
int m_size
Number of processes.
Definition: Comm.h:142
bool Nektar::LibUtilities::CommMpi::v_TreatAsRankZero ( void  )
protectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 136 of file CommMpi.cpp.

References m_rank.

137  {
138  if(m_rank == 0)
139  {
140  return true;
141  }
142  else
143  {
144  return false;
145  }
146  return true;
147  }

Member Data Documentation

std::string Nektar::LibUtilities::CommMpi::className
static
Initial value:
= GetCommFactory().RegisterCreatorFunction(
"ParallelMPI",
CommMpi::create,
"Parallel communication using MPI.")

Name of class.

Definition at line 71 of file CommMpi.h.

MPI_Comm Nektar::LibUtilities::CommMpi::m_comm
private

Definition at line 132 of file CommMpi.h.

Referenced by CommMpi(), GetComm(), v_AllReduce(), v_AlltoAll(), v_AlltoAllv(), v_Block(), v_Recv(), v_Send(), v_SendRecv(), v_SendRecvReplace(), and v_SplitComm().

int Nektar::LibUtilities::CommMpi::m_rank
private

Definition at line 133 of file CommMpi.h.

Referenced by CommMpi(), v_GetRank(), v_SplitComm(), and v_TreatAsRankZero().

Generated on Tue Dec 8 2015 23:45:36 for Nektar++ by   doxygen 1.8.9.1