36 #ifdef NEKTAR_USING_PETSC
45 namespace LibUtilities
51 "Parallel communication using MPI.");
60 MPI_Initialized(&init);
61 ASSERTL0(!init,
"MPI has already been initialised.");
63 int retval = MPI_Init(&narg, &arg);
64 if (retval != MPI_SUCCESS)
66 ASSERTL0(
false,
"Failed to initialise MPI");
73 #ifdef NEKTAR_USING_PETSC
74 PetscInitializeNoArguments();
118 #ifdef NEKTAR_USING_PETSC
165 MPI_Ssend( pData.get(),
166 (int) pData.num_elements(),
174 MPI_Send( pData.get(),
175 (int) pData.num_elements(),
190 MPI_Recv( pData.get(),
191 (int) pData.num_elements(),
210 MPI_Ssend( pData.get(),
211 (int) pData.num_elements(),
219 MPI_Send( pData.get(),
220 (int) pData.num_elements(),
235 MPI_Recv( pData.get(),
236 (int) pData.num_elements(),
255 MPI_Ssend( &pData[0],
302 int retval = MPI_Sendrecv(pSendData.get(),
303 (int) pSendData.num_elements(),
308 (int) pRecvData.num_elements(),
316 "MPI error performing send-receive of data.");
329 int retval = MPI_Sendrecv(pSendData.get(),
330 (int) pSendData.num_elements(),
335 (int) pRecvData.num_elements(),
343 "MPI error performing send-receive of data.");
354 int retval = MPI_Sendrecv_replace(pSendData.get(),
355 (int) pSendData.num_elements(),
365 "MPI error performing Send-Receive-Replace of data.");
378 int retval = MPI_Sendrecv_replace(pSendData.get(),
379 (int) pSendData.num_elements(),
389 "MPI error performing Send-Receive-Replace of data.");
409 default: vOp = MPI_SUM;
break;
411 int retval = MPI_Allreduce( MPI_IN_PLACE,
419 "MPI error performing All-reduce.");
439 default: vOp = MPI_SUM;
break;
441 int retval = MPI_Allreduce( MPI_IN_PLACE,
449 "MPI error performing All-reduce.");
469 default: vOp = MPI_SUM;
break;
471 int retval = MPI_Allreduce( MPI_IN_PLACE,
473 (int) pData.num_elements(),
479 "MPI error performing All-reduce.");
499 default: vOp = MPI_SUM;
break;
501 int retval = MPI_Allreduce( MPI_IN_PLACE,
503 (int) pData.num_elements(),
509 "MPI error performing All-reduce.");
529 default: vOp = MPI_SUM;
break;
531 int retval = MPI_Allreduce( MPI_IN_PLACE,
539 "MPI error performing All-reduce.");
548 int retval = MPI_Alltoall(pSendData.get(),
549 (int) pSendData.num_elements()/
GetSize(),
552 (int) pRecvData.num_elements()/
GetSize(),
557 "MPI error performing All-to-All.");
566 int retval = MPI_Alltoall(pSendData.get(),
567 (int) pSendData.num_elements()/
GetSize(),
570 (int) pRecvData.num_elements()/
GetSize(),
575 "MPI error performing All-to-All.");
589 int retval = MPI_Alltoallv(pSendData.get(),
590 pSendDataSizeMap.get(),
591 pSendDataOffsetMap.get(),
594 pRecvDataSizeMap.get(),
595 pRecvDataOffsetMap.get(),
600 "MPI error performing All-to-All-v.");
613 int retval = MPI_Alltoallv(pSendData.get(),
614 pSendDataSizeMap.get(),
615 pSendDataOffsetMap.get(),
618 pRecvDataSizeMap.get(),
619 pRecvDataOffsetMap.get(),
624 "MPI error performing All-to-All-v.");
637 "Rows/Columns do not match comm size.");
642 int myCol =
m_rank % pColumns;
643 int myRow = (
m_rank - myCol) / pColumns;
648 MPI_Comm_split(
m_comm, myRow, myCol, &newComm);
654 MPI_Comm_split(
m_comm, myCol, myRow, &newComm);
#define ASSERTL0(condition, msg)
ReduceOperator
Type of operation to perform in AllReduce.
virtual void v_Finalise()
virtual void v_SendRecv(int pSendProc, Array< OneD, NekDouble > &pSendData, int pRecvProc, Array< OneD, NekDouble > &pRecvData)
CommSharedPtr m_commColumn
Column communicator.
virtual void v_Recv(int pProc, Array< OneD, NekDouble > &pData)
std::string m_type
Type of communication.
CommFactory & GetCommFactory()
CommSharedPtr m_commRow
Row communicator.
virtual void v_SendRecvReplace(int pSendProc, int pRecvProc, Array< OneD, NekDouble > &pSendData)
static CommSharedPtr create(int narg, char *arg[])
Creates an instance of this class.
virtual void v_SplitComm(int pRows, int pColumns)
CommMpi(int narg, char *arg[])
virtual void v_AllReduce(NekDouble &pData, enum ReduceOperator pOp)
virtual bool v_TreatAsRankZero(void)
virtual void v_AlltoAll(Array< OneD, NekDouble > &pSendData, Array< OneD, NekDouble > &pRecvData)
virtual void v_Send(int pProc, Array< OneD, NekDouble > &pData)
Base communications class.
static std::string className
Name of class.
virtual void v_AlltoAllv(Array< OneD, NekDouble > &pSendData, Array< OneD, int > &pSendDataSizeMap, Array< OneD, int > &pSendDataOffsetMap, Array< OneD, NekDouble > &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)
int GetSize()
Returns number of processes.
int m_size
Number of processes.
tKey RegisterCreatorFunction(tKey idKey, CreatorFunction classCreator, tDescription pDesc="")
Register a class with the factory.