35#ifdef NEKTAR_USING_PETSC
53 MPI_Initialized(&init);
57 ASSERTL0(MPI_Init(&narg, &arg) == MPI_SUCCESS,
58 "Failed to initialise MPI");
73#ifdef NEKTAR_USING_PETSC
74 PetscInitializeNoArguments();
99 if (!flag &&
m_comm != MPI_COMM_WORLD)
118#ifdef NEKTAR_USING_PETSC
122 MPI_Finalized(&flag);
158 int version, subversion;
159 int retval = MPI_Get_version(&version, &subversion);
161 ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing GetVersion.");
163 return std::make_tuple(version, subversion, 0);
189 MPI_Ssend(buf, count, dt, dest, 0,
m_comm);
193 MPI_Send(buf, count, dt, dest, 0,
m_comm);
202 MPI_Recv(buf, count, dt, source, 0,
m_comm, MPI_STATUS_IGNORE);
209 int dest,
void *recvbuf,
int recvcount,
213 int retval = MPI_Sendrecv(sendbuf, sendcount, sendtype, dest, 0, recvbuf,
214 recvcount, recvtype, source, 0,
m_comm, &status);
217 "MPI error performing send-receive of data.");
245 int retval = MPI_Allreduce(MPI_IN_PLACE, buf, count, dt, vOp,
m_comm);
247 ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing All-reduce.");
256 int retval = MPI_Alltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount,
259 ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing All-to-All.");
267 int recvcounts[],
int rdispls[],
270 int retval = MPI_Alltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf,
271 recvcounts, rdispls, recvtype,
m_comm);
273 ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing All-to-All-v.");
282 int retval = MPI_Allgather(sendbuf, sendcount, sendtype, recvbuf, recvcount,
285 ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing Allgather.");
292 void *recvbuf,
int recvcounts[],
int rdispls[],
295 int retval = MPI_Allgatherv(sendbuf, sendcount, sendtype, recvbuf,
296 recvcounts, rdispls, recvtype,
m_comm);
298 ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing Allgatherv.");
307 int retval = MPI_Allgatherv(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, recvbuf,
308 recvcounts, rdispls, recvtype,
m_comm);
310 ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing Allgatherv.");
318 int retval = MPI_Bcast(
buffer, count, dt, root,
m_comm);
320 ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing Bcast-v.");
330 int retval = MPI_Gather(sendbuf, sendcount, sendtype, recvbuf, recvcount,
333 ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing Gather.");
343 int retval = MPI_Scatter(sendbuf, sendcount, sendtype, recvbuf, recvcount,
346 ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing Scatter.");
353 [[maybe_unused]]
int indegree, [[maybe_unused]]
const int sources[],
354 [[maybe_unused]]
const int sourceweights[], [[maybe_unused]]
int reorder)
357 ASSERTL0(
false,
"MPI_Dist_graph_create_adjacent is not supported in your "
358 "installed MPI version.");
360 int retval = MPI_Dist_graph_create_adjacent(
361 m_comm, indegree, sources, sourceweights, indegree, sources,
362 sourceweights, MPI_INFO_NULL, reorder, &
m_comm);
365 "MPI error performing Dist_graph_create_adjacent.")
373 [[maybe_unused]]
void *sendbuf, [[maybe_unused]]
int sendcounts[],
374 [[maybe_unused]]
int sdispls[], [[maybe_unused]]
CommDataType sendtype,
375 [[maybe_unused]]
void *recvbuf, [[maybe_unused]]
int recvcounts[],
376 [[maybe_unused]]
int rdispls[], [[maybe_unused]]
CommDataType recvtype)
379 ASSERTL0(
false,
"MPI_Neighbor_alltoallv is not supported in your "
380 "installed MPI version.");
383 MPI_Neighbor_alltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf,
384 recvcounts, rdispls, recvtype,
m_comm);
386 ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing NeighborAllToAllV.");
397 std::static_pointer_cast<CommRequestMpi>(request);
398 MPI_Irsend(buf, count, dt, dest, 0,
m_comm, req->GetRequest(
loc));
408 std::static_pointer_cast<CommRequestMpi>(request);
409 MPI_Isend(buf, count, dt, dest, 0,
m_comm, req->GetRequest(
loc));
419 std::static_pointer_cast<CommRequestMpi>(request);
420 MPI_Send_init(buf, count, dt, dest, 0,
m_comm, req->GetRequest(
loc));
430 std::static_pointer_cast<CommRequestMpi>(request);
431 MPI_Irecv(buf, count, dt, source, 0,
m_comm, req->GetRequest(
loc));
441 std::static_pointer_cast<CommRequestMpi>(request);
442 MPI_Recv_init(buf, count, dt, source, 0,
m_comm, req->GetRequest(
loc));
451 std::static_pointer_cast<CommRequestMpi>(request);
452 if (req->GetNumRequest() != 0)
454 MPI_Startall(req->GetNumRequest(), req->GetRequest(0));
464 std::static_pointer_cast<CommRequestMpi>(request);
465 if (req->GetNumRequest() != 0)
467 MPI_Waitall(req->GetNumRequest(), req->GetRequest(0),
468 MPI_STATUSES_IGNORE);
489 "Rows/Columns/Time do not match comm size.");
504 int myCol =
m_rank % pColumns;
505 int myRow = (
m_rank - myCol) / pColumns;
510 MPI_Comm_split(
m_comm, myRow, myCol, &newComm);
516 MPI_Comm_split(
m_comm, myCol, myRow, &newComm);
521 constexpr int dims = 3;
522 const int sizes[dims] = {pRows, pColumns, pTime};
523 const int periods[dims] = {0, 0, 0};
524 constexpr int reorder = 1;
526 MPI_Cart_create(
m_comm, dims, sizes, periods, reorder, &gridComm);
528 constexpr int keepRow[dims] = {0, 1, 0};
529 MPI_Cart_sub(gridComm, keepRow, &newComm);
532 constexpr int keepCol[dims] = {1, 0, 0};
533 MPI_Cart_sub(gridComm, keepCol, &newComm);
536 constexpr int keepTime[dims] = {0, 0, 1};
537 MPI_Cart_sub(gridComm, keepTime, &newComm);
540 constexpr int keepSpace[dims] = {1, 1, 0};
541 MPI_Cart_sub(gridComm, keepSpace, &newComm);
555 MPI_Comm_split(
m_comm, flag ? flag : MPI_UNDEFINED, 0, &newComm);
560 return std::shared_ptr<Comm>();
565 return std::shared_ptr<Comm>(
new CommMpi(newComm));
574 std::pair<CommSharedPtr, CommSharedPtr> ret;
577 ASSERTL0(
false,
"Not implemented for non-MPI-3 versions.");
581 MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED,
m_rank,
582 MPI_INFO_NULL, &nodeComm);
586 ret.first = std::shared_ptr<Comm>(
new CommMpi(nodeComm));
588 if (ret.first->GetRank() == 0)
590 ret.second->SplitComm(1, ret.second->GetSize());
#define ASSERTL0(condition, msg)
Base communications class.
CommSharedPtr m_commColumn
Column communicator.
CommSharedPtr m_commRow
Row communicator.
int GetSize() const
Returns number of processes.
int m_size
Number of processes.
std::string m_type
Type of communication.
CommSharedPtr m_commSpace
void v_WaitAll(CommRequestSharedPtr request) final
void v_AlltoAll(void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype) final
CommRequestSharedPtr v_CreateRequest(int num) final
CommMpi(int narg, char *arg[])
void v_DistGraphCreateAdjacent(int indegree, const int sources[], const int sourceweights[], int reorder) final
void v_Bcast(void *buffer, int count, CommDataType dt, int root) final
void v_AllGatherv(void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype) final
CommSharedPtr v_CommCreateIf(int flag) final
std::pair< CommSharedPtr, CommSharedPtr > v_SplitCommNode() final
void v_SendRecv(void *sendbuf, int sendcount, CommDataType sendtype, int dest, void *recvbuf, int recvcount, CommDataType recvtype, int source) final
void v_NeighborAlltoAllv(void *sendbuf, int sendcounts[], int sensdispls[], CommDataType sendtype, void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype) final
void v_AllGather(void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype) final
void v_Recv(void *buf, int count, CommDataType dt, int source) final
void v_StartAll(CommRequestSharedPtr request) final
void v_AllReduce(void *buf, int count, CommDataType dt, enum ReduceOperator pOp) final
void v_Finalise() override
void v_Irecv(void *buf, int count, CommDataType dt, int source, CommRequestSharedPtr request, int loc) final
void v_Irsend(void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc) final
void v_Gather(void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype, int root) final
static std::string className
Name of class.
void v_RecvInit(void *buf, int count, CommDataType dt, int source, CommRequestSharedPtr request, int loc) final
static CommSharedPtr create(int narg, char *arg[])
Creates an instance of this class.
void v_Isend(void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc) final
void v_AlltoAllv(void *sendbuf, int sendcounts[], int sensdispls[], CommDataType sendtype, void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype) final
bool v_TreatAsRankZero() final
void v_Send(void *buf, int count, CommDataType dt, int dest) final
void v_SplitComm(int pRows, int pColumns, int pTime) override
void v_Scatter(void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype, int root) final
std::tuple< int, int, int > v_GetVersion() final
void v_SendInit(void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc) final
Class for communicator request type.
tKey RegisterCreatorFunction(tKey idKey, CreatorFunction classCreator, std::string pDesc="")
Register a class with the factory.
unsigned int CommDataType
std::shared_ptr< CommRequest > CommRequestSharedPtr
std::shared_ptr< CommRequestMpi > CommRequestMpiSharedPtr
CommFactory & GetCommFactory()
ReduceOperator
Type of operation to perform in AllReduce.
std::shared_ptr< Comm > CommSharedPtr
Pointer to a Communicator object.