A global linear system. More...

#include <CommMpi.h>

Inheritance diagram for Nektar::LibUtilities::CommMpi:

Public Member Functions
	CommMpi (int narg, char *arg[])

virtual	~CommMpi () override

MPI_Comm	GetComm ()

Public Member Functions inherited from Nektar::LibUtilities::Comm
	Comm (int narg, char *arg[])

virtual	~Comm ()

void	Finalise ()

int	GetSize () const
	Returns number of processes. More...

int	GetRank ()

const std::string &	GetType () const

void	Block ()
	Block execution until all processes reach this point. More...

NekDouble	Wtime ()
	Return the time in seconds. More...

template<class T >
void	Send (int pProc, T &pData)

template<class T >
void	Recv (int pProc, T &pData)

template<class T >
void	SendRecv (int pSendProc, T &pSendData, int pRecvProc, T &pRecvData)

template<class T >
void	SendRecvReplace (int pSendProc, int pRecvProc, T &pData)

template<class T >
void	AllReduce (T &pData, enum ReduceOperator pOp)

template<class T >
void	AlltoAll (T &pSendData, T &pRecvData)

template<class T1 , class T2 >
void	AlltoAllv (T1 &pSendData, T2 &pSendDataSizeMap, T2 &pSendDataOffsetMap, T1 &pRecvData, T2 &pRecvDataSizeMap, T2 &pRecvDataOffsetMap)

template<class T >
void	AllGather (T &pSendData, T &pRecvData)

template<class T >
void	AllGatherv (T &pSendData, T &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)

template<class T >
void	AllGatherv (T &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)

template<class T >
void	Bcast (T &pData, int pRoot)

template<class T >
void	Exscan (T &pData, enum ReduceOperator pOp, T &ans)

template<class T >
T	Gather (int rootProc, T &val)

template<class T >
T	Scatter (int rootProc, T &pData)

template<class T >
void	DistGraphCreateAdjacent (T &sources, T &sourceweights, int reorder)

template<class T1 , class T2 >
void	NeighborAlltoAllv (T1 &pSendData, T2 &pSendDataSizeMap, T2 &pSendDataOffsetMap, T1 &pRecvData, T2 &pRecvDataSizeMap, T2 &pRecvDataOffsetMap)

template<class T >
void	Irsend (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)

template<class T >
void	SendInit (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)

template<class T >
void	Irecv (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)

template<class T >
void	RecvInit (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)

void	StartAll (const CommRequestSharedPtr &request)

void	WaitAll (const CommRequestSharedPtr &request)

CommRequestSharedPtr	CreateRequest (int num)

CommSharedPtr	CommCreateIf (int flag)
	If the flag is non-zero create a new communicator. More...

void	SplitComm (int pRows, int pColumns)
	Splits this communicator into a grid of size pRows*pColumns and creates row and column communicators. By default the communicator is a single row. More...

CommSharedPtr	GetRowComm ()
	Retrieve the row communicator to which this process belongs. More...

CommSharedPtr	GetColumnComm ()
	Retrieve the column communicator to which this process belongs. More...

bool	TreatAsRankZero ()

bool	IsSerial ()

std::tuple< int, int, int >	GetVersion ()

bool	RemoveExistingFiles ()

std::pair< CommSharedPtr, CommSharedPtr >	SplitCommNode ()

Static Public Member Functions
static CommSharedPtr	create (int narg, char *arg[])
	Creates an instance of this class. More...

Static Public Attributes
static std::string	className
	Name of class. More...

Protected Member Functions
	CommMpi (MPI_Comm pComm)

virtual void	v_Finalise () override

virtual int	v_GetRank () final

virtual void	v_Block () final

virtual double	v_Wtime () final

virtual bool	v_TreatAsRankZero () final

virtual bool	v_IsSerial () final

virtual std::tuple< int, int, int >	v_GetVersion () final

virtual void	v_Send (void *buf, int count, CommDataType dt, int dest) final

virtual void	v_Recv (void *buf, int count, CommDataType dt, int source) final

virtual void	v_SendRecv (void sendbuf, int sendcount, CommDataType sendtype, int dest, void recvbuf, int recvcount, CommDataType recvtype, int source) final

virtual void	v_SendRecvReplace (void *buf, int count, CommDataType dt, int pSendProc, int pRecvProc) final

virtual void	v_AllReduce (void *buf, int count, CommDataType dt, enum ReduceOperator pOp) final

virtual void	v_AlltoAll (void sendbuf, int sendcount, CommDataType sendtype, void recvbuf, int recvcount, CommDataType recvtype) final

virtual void	v_AlltoAllv (void sendbuf, int sendcounts[], int sensdispls[], CommDataType sendtype, void recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype) final

virtual void	v_AllGather (void sendbuf, int sendcount, CommDataType sendtype, void recvbuf, int recvcount, CommDataType recvtype) final

virtual void	v_AllGatherv (void sendbuf, int sendcount, CommDataType sendtype, void recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype) final

virtual void	v_AllGatherv (void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype) final

virtual void	v_Bcast (void *buffer, int count, CommDataType dt, int root) final

virtual void	v_Exscan (Array< OneD, unsigned long long > &pData, enum ReduceOperator pOp, Array< OneD, unsigned long long > &ans) final

virtual void	v_Gather (void sendbuf, int sendcount, CommDataType sendtype, void recvbuf, int recvcount, CommDataType recvtype, int root) final

virtual void	v_Scatter (void sendbuf, int sendcount, CommDataType sendtype, void recvbuf, int recvcount, CommDataType recvtype, int root) final

virtual void	v_DistGraphCreateAdjacent (int indegree, const int sources[], const int sourceweights[], int reorder) final

virtual void	v_NeighborAlltoAllv (void sendbuf, int sendcounts[], int sensdispls[], CommDataType sendtype, void recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype) final

virtual void	v_Irsend (void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc) final

virtual void	v_SendInit (void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc) final

virtual void	v_Irecv (void *buf, int count, CommDataType dt, int source, CommRequestSharedPtr request, int loc) final

virtual void	v_RecvInit (void *buf, int count, CommDataType dt, int source, CommRequestSharedPtr request, int loc) final

virtual void	v_StartAll (CommRequestSharedPtr request) final

virtual void	v_WaitAll (CommRequestSharedPtr request) final

virtual CommRequestSharedPtr	v_CreateRequest (int num) final

virtual void	v_SplitComm (int pRows, int pColumns) override

virtual CommSharedPtr	v_CommCreateIf (int flag) final

virtual std::pair< CommSharedPtr, CommSharedPtr >	v_SplitCommNode () final

Protected Member Functions inherited from Nektar::LibUtilities::Comm
	Comm ()

virtual bool	v_RemoveExistingFiles ()

Protected Attributes
MPI_Comm	m_comm

int	m_rank {}

Protected Attributes inherited from Nektar::LibUtilities::Comm
int	m_size
	Number of processes. More...

std::string	m_type
	Type of communication. More...

CommSharedPtr	m_commRow
	Row communicator. More...

CommSharedPtr	m_commColumn
	Column communicator. More...

Detailed Description

A global linear system.

Definition at line 91 of file CommMpi.h.

Constructor & Destructor Documentation

◆ CommMpi() [1/2]

Nektar::LibUtilities::CommMpi::CommMpi	(	int	narg,
		char *	arg[]
	)

Definition at line 53 of file CommMpi.cpp.

                                       : Comm(narg, arg)
 {
     int init = 0;
     MPI_Initialized(&init);
     ASSERTL0(!init, "MPI has already been initialised.");
  
     int retval = MPI_Init(&narg, &arg);
     if (retval != MPI_SUCCESS)
     {
         ASSERTL0(false, "Failed to initialise MPI");
     }
  
     m_comm = MPI_COMM_WORLD;
     MPI_Comm_size(m_comm, &m_size);
     MPI_Comm_rank(m_comm, &m_rank);
  
 #ifdef NEKTAR_USING_PETSC
     PetscInitializeNoArguments();
 #endif
  
     m_type = "Parallel MPI";
 }

References ASSERTL0, m_comm, m_rank, Nektar::LibUtilities::Comm::m_size, and Nektar::LibUtilities::Comm::m_type.

Referenced by v_CommCreateIf(), v_SplitComm(), and v_SplitCommNode().

◆ ~CommMpi()

Nektar::LibUtilities::CommMpi::~CommMpi ( )

overridevirtual

Definition at line 91 of file CommMpi.cpp.

 {
     int flag;
     MPI_Finalized(&flag);
     if (!flag && m_comm != MPI_COMM_WORLD)
     {
         MPI_Comm_free(&m_comm);
     }
 }

References m_comm.

◆ CommMpi() [2/2]

Nektar::LibUtilities::CommMpi::CommMpi ( MPI_Comm pComm )

explicitprotected

Definition at line 79 of file CommMpi.cpp.

                                : Comm()
 {
     m_comm = pComm;
     MPI_Comm_size(m_comm, &m_size);
     MPI_Comm_rank(m_comm, &m_rank);
  
     m_type = "Parallel MPI";
 }

References m_comm, m_rank, Nektar::LibUtilities::Comm::m_size, and Nektar::LibUtilities::Comm::m_type.

Member Function Documentation

◆ create()

static CommSharedPtr Nektar::LibUtilities::CommMpi::create	(	int	narg,
		char *	arg[]
	)

inlinestatic

Creates an instance of this class.

Definition at line 95 of file CommMpi.h.

     {
         return MemoryManager<CommMpi>::AllocateSharedPtr(narg, arg);
     }

References Nektar::MemoryManager< DataType >::AllocateSharedPtr().

◆ GetComm()

MPI_Comm Nektar::LibUtilities::CommMpi::GetComm ( )

Definition at line 104 of file CommMpi.cpp.

 {
     return m_comm;
 }

References m_comm.

◆ v_AllGather()

void Nektar::LibUtilities::CommMpi::v_AllGather	(	void *	sendbuf,
		int	sendcount,
		CommDataType	sendtype,
		void *	recvbuf,
		int	recvcount,
		CommDataType	recvtype
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 288 of file CommMpi.cpp.

 {
     int retval = MPI_Allgather(sendbuf, sendcount, sendtype, recvbuf, recvcount,
                                recvtype, m_comm);
  
     ASSERTL0(retval == MPI_SUCCESS, "MPI error performing Allgather.");
 }

References ASSERTL0, and m_comm.

◆ v_AllGatherv() [1/2]

void Nektar::LibUtilities::CommMpi::v_AllGatherv	(	void *	recvbuf,
		int	recvcounts[],
		int	rdispls[],
		CommDataType	recvtype
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 307 of file CommMpi.cpp.

 {
     int retval = MPI_Allgatherv(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, recvbuf,
                                 recvcounts, rdispls, recvtype, m_comm);
  
     ASSERTL0(retval == MPI_SUCCESS, "MPI error performing Allgatherv.");
 }

References ASSERTL0, and m_comm.

◆ v_AllGatherv() [2/2]

void Nektar::LibUtilities::CommMpi::v_AllGatherv	(	void *	sendbuf,
		int	sendcount,
		CommDataType	sendtype,
		void *	recvbuf,
		int	recvcounts[],
		int	rdispls[],
		CommDataType	recvtype
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 297 of file CommMpi.cpp.

 {
     int retval = MPI_Allgatherv(sendbuf, sendcount, sendtype, recvbuf,
                                 recvcounts, rdispls, recvtype, m_comm);
  
     ASSERTL0(retval == MPI_SUCCESS, "MPI error performing Allgather.");
 }

References ASSERTL0, and m_comm.

◆ v_AllReduce()

void Nektar::LibUtilities::CommMpi::v_AllReduce	(	void *	buf,
		int	count,
		CommDataType	dt,
		enum ReduceOperator	pOp
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 232 of file CommMpi.cpp.

 {
     if (GetSize() == 1)
     {
         return;
     }
  
     MPI_Op vOp;
     switch (pOp)
     {
         case ReduceMax:
             vOp = MPI_MAX;
             break;
         case ReduceMin:
             vOp = MPI_MIN;
             break;
         case ReduceSum:
         default:
             vOp = MPI_SUM;
             break;
     }
     int retval = MPI_Allreduce(MPI_IN_PLACE, buf, count, dt, vOp, m_comm);
  
     ASSERTL0(retval == MPI_SUCCESS, "MPI error performing All-reduce.");
 }

References ASSERTL0, Nektar::LibUtilities::Comm::GetSize(), m_comm, Nektar::LibUtilities::ReduceMax, Nektar::LibUtilities::ReduceMin, and Nektar::LibUtilities::ReduceSum.

◆ v_AlltoAll()

void Nektar::LibUtilities::CommMpi::v_AlltoAll	(	void *	sendbuf,
		int	sendcount,
		CommDataType	sendtype,
		void *	recvbuf,
		int	recvcount,
		CommDataType	recvtype
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 262 of file CommMpi.cpp.

 {
     int retval = MPI_Alltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount,
                               recvtype, m_comm);
  
     ASSERTL0(retval == MPI_SUCCESS, "MPI error performing All-to-All.");
 }

References ASSERTL0, and m_comm.

◆ v_AlltoAllv()

void Nektar::LibUtilities::CommMpi::v_AlltoAllv	(	void *	sendbuf,
		int	sendcounts[],
		int	sensdispls[],
		CommDataType	sendtype,
		void *	recvbuf,
		int	recvcounts[],
		int	rdispls[],
		CommDataType	recvtype
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 274 of file CommMpi.cpp.

 {
     int retval = MPI_Alltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf,
                                recvcounts, rdispls, recvtype, m_comm);
  
     ASSERTL0(retval == MPI_SUCCESS, "MPI error performing All-to-All-v.");
 }

References ASSERTL0, and m_comm.

◆ v_Bcast()

void Nektar::LibUtilities::CommMpi::v_Bcast	(	void *	buffer,
		int	count,
		CommDataType	dt,
		int	root
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 316 of file CommMpi.cpp.

 {
     int retval = MPI_Bcast(buffer, count, dt, root, m_comm);
     ASSERTL0(retval == MPI_SUCCESS, "MPI error performing Bcast-v.");
 }

References ASSERTL0, and m_comm.

◆ v_Block()

void Nektar::LibUtilities::CommMpi::v_Block ( )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 162 of file CommMpi.cpp.

 {
     MPI_Barrier(m_comm);
 }

References m_comm.

◆ v_CommCreateIf()

CommSharedPtr Nektar::LibUtilities::CommMpi::v_CommCreateIf ( int flag )

finalprotectedvirtual

Create a new communicator if the flag is non-zero.

Implements Nektar::LibUtilities::Comm.

Definition at line 488 of file CommMpi.cpp.

 {
     MPI_Comm newComm;
     // color == MPI_UNDEF => not in the new communicator
     // key == 0 on all => use rank to order them. OpenMPI, at least,
     // implies this is faster than ordering them ourselves.
     MPI_Comm_split(m_comm, flag ? 0 : MPI_UNDEFINED, 0, &newComm);
  
     if (flag == 0)
     {
         // flag == 0 => get back MPI_COMM_NULL, return a null ptr instead.
         return std::shared_ptr<Comm>();
     }
     else
     {
         // Return a real communicator
         return std::shared_ptr<Comm>(new CommMpi(newComm));
     }
 }

References CommMpi(), and m_comm.

Referenced by v_SplitCommNode().

◆ v_CreateRequest()

CommRequestSharedPtr Nektar::LibUtilities::CommMpi::v_CreateRequest ( int num )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 450 of file CommMpi.cpp.

 {
     return std::shared_ptr<CommRequest>(new CommRequestMpi(num));
 }

◆ v_DistGraphCreateAdjacent()

void Nektar::LibUtilities::CommMpi::v_DistGraphCreateAdjacent	(	int	indegree,
		const int	sources[],
		const int	sourceweights[],
		int	reorder
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 368 of file CommMpi.cpp.

 {
 #if MPI_VERSION < 3
     boost::ignore_unused(indegree, sources, sourceweights, reorder);
     ASSERTL0(false, "MPI_Dist_graph_create_adjacent is not supported in your "
                     "installed MPI version.");
 #else
     int retval = MPI_Dist_graph_create_adjacent(
         m_comm, indegree, sources, sourceweights, indegree, sources,
         sourceweights, MPI_INFO_NULL, reorder, &m_comm);
  
     ASSERTL0(retval == MPI_SUCCESS,
              "MPI error performing Dist_graph_create_adjacent.")
 #endif
 }

References ASSERTL0, and m_comm.

◆ v_Exscan()

void Nektar::LibUtilities::CommMpi::v_Exscan	(	Array< OneD, unsigned long long > &	pData,
		enum ReduceOperator	pOp,
		Array< OneD, unsigned long long > &	ans
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 322 of file CommMpi.cpp.

 {
     int n = pData.size();
     ASSERTL0(n == ans.size(), "Array sizes differ in Exscan");
  
     MPI_Op vOp;
     switch (pOp)
     {
         case ReduceMax:
             vOp = MPI_MAX;
             break;
         case ReduceMin:
             vOp = MPI_MIN;
             break;
         case ReduceSum:
         default:
             vOp = MPI_SUM;
             break;
     }
  
     int retval = MPI_Exscan(pData.get(), ans.get(), n, MPI_UNSIGNED_LONG_LONG,
                             vOp, m_comm);
     ASSERTL0(retval == MPI_SUCCESS, "MPI error performing Exscan-v.");
 }

References ASSERTL0, m_comm, MPI_UNSIGNED_LONG_LONG, Nektar::LibUtilities::ReduceMax, Nektar::LibUtilities::ReduceMin, and Nektar::LibUtilities::ReduceSum.

◆ v_Finalise()

void Nektar::LibUtilities::CommMpi::v_Finalise ( )

overrideprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Reimplemented in Nektar::LibUtilities::CommCwipi.

Definition at line 112 of file CommMpi.cpp.

 {
 #ifdef NEKTAR_USING_PETSC
     PetscFinalize();
 #endif
     int flag;
     MPI_Finalized(&flag);
     if (!flag)
     {
         MPI_Finalize();
     }
 }

Referenced by Nektar::LibUtilities::CommCwipi::v_Finalise().

◆ v_Gather()

void Nektar::LibUtilities::CommMpi::v_Gather	(	void *	sendbuf,
		int	sendcount,
		CommDataType	sendtype,
		void *	recvbuf,
		int	recvcount,
		CommDataType	recvtype,
		int	root
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 349 of file CommMpi.cpp.

 {
     int retval = MPI_Gather(sendbuf, sendcount, sendtype, recvbuf, recvcount,
                             recvtype, root, m_comm);
  
     ASSERTL0(retval == MPI_SUCCESS, "MPI error performing Gather.");
 }

References ASSERTL0, and m_comm.

◆ v_GetRank()

int Nektar::LibUtilities::CommMpi::v_GetRank ( void )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 128 of file CommMpi.cpp.

 {
     return m_rank;
 }

References m_rank.

◆ v_GetVersion()

std::tuple< int, int, int > Nektar::LibUtilities::CommMpi::v_GetVersion ( )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 149 of file CommMpi.cpp.

 {
     int version, subversion;
     int retval = MPI_Get_version(&version, &subversion);
  
     ASSERTL0(retval == MPI_SUCCESS, "MPI error performing GetVersion.");
  
     return std::make_tuple(version, subversion, 0);
 }

References ASSERTL0.

◆ v_Irecv()

void Nektar::LibUtilities::CommMpi::v_Irecv	(	void *	buf,
		int	count,
		CommDataType	dt,
		int	source,
		CommRequestSharedPtr	request,
		int	loc
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 420 of file CommMpi.cpp.

 {
     CommRequestMpiSharedPtr req =
         std::static_pointer_cast<CommRequestMpi>(request);
     MPI_Irecv(buf, count, dt, source, 0, m_comm, req->GetRequest(loc));
 }

References CG_Iterations::loc, and m_comm.

◆ v_Irsend()

void Nektar::LibUtilities::CommMpi::v_Irsend	(	void *	buf,
		int	count,
		CommDataType	dt,
		int	dest,
		CommRequestSharedPtr	request,
		int	loc
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 404 of file CommMpi.cpp.

 {
     CommRequestMpiSharedPtr req =
         std::static_pointer_cast<CommRequestMpi>(request);
     MPI_Irsend(buf, count, dt, dest, 0, m_comm, req->GetRequest(loc));
 }

References CG_Iterations::loc, and m_comm.

◆ v_IsSerial()

bool Nektar::LibUtilities::CommMpi::v_IsSerial ( void )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 144 of file CommMpi.cpp.

 {
     return m_size == 1;
 }

References Nektar::LibUtilities::Comm::m_size.

◆ v_NeighborAlltoAllv()

void Nektar::LibUtilities::CommMpi::v_NeighborAlltoAllv	(	void *	sendbuf,
		int	sendcounts[],
		int	sensdispls[],
		CommDataType	sendtype,
		void *	recvbuf,
		int	recvcounts[],
		int	rdispls[],
		CommDataType	recvtype
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 385 of file CommMpi.cpp.

 {
 #if MPI_VERSION < 3
     boost::ignore_unused(sendbuf, sendcounts, sdispls, sendtype, recvbuf,
                          recvcounts, rdispls, recvtype);
     ASSERTL0(false, "MPI_Neighbor_alltoallv is not supported in your "
                     "installed MPI version.");
 #else
     int retval =
         MPI_Neighbor_alltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf,
                                recvcounts, rdispls, recvtype, m_comm);
  
     ASSERTL0(retval == MPI_SUCCESS, "MPI error performing NeighborAllToAllV.");
 #endif
 }

References ASSERTL0, and m_comm.

◆ v_Recv()

void Nektar::LibUtilities::CommMpi::v_Recv	(	void *	buf,
		int	count,
		CommDataType	dt,
		int	source
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 193 of file CommMpi.cpp.

 {
     MPI_Recv(buf, count, dt, source, 0, m_comm, MPI_STATUS_IGNORE);
     // ASSERTL0(status.MPI_ERROR == MPI_SUCCESS,
     //         "MPI error receiving data.");
 }

References m_comm.

◆ v_RecvInit()

void Nektar::LibUtilities::CommMpi::v_RecvInit	(	void *	buf,
		int	count,
		CommDataType	dt,
		int	source,
		CommRequestSharedPtr	request,
		int	loc
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 428 of file CommMpi.cpp.

 {
     CommRequestMpiSharedPtr req =
         std::static_pointer_cast<CommRequestMpi>(request);
     MPI_Recv_init(buf, count, dt, source, 0, m_comm, req->GetRequest(loc));
 }

References CG_Iterations::loc, and m_comm.

◆ v_Scatter()

void Nektar::LibUtilities::CommMpi::v_Scatter	(	void *	sendbuf,
		int	sendcount,
		CommDataType	sendtype,
		void *	recvbuf,
		int	recvcount,
		CommDataType	recvtype,
		int	root
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 359 of file CommMpi.cpp.

 {
     int retval = MPI_Scatter(sendbuf, sendcount, sendtype, recvbuf, recvcount,
                              recvtype, root, m_comm);
     ASSERTL0(retval == MPI_SUCCESS, "MPI error performing Scatter.");
 }

References ASSERTL0, and m_comm.

◆ v_Send()

void Nektar::LibUtilities::CommMpi::v_Send	(	void *	buf,
		int	count,
		CommDataType	dt,
		int	dest
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 178 of file CommMpi.cpp.

 {
     if (MPISYNC)
     {
         MPI_Ssend(buf, count, dt, dest, 0, m_comm);
     }
     else
     {
         MPI_Send(buf, count, dt, dest, 0, m_comm);
     }
 }

References m_comm, and MPISYNC.

◆ v_SendInit()

void Nektar::LibUtilities::CommMpi::v_SendInit	(	void *	buf,
		int	count,
		CommDataType	dt,
		int	dest,
		CommRequestSharedPtr	request,
		int	loc
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 412 of file CommMpi.cpp.

 {
     CommRequestMpiSharedPtr req =
         std::static_pointer_cast<CommRequestMpi>(request);
     MPI_Send_init(buf, count, dt, dest, 0, m_comm, req->GetRequest(loc));
 }

References CG_Iterations::loc, and m_comm.

◆ v_SendRecv()

void Nektar::LibUtilities::CommMpi::v_SendRecv	(	void *	sendbuf,
		int	sendcount,
		CommDataType	sendtype,
		int	dest,
		void *	recvbuf,
		int	recvcount,
		CommDataType	recvtype,
		int	source
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 203 of file CommMpi.cpp.

 {
     MPI_Status status;
     int retval = MPI_Sendrecv(sendbuf, sendcount, sendtype, dest, 0, recvbuf,
                               recvcount, recvtype, source, 0, m_comm, &status);
  
     ASSERTL0(retval == MPI_SUCCESS,
              "MPI error performing send-receive of data.");
 }

References ASSERTL0, and m_comm.

◆ v_SendRecvReplace()

void Nektar::LibUtilities::CommMpi::v_SendRecvReplace	(	void *	buf,
		int	count,
		CommDataType	dt,
		int	pSendProc,
		int	pRecvProc
	)

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 218 of file CommMpi.cpp.

 {
     MPI_Status status;
     int retval = MPI_Sendrecv_replace(buf, count, dt, pRecvProc, 0, pSendProc,
                                       0, m_comm, &status);
  
     ASSERTL0(retval == MPI_SUCCESS,
              "MPI error performing Send-Receive-Replace of data.");
 }

References ASSERTL0, and m_comm.

◆ v_SplitComm()

void Nektar::LibUtilities::CommMpi::v_SplitComm	(	int	pRows,
		int	pColumns
	)

overrideprotectedvirtual

Processes are considered as a grid of size pRows*pColumns. Comm objects are created corresponding to the rows and columns of this grid. The row and column to which this process belongs is stored in m_commRow and m_commColumn.

Implements Nektar::LibUtilities::Comm.

Definition at line 461 of file CommMpi.cpp.

 {
     ASSERTL0(pRows * pColumns == m_size,
              "Rows/Columns do not match comm size.");
  
     MPI_Comm newComm;
  
     // Compute row and column in grid.
     int myCol = m_rank % pColumns;
     int myRow = (m_rank - myCol) / pColumns;
  
     // Split Comm into rows - all processes with same myRow are put in
     // the same communicator. The rank within this communicator is the
     // column index.
     MPI_Comm_split(m_comm, myRow, myCol, &newComm);
     m_commRow = std::shared_ptr<Comm>(new CommMpi(newComm));
  
     // Split Comm into columns - all processes with same myCol are put
     // in the same communicator. The rank within this communicator is
     // the row index.
     MPI_Comm_split(m_comm, myCol, myRow, &newComm);
     m_commColumn = std::shared_ptr<Comm>(new CommMpi(newComm));
 }

References ASSERTL0, CommMpi(), m_comm, Nektar::LibUtilities::Comm::m_commColumn, Nektar::LibUtilities::Comm::m_commRow, m_rank, and Nektar::LibUtilities::Comm::m_size.

◆ v_SplitCommNode()

std::pair< CommSharedPtr, CommSharedPtr > Nektar::LibUtilities::CommMpi::v_SplitCommNode ( )

finalprotectedvirtual

Reimplemented from Nektar::LibUtilities::Comm.

Definition at line 508 of file CommMpi.cpp.

 {
     std::pair<CommSharedPtr, CommSharedPtr> ret;
  
 #if MPI_VERSION < 3
     ASSERTL0(false, "Not implemented for non-MPI-3 versions.");
 #else
     // Create an intra-node communicator.
     MPI_Comm nodeComm;
     MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, m_rank,
                         MPI_INFO_NULL, &nodeComm);
  
     // For rank 0 of the intra-node communicator, split the main
     // communicator. Everyone else will get a null communicator.
     ret.first = std::shared_ptr<Comm>(new CommMpi(nodeComm));
     ret.second = CommMpi::v_CommCreateIf(ret.first->GetRank() == 0);
     if(ret.first->GetRank() == 0)
     {
         ret.second->SplitComm(1, ret.second->GetSize());
     }
 #endif
  
     return ret;
 }

References ASSERTL0, CommMpi(), m_rank, and v_CommCreateIf().

◆ v_StartAll()

void Nektar::LibUtilities::CommMpi::v_StartAll ( CommRequestSharedPtr request )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 436 of file CommMpi.cpp.

 {
     CommRequestMpiSharedPtr req =
         std::static_pointer_cast<CommRequestMpi>(request);
     MPI_Startall(req->GetNumRequest(), req->GetRequest(0));
 }

◆ v_TreatAsRankZero()

bool Nektar::LibUtilities::CommMpi::v_TreatAsRankZero ( void )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 136 of file CommMpi.cpp.

 {
     return m_rank == 0;
 }

References m_rank.

◆ v_WaitAll()

void Nektar::LibUtilities::CommMpi::v_WaitAll ( CommRequestSharedPtr request )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 443 of file CommMpi.cpp.

 {
     CommRequestMpiSharedPtr req =
         std::static_pointer_cast<CommRequestMpi>(request);
     MPI_Waitall(req->GetNumRequest(), req->GetRequest(0), MPI_STATUSES_IGNORE);
 }

◆ v_Wtime()

double Nektar::LibUtilities::CommMpi::v_Wtime ( )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 170 of file CommMpi.cpp.

 {
     return MPI_Wtime();
 }

Member Data Documentation

◆ className

std::string Nektar::LibUtilities::CommMpi::className

static

Initial value:

= GetCommFactory().RegisterCreatorFunction(

"ParallelMPI", CommMpi::create, "Parallel communication using MPI.")

Nektar::LibUtilities::CommMpi::create

static CommSharedPtr create(int narg, char *arg[])

Creates an instance of this class.

Definition: CommMpi.h:95

Nektar::LibUtilities::NekFactory::RegisterCreatorFunction

tKey RegisterCreatorFunction(tKey idKey, CreatorFunction classCreator, std::string pDesc="")

Register a class with the factory.

Definition: NekFactory.hpp:200

Nektar::LibUtilities::GetCommFactory

CommFactory & GetCommFactory()

Definition: Communication/Comm.cpp:67

Name of class.

Definition at line 101 of file CommMpi.h.

◆ m_comm

MPI_Comm Nektar::LibUtilities::CommMpi::m_comm

protected

Definition at line 109 of file CommMpi.h.

Referenced by Nektar::LibUtilities::CommCwipi::CommCwipi(), CommMpi(), GetComm(), v_AllGather(), v_AllGatherv(), v_AllReduce(), v_AlltoAll(), v_AlltoAllv(), v_Bcast(), v_Block(), v_CommCreateIf(), v_DistGraphCreateAdjacent(), v_Exscan(), v_Gather(), v_Irecv(), v_Irsend(), v_NeighborAlltoAllv(), v_Recv(), v_RecvInit(), v_Scatter(), v_Send(), v_SendInit(), v_SendRecv(), v_SendRecvReplace(), v_SplitComm(), and ~CommMpi().

◆ m_rank

int Nektar::LibUtilities::CommMpi::m_rank {}

protected

Definition at line 110 of file CommMpi.h.

Referenced by Nektar::LibUtilities::CommCwipi::CommCwipi(), CommMpi(), v_GetRank(), v_SplitComm(), v_SplitCommNode(), and v_TreatAsRankZero().

Public Member Functions

Static Public Member Functions

Static Public Attributes

Protected Member Functions

Protected Attributes

Detailed Description

Constructor & Destructor Documentation

◆ CommMpi() [1/2]

◆ ~CommMpi()

◆ CommMpi() [2/2]

Member Function Documentation

◆ create()

◆ GetComm()

◆ v_AllGather()

◆ v_AllGatherv() [1/2]

◆ v_AllGatherv() [2/2]

◆ v_AllReduce()

◆ v_AlltoAll()

◆ v_AlltoAllv()

◆ v_Bcast()

◆ v_Block()

◆ v_CommCreateIf()

◆ v_CreateRequest()

◆ v_DistGraphCreateAdjacent()

◆ v_Exscan()

◆ v_Finalise()

◆ v_Gather()

◆ v_GetRank()

◆ v_GetVersion()

◆ v_Irecv()

◆ v_Irsend()

◆ v_IsSerial()

◆ v_NeighborAlltoAllv()

◆ v_Recv()

◆ v_RecvInit()

◆ v_Scatter()

◆ v_Send()

◆ v_SendInit()

◆ v_SendRecv()

◆ v_SendRecvReplace()

◆ v_SplitComm()

◆ v_SplitCommNode()

◆ v_StartAll()

◆ v_TreatAsRankZero()

◆ v_WaitAll()

◆ v_Wtime()

Member Data Documentation

◆ className

◆ m_comm

◆ m_rank