Nektar::LibUtilities::CommMpi Class Reference

A global linear system. More...

#include <CommMpi.h>

Inheritance diagram for Nektar::LibUtilities::CommMpi:

Public Member Functions
	CommMpi (int narg, char *arg[])
	~CommMpi () override
MPI_Comm	GetComm ()
Public Member Functions inherited from Nektar::LibUtilities::Comm
	Comm (int narg, char *arg[])
virtual	~Comm ()
void	Finalise ()
int	GetSize () const
	Returns number of processes.
int	GetRank ()
const std::string &	GetType () const
bool	TreatAsRankZero ()
bool	IsSerial ()
bool	IsParallelInTime ()
std::tuple< int, int, int >	GetVersion ()
void	Block ()
	Block execution until all processes reach this point.
NekDouble	Wtime ()
	Return the time in seconds.
template<class T >
void	Send (int pProc, T &pData)
template<class T >
void	Recv (int pProc, T &pData)
template<class T >
void	SendRecv (int pSendProc, T &pSendData, int pRecvProc, T &pRecvData)
template<class T >
void	AllReduce (T &pData, enum ReduceOperator pOp)
template<class T >
void	AlltoAll (T &pSendData, T &pRecvData)
template<class T1 , class T2 >
void	AlltoAllv (T1 &pSendData, T2 &pSendDataSizeMap, T2 &pSendDataOffsetMap, T1 &pRecvData, T2 &pRecvDataSizeMap, T2 &pRecvDataOffsetMap)
template<class T >
void	AllGather (T &pSendData, T &pRecvData)
template<class T >
void	AllGatherv (T &pSendData, T &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)
template<class T >
void	AllGatherv (T &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)
template<class T >
void	Bcast (T &pData, int pRoot)
template<class T >
T	Gather (int rootProc, T &val)
template<class T >
T	Scatter (int rootProc, T &pData)
template<class T >
void	DistGraphCreateAdjacent (T &sources, T &sourceweights, int reorder)
template<class T1 , class T2 >
void	NeighborAlltoAllv (T1 &pSendData, T2 &pSendDataSizeMap, T2 &pSendDataOffsetMap, T1 &pRecvData, T2 &pRecvDataSizeMap, T2 &pRecvDataOffsetMap)
template<class T >
void	Irsend (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	Isend (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	SendInit (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	Irecv (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	RecvInit (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
void	StartAll (const CommRequestSharedPtr &request)
void	WaitAll (const CommRequestSharedPtr &request)
CommRequestSharedPtr	CreateRequest (int num)
CommSharedPtr	CommCreateIf (int flag)
	If the flag is non-zero create a new communicator.
void	SplitComm (int pRows, int pColumns, int pTime=0)
	Splits this communicator into a grid of size pRows*pColumns and creates row and column communicators. By default the communicator is a single row.
CommSharedPtr	GetRowComm ()
	Retrieve the row communicator to which this process belongs.
CommSharedPtr	GetColumnComm ()
	Retrieve the column communicator to which this process belongs.
CommSharedPtr	GetTimeComm ()
	Retrieve the time communicator to which this process belongs.
CommSharedPtr	GetSpaceComm ()
	Retrieve the space communicator to which this process belongs.
bool	RemoveExistingFiles ()
std::pair< CommSharedPtr, CommSharedPtr >	SplitCommNode ()

Static Public Member Functions
static CommSharedPtr	create (int narg, char *arg[])
	Creates an instance of this class.

Static Public Attributes
static std::string	className
	Name of class.

Protected Member Functions
	CommMpi (MPI_Comm pComm)
void	v_Finalise () override
int	v_GetRank () final
bool	v_TreatAsRankZero () final
bool	v_IsSerial () final
std::tuple< int, int, int >	v_GetVersion () final
void	v_Block () final
double	v_Wtime () final
void	v_Send (const void *buf, int count, CommDataType dt, int dest) final
void	v_Recv (void *buf, int count, CommDataType dt, int source) final
void	v_SendRecv (const void *sendbuf, int sendcount, CommDataType sendtype, int dest, void *recvbuf, int recvcount, CommDataType recvtype, int source) final
void	v_AllReduce (void *buf, int count, CommDataType dt, enum ReduceOperator pOp) final
void	v_AlltoAll (const void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype) final
void	v_AlltoAllv (const void *sendbuf, const int *sendcounts, const int *senddispls, CommDataType sendtype, void *recvbuf, const int *recvcounts, const int *recvdispls, CommDataType recvtype) final
void	v_AllGather (const void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype) final
void	v_AllGatherv (const void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, const int *recvcounts, const int *recvdispls, CommDataType recvtype) final
void	v_AllGatherv (void *recvbuf, const int *recvcounts, const int *recvdispls, CommDataType recvtype) final
void	v_Bcast (void *buffer, int count, CommDataType dt, int root) final
void	v_Gather (const void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype, int root) final
void	v_Scatter (const void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype, int root) final
void	v_DistGraphCreateAdjacent (int indegree, const int *sources, const int *sourceweights, int reorder) final
void	v_NeighborAlltoAllv (const void *sendbuf, const int *sendcounts, const int *senddispls, CommDataType sendtype, void *recvbuf, const int *recvcounts, const int *recvdispls, CommDataType recvtype) final
void	v_Irsend (const void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc) final
void	v_Isend (const void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc) final
void	v_SendInit (const void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc) final
void	v_Irecv (void *buf, int count, CommDataType dt, int source, CommRequestSharedPtr request, int loc) final
void	v_RecvInit (void *buf, int count, CommDataType dt, int source, CommRequestSharedPtr request, int loc) final
void	v_StartAll (CommRequestSharedPtr request) final
void	v_WaitAll (CommRequestSharedPtr request) final
CommRequestSharedPtr	v_CreateRequest (int num) final
void	v_SplitComm (int pRows, int pColumns, int pTime) override
CommSharedPtr	v_CommCreateIf (int flag) final
std::pair< CommSharedPtr, CommSharedPtr >	v_SplitCommNode () final
Protected Member Functions inherited from Nektar::LibUtilities::Comm
	Comm ()

Protected Attributes
MPI_Comm	m_comm
int	m_rank {}
bool	m_controls_mpi
Protected Attributes inherited from Nektar::LibUtilities::Comm
int	m_size
	Number of processes.
std::string	m_type
	Type of communication.
CommSharedPtr	m_commRow
	Row communicator.
CommSharedPtr	m_commColumn
	Column communicator.
CommSharedPtr	m_commTime
CommSharedPtr	m_commSpace

Detailed Description

A global linear system.

Definition at line 89 of file CommMpi.h.

Constructor & Destructor Documentation

CommMpi() [1/2]

Nektar::LibUtilities::CommMpi::CommMpi	(	int	narg,
		char *	arg[]
	)

Definition at line 50 of file CommMpi.cpp.

                                      : Comm(narg, arg)
{
    int init = 0;
    MPI_Initialized(&init);
 
    if (!init)
    {
        int thread_support = 0;
        if (MPI_Init_thread(&narg, &arg, MPI_THREAD_MULTIPLE,
                            &thread_support) != MPI_SUCCESS)
        {
            NEKERROR(
                ErrorUtil::ewarning,
                "Initializing MPI using MPI_Init, if scotch version > 6 and is "
                "compiled with multi-threading, it might cause deadlocks.")
            ASSERTL0(MPI_Init(&narg, &arg) == MPI_SUCCESS,
                     "Failed to initialise MPI");
        }
        // store bool to indicate that Nektar++ is in charge of finalizing MPI.
        m_controls_mpi = true;
    }
    else
    {
        // Another code is in charge of finalizing MPI and this is not the
        // responsiblity of Nektar++
        m_controls_mpi = false;
    }
 
    m_comm = MPI_COMM_WORLD;
    MPI_Comm_size(m_comm, &m_size);
    MPI_Comm_rank(m_comm, &m_rank);
 
#ifdef NEKTAR_USING_PETSC
    PetscInitializeNoArguments();
#endif
 
    m_type = "Parallel MPI";
}

References ASSERTL0, Nektar::ErrorUtil::ewarning, m_comm, m_controls_mpi, m_rank, Nektar::LibUtilities::Comm::m_size, Nektar::LibUtilities::Comm::m_type, and NEKERROR.

~CommMpi()

Nektar::LibUtilities::CommMpi::~CommMpi ( )

override

Definition at line 104 of file CommMpi.cpp.

{
    int flag;
    MPI_Finalized(&flag);
    if (!flag && m_comm != MPI_COMM_WORLD)
    {
        MPI_Comm_free(&m_comm);
    }
}

References m_comm.

CommMpi() [2/2]

Nektar::LibUtilities::CommMpi::CommMpi ( MPI_Comm  pComm )

explicitprotected

Definition at line 92 of file CommMpi.cpp.

                               : Comm()
{
    m_comm = pComm;
    MPI_Comm_size(m_comm, &m_size);
    MPI_Comm_rank(m_comm, &m_rank);
 
    m_type = "Parallel MPI";
}

References m_comm, m_rank, Nektar::LibUtilities::Comm::m_size, and Nektar::LibUtilities::Comm::m_type.

Member Function Documentation

create()

static CommSharedPtr Nektar::LibUtilities::CommMpi::create ( int  narg, char *  arg[]  )

inlinestatic

Creates an instance of this class.

Definition at line 93 of file CommMpi.h.

    {
        return MemoryManager<CommMpi>::AllocateSharedPtr(narg, arg);
    }

References Nektar::MemoryManager< DataType >::AllocateSharedPtr().

GetComm()

MPI_Comm Nektar::LibUtilities::CommMpi::GetComm

(

)

Definition at line 117 of file CommMpi.cpp.

{
    return m_comm;
}

References m_comm.

v_AllGather()

void Nektar::LibUtilities::CommMpi::v_AllGather ( const void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 290 of file CommMpi.cpp.

{
    int retval = MPI_Allgather(sendbuf, sendcount, sendtype, recvbuf, recvcount,
                               recvtype, m_comm);
 
    ASSERTL0(retval == MPI_SUCCESS, "MPI error performing Allgather.");
}

References ASSERTL0, and m_comm.

v_AllGatherv() [1/2]

void Nektar::LibUtilities::CommMpi::v_AllGatherv ( const void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, const int *  recvcounts, const int *  recvdispls, CommDataType  recvtype  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 303 of file CommMpi.cpp.

{
    int retval = MPI_Allgatherv(sendbuf, sendcount, sendtype, recvbuf,
                                recvcounts, recvdispls, recvtype, m_comm);
 
    ASSERTL0(retval == MPI_SUCCESS, "MPI error performing Allgatherv.");
}

References ASSERTL0, and m_comm.

v_AllGatherv() [2/2]

void Nektar::LibUtilities::CommMpi::v_AllGatherv ( void *  recvbuf, const int *  recvcounts, const int *  recvdispls, CommDataType  recvtype  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 317 of file CommMpi.cpp.

{
    int retval = MPI_Allgatherv(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, recvbuf,
                                recvcounts, recvdispls, recvtype, m_comm);
 
    ASSERTL0(retval == MPI_SUCCESS, "MPI error performing Allgatherv.");
}

References ASSERTL0, and m_comm.

v_AllReduce()

void Nektar::LibUtilities::CommMpi::v_AllReduce ( void *  buf, int  count, CommDataType  dt, enum ReduceOperator  pOp  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 232 of file CommMpi.cpp.

{
    if (GetSize() == 1)
    {
        return;
    }
 
    MPI_Op vOp;
    switch (pOp)
    {
        case ReduceMax:
            vOp = MPI_MAX;
            break;
        case ReduceMin:
            vOp = MPI_MIN;
            break;
        case ReduceSum:
        default:
            vOp = MPI_SUM;
            break;
    }
    int retval = MPI_Allreduce(MPI_IN_PLACE, buf, count, dt, vOp, m_comm);
 
    ASSERTL0(retval == MPI_SUCCESS, "MPI error performing All-reduce.");
}

References ASSERTL0, Nektar::LibUtilities::Comm::GetSize(), m_comm, Nektar::LibUtilities::ReduceMax, Nektar::LibUtilities::ReduceMin, and Nektar::LibUtilities::ReduceSum.

v_AlltoAll()

void Nektar::LibUtilities::CommMpi::v_AlltoAll ( const void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 262 of file CommMpi.cpp.

{
    int retval = MPI_Alltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount,
                              recvtype, m_comm);
 
    ASSERTL0(retval == MPI_SUCCESS, "MPI error performing All-to-All.");
}

References ASSERTL0, and m_comm.

v_AlltoAllv()

void Nektar::LibUtilities::CommMpi::v_AlltoAllv ( const void *  sendbuf, const int *  sendcounts, const int *  senddispls, CommDataType  sendtype, void *  recvbuf, const int *  recvcounts, const int *  recvdispls, CommDataType  recvtype  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 275 of file CommMpi.cpp.

{
    int retval =
        MPI_Alltoallv(sendbuf, sendcounts, senddispls, sendtype, recvbuf,
                      recvcounts, recvdispls, recvtype, m_comm);
 
    ASSERTL0(retval == MPI_SUCCESS, "MPI error performing All-to-All-v.");
}

References ASSERTL0, and m_comm.

v_Bcast()

void Nektar::LibUtilities::CommMpi::v_Bcast ( void *  buffer, int  count, CommDataType  dt, int  root  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 329 of file CommMpi.cpp.

{
    int retval = MPI_Bcast(buffer, count, dt, root, m_comm);
 
    ASSERTL0(retval == MPI_SUCCESS, "MPI error performing Bcast-v.");
}

References ASSERTL0, and m_comm.

v_Block()

void Nektar::LibUtilities::CommMpi::v_Block ( )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 178 of file CommMpi.cpp.

{
    MPI_Barrier(m_comm);
}

References m_comm.

v_CommCreateIf()

CommSharedPtr Nektar::LibUtilities::CommMpi::v_CommCreateIf ( int  flag )

finalprotectedvirtual

Create a new communicator if the flag is non-zero.

Implements Nektar::LibUtilities::Comm.

Definition at line 571 of file CommMpi.cpp.

{
    MPI_Comm newComm;
    // color == MPI_UNDEF => not in the new communicator
    // key == 0 on all => use rank to order them. OpenMPI, at least,
    // implies this is faster than ordering them ourselves.
    MPI_Comm_split(m_comm, flag ? flag : MPI_UNDEFINED, 0, &newComm);
 
    if (flag == 0)
    {
        // flag == 0 => get back MPI_COMM_NULL, return a null ptr instead.
        return std::shared_ptr<Comm>();
    }
    else
    {
        // Return a real communicator
        return std::shared_ptr<Comm>(new CommMpi(newComm));
    }
}

References m_comm.

Referenced by v_SplitCommNode().

v_CreateRequest()

CommRequestSharedPtr Nektar::LibUtilities::CommMpi::v_CreateRequest ( int  num )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 491 of file CommMpi.cpp.

{
    return std::shared_ptr<CommRequest>(new CommRequestMpi(num));
}

v_DistGraphCreateAdjacent()

void Nektar::LibUtilities::CommMpi::v_DistGraphCreateAdjacent ( int  indegree, const int *  sources, const int *  sourceweights, int  reorder  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 365 of file CommMpi.cpp.

{
#if MPI_VERSION < 3
    ASSERTL0(false, "MPI_Dist_graph_create_adjacent is not supported in your "
                    "installed MPI version.");
#else
    int retval = MPI_Dist_graph_create_adjacent(
        m_comm, indegree, sources, sourceweights, indegree, sources,
        sourceweights, MPI_INFO_NULL, reorder, &m_comm);
 
    ASSERTL0(retval == MPI_SUCCESS,
             "MPI error performing Dist_graph_create_adjacent.")
#endif
}

References ASSERTL0, and m_comm.

v_Finalise()

void Nektar::LibUtilities::CommMpi::v_Finalise ( )

overrideprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 125 of file CommMpi.cpp.

{
#ifdef NEKTAR_USING_PETSC
    PetscFinalize();
#endif
    int flag;
    MPI_Finalized(&flag);
    if ((!flag) && m_controls_mpi)
    {
        MPI_Finalize();
    }
}

References m_controls_mpi.

Referenced by Nektar::LibUtilities::CommCwipi::v_Finalise().

v_Gather()

void Nektar::LibUtilities::CommMpi::v_Gather ( const void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype, int  root  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 339 of file CommMpi.cpp.

{
    int retval = MPI_Gather(sendbuf, sendcount, sendtype, recvbuf, recvcount,
                            recvtype, root, m_comm);
 
    ASSERTL0(retval == MPI_SUCCESS, "MPI error performing Gather.");
}

References ASSERTL0, and m_comm.

v_GetRank()

int Nektar::LibUtilities::CommMpi::v_GetRank ( void  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 141 of file CommMpi.cpp.

{
    return m_rank;
}

References m_rank.

v_GetVersion()

std::tuple< int, int, int > Nektar::LibUtilities::CommMpi::v_GetVersion ( )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 165 of file CommMpi.cpp.

{
    int version, subversion;
    int retval = MPI_Get_version(&version, &subversion);
 
    ASSERTL0(retval == MPI_SUCCESS, "MPI error performing GetVersion.");
 
    return std::make_tuple(version, subversion, 0);
}

References ASSERTL0.

v_Irecv()

void Nektar::LibUtilities::CommMpi::v_Irecv ( void *  buf, int  count, CommDataType  dt, int  source, CommRequestSharedPtr  request, int  loc  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 442 of file CommMpi.cpp.

{
    CommRequestMpiSharedPtr req =
        std::static_pointer_cast<CommRequestMpi>(request);
    MPI_Irecv(buf, count, dt, source, 0, m_comm, req->GetRequest(loc));
}

References m_comm.

v_Irsend()

void Nektar::LibUtilities::CommMpi::v_Irsend ( const void *  buf, int  count, CommDataType  dt, int  dest, CommRequestSharedPtr  request, int  loc  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 409 of file CommMpi.cpp.

{
    CommRequestMpiSharedPtr req =
        std::static_pointer_cast<CommRequestMpi>(request);
    MPI_Irsend(buf, count, dt, dest, 0, m_comm, req->GetRequest(loc));
}

References m_comm.

v_Isend()

void Nektar::LibUtilities::CommMpi::v_Isend ( const void *  buf, int  count, CommDataType  dt, int  dest, CommRequestSharedPtr  request, int  loc  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 420 of file CommMpi.cpp.

{
    CommRequestMpiSharedPtr req =
        std::static_pointer_cast<CommRequestMpi>(request);
    MPI_Isend(buf, count, dt, dest, 0, m_comm, req->GetRequest(loc));
}

References m_comm.

v_IsSerial()

bool Nektar::LibUtilities::CommMpi::v_IsSerial ( void  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 157 of file CommMpi.cpp.

{
    return m_size == 1;
}

References Nektar::LibUtilities::Comm::m_size.

v_NeighborAlltoAllv()

void Nektar::LibUtilities::CommMpi::v_NeighborAlltoAllv ( const void *  sendbuf, const int *  sendcounts, const int *  senddispls, CommDataType  sendtype, void *  recvbuf, const int *  recvcounts, const int *  recvdispls, CommDataType  recvtype  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 385 of file CommMpi.cpp.

{
#if MPI_VERSION < 3
    ASSERTL0(false, "MPI_Neighbor_alltoallv is not supported in your "
                    "installed MPI version.");
#else
    int retval = MPI_Neighbor_alltoallv(sendbuf, sendcounts, senddispls,
                                        sendtype, recvbuf, recvcounts,
                                        recvdispls, recvtype, m_comm);
 
    ASSERTL0(retval == MPI_SUCCESS, "MPI error performing NeighborAllToAllV.");
#endif
}

References ASSERTL0, and m_comm.

v_Recv()

void Nektar::LibUtilities::CommMpi::v_Recv ( void *  buf, int  count, CommDataType  dt, int  source  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 209 of file CommMpi.cpp.

{
    MPI_Recv(buf, count, dt, source, 0, m_comm, MPI_STATUS_IGNORE);
}

References m_comm.

v_RecvInit()

void Nektar::LibUtilities::CommMpi::v_RecvInit ( void *  buf, int  count, CommDataType  dt, int  source, CommRequestSharedPtr  request, int  loc  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 453 of file CommMpi.cpp.

{
    CommRequestMpiSharedPtr req =
        std::static_pointer_cast<CommRequestMpi>(request);
    MPI_Recv_init(buf, count, dt, source, 0, m_comm, req->GetRequest(loc));
}

References m_comm.

v_Scatter()

void Nektar::LibUtilities::CommMpi::v_Scatter ( const void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype, int  root  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 352 of file CommMpi.cpp.

{
    int retval = MPI_Scatter(sendbuf, sendcount, sendtype, recvbuf, recvcount,
                             recvtype, root, m_comm);
 
    ASSERTL0(retval == MPI_SUCCESS, "MPI error performing Scatter.");
}

References ASSERTL0, and m_comm.

v_Send()

void Nektar::LibUtilities::CommMpi::v_Send ( const void *  buf, int  count, CommDataType  dt, int  dest  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 194 of file CommMpi.cpp.

{
    if (MPISYNC)
    {
        MPI_Ssend(buf, count, dt, dest, 0, m_comm);
    }
    else
    {
        MPI_Send(buf, count, dt, dest, 0, m_comm);
    }
}

References m_comm, and MPISYNC.

v_SendInit()

void Nektar::LibUtilities::CommMpi::v_SendInit ( const void *  buf, int  count, CommDataType  dt, int  dest, CommRequestSharedPtr  request, int  loc  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 431 of file CommMpi.cpp.

{
    CommRequestMpiSharedPtr req =
        std::static_pointer_cast<CommRequestMpi>(request);
    MPI_Send_init(buf, count, dt, dest, 0, m_comm, req->GetRequest(loc));
}

References m_comm.

v_SendRecv()

void Nektar::LibUtilities::CommMpi::v_SendRecv ( const void *  sendbuf, int  sendcount, CommDataType  sendtype, int  dest, void *  recvbuf, int  recvcount, CommDataType  recvtype, int  source  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 217 of file CommMpi.cpp.

{
    MPI_Status status;
    int retval = MPI_Sendrecv(sendbuf, sendcount, sendtype, dest, 0, recvbuf,
                              recvcount, recvtype, source, 0, m_comm, &status);
 
    ASSERTL0(retval == MPI_SUCCESS,
             "MPI error performing send-receive of data.");
}

References ASSERTL0, and m_comm.

v_SplitComm()

void Nektar::LibUtilities::CommMpi::v_SplitComm ( int  pRows, int  pColumns, int  pTime  )

overrideprotectedvirtual

Processes are considered as a grid of size pRows*pColumns. Comm objects are created corresponding to the rows and columns of this grid. The row and column to which this process belongs is stored in m_commRow and m_commColumn.

Implements Nektar::LibUtilities::Comm.

Definition at line 502 of file CommMpi.cpp.

{
    if (pTime == 0)
    {
        ASSERTL0(pRows * pColumns == m_size,
                 "Rows/Columns do not match comm size.");
 
        MPI_Comm newComm;
 
        // Compute row and column in grid.
        int myCol = m_rank % pColumns;
        int myRow = (m_rank - myCol) / pColumns;
 
        // Split Comm into rows - all processes with same myRow are put in
        // the same communicator. The rank within this communicator is the
        // column index.
        MPI_Comm_split(m_comm, myRow, myCol, &newComm);
        m_commRow = std::shared_ptr<Comm>(new CommMpi(newComm));
 
        // Split Comm into columns - all processes with same myCol are put
        // in the same communicator. The rank within this communicator is
        // the row index.
        MPI_Comm_split(m_comm, myCol, myRow, &newComm);
        m_commColumn = std::shared_ptr<Comm>(new CommMpi(newComm));
    }
    else
    {
        ASSERTL0(pRows * pColumns * pTime == m_size,
                 "Rows/Columns/Time do not match comm size.");
 
        MPI_Comm newComm;
 
        // Compute row and column in grid.
        int mySpace = m_rank % (pRows * pColumns);
        int myTime  = (m_rank - mySpace) / (pRows * pColumns);
        int myCol   = mySpace % pColumns;
        int myRow   = (mySpace - myCol) / pColumns;
 
        // Split Comm - all processes with same mySpace are put in
        // the same communicator. The rank within this communicator is the
        // time index.
        MPI_Comm_split(m_comm, mySpace, myTime, &newComm);
        m_commTime = std::shared_ptr<Comm>(new CommMpi(newComm));
 
        // Split Comm - all processes with same myTime are put in
        // the same communicator. The rank within this communicator is the
        // spatial index.
        MPI_Comm_split(m_comm, myTime, mySpace, &newComm);
        m_commSpace = std::shared_ptr<Comm>(new CommMpi(newComm));
 
        // Split Comm into rows - all processes with same myRow are put in
        // the same communicator. The rank within this communicator is the
        // column index.
        MPI_Comm_split(m_comm, myRow + pRows * pColumns * myTime, myCol,
                       &newComm);
        m_commRow = std::shared_ptr<Comm>(new CommMpi(newComm));
 
        // Split Comm into columns - all processes with same myCol are put
        // in the same communicator. The rank within this communicator is
        // the row index.
        MPI_Comm_split(m_comm, myCol + pRows * pColumns * myTime, myRow,
                       &newComm);
        m_commColumn = std::shared_ptr<Comm>(new CommMpi(newComm));
    }
}

References ASSERTL0, m_comm, Nektar::LibUtilities::Comm::m_commColumn, Nektar::LibUtilities::Comm::m_commRow, Nektar::LibUtilities::Comm::m_commSpace, Nektar::LibUtilities::Comm::m_commTime, m_rank, and Nektar::LibUtilities::Comm::m_size.

v_SplitCommNode()

std::pair< CommSharedPtr, CommSharedPtr > Nektar::LibUtilities::CommMpi::v_SplitCommNode ( )

finalprotectedvirtual

Reimplemented from Nektar::LibUtilities::Comm.

Definition at line 594 of file CommMpi.cpp.

{
    std::pair<CommSharedPtr, CommSharedPtr> ret;
 
#if MPI_VERSION < 3
    ASSERTL0(false, "Not implemented for non-MPI-3 versions.");
#else
    // Create an intra-node communicator.
    MPI_Comm nodeComm;
    MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, m_rank,
                        MPI_INFO_NULL, &nodeComm);
 
    // For rank 0 of the intra-node communicator, split the main
    // communicator. Everyone else will get a null communicator.
    ret.first  = std::shared_ptr<Comm>(new CommMpi(nodeComm));
    ret.second = CommMpi::v_CommCreateIf(ret.first->GetRank() == 0);
    if (ret.first->GetRank() == 0)
    {
        ret.second->SplitComm(1, ret.second->GetSize());
    }
#endif
 
    return ret;
}

References ASSERTL0, m_rank, and v_CommCreateIf().

v_StartAll()

void Nektar::LibUtilities::CommMpi::v_StartAll ( CommRequestSharedPtr  request )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 464 of file CommMpi.cpp.

{
    CommRequestMpiSharedPtr req =
        std::static_pointer_cast<CommRequestMpi>(request);
    if (req->GetNumRequest() != 0)
    {
        MPI_Startall(req->GetNumRequest(), req->GetRequest(0));
    }
}

v_TreatAsRankZero()

bool Nektar::LibUtilities::CommMpi::v_TreatAsRankZero ( void  )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 149 of file CommMpi.cpp.

{
    return m_rank == 0;
}

References m_rank.

v_WaitAll()

void Nektar::LibUtilities::CommMpi::v_WaitAll ( CommRequestSharedPtr  request )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 477 of file CommMpi.cpp.

{
    CommRequestMpiSharedPtr req =
        std::static_pointer_cast<CommRequestMpi>(request);
    if (req->GetNumRequest() != 0)
    {
        MPI_Waitall(req->GetNumRequest(), req->GetRequest(0),
                    MPI_STATUSES_IGNORE);
    }
}

v_Wtime()

double Nektar::LibUtilities::CommMpi::v_Wtime ( )

finalprotectedvirtual

Implements Nektar::LibUtilities::Comm.

Definition at line 186 of file CommMpi.cpp.

{
    return MPI_Wtime();
}

Member Data Documentation

className

std::string Nektar::LibUtilities::CommMpi::className

static

Initial value:

= GetCommFactory().RegisterCreatorFunction(
    "ParallelMPI", CommMpi::create, "Parallel communication using MPI.")

Name of class.

Definition at line 99 of file CommMpi.h.

m_comm

MPI_Comm Nektar::LibUtilities::CommMpi::m_comm

protected

Definition at line 107 of file CommMpi.h.

Referenced by Nektar::LibUtilities::CommCwipi::CommCwipi(), CommMpi(), CommMpi(), GetComm(), v_AllGather(), v_AllGatherv(), v_AllGatherv(), v_AllReduce(), v_AlltoAll(), v_AlltoAllv(), v_Bcast(), v_Block(), v_CommCreateIf(), v_DistGraphCreateAdjacent(), v_Gather(), v_Irecv(), v_Irsend(), v_Isend(), v_NeighborAlltoAllv(), v_Recv(), v_RecvInit(), v_Scatter(), v_Send(), v_SendInit(), v_SendRecv(), v_SplitComm(), and ~CommMpi().

m_controls_mpi

bool Nektar::LibUtilities::CommMpi::m_controls_mpi

protected

Definition at line 109 of file CommMpi.h.

Referenced by CommMpi(), and v_Finalise().

m_rank

int Nektar::LibUtilities::CommMpi::m_rank {}

protected

Definition at line 108 of file CommMpi.h.

{};

Referenced by Nektar::LibUtilities::CommCwipi::CommCwipi(), CommMpi(), CommMpi(), v_GetRank(), v_SplitComm(), v_SplitCommNode(), and v_TreatAsRankZero().