Nektar::LibUtilities::Comm Class Reference

Base communications class. More...

#include <Comm.h>

Inheritance diagram for Nektar::LibUtilities::Comm:

Public Member Functions
	Comm (int narg, char *arg[])
virtual	~Comm ()
void	Finalise ()
int	GetSize () const
	Returns number of processes. More...
int	GetRank ()
const std::string &	GetType () const
bool	TreatAsRankZero ()
bool	IsSerial ()
bool	IsParallelInTime ()
std::tuple< int, int, int >	GetVersion ()
void	Block ()
	Block execution until all processes reach this point. More...
NekDouble	Wtime ()
	Return the time in seconds. More...
template<class T >
void	Send (int pProc, T &pData)
template<class T >
void	Recv (int pProc, T &pData)
template<class T >
void	SendRecv (int pSendProc, T &pSendData, int pRecvProc, T &pRecvData)
template<class T >
void	AllReduce (T &pData, enum ReduceOperator pOp)
template<class T >
void	AlltoAll (T &pSendData, T &pRecvData)
template<class T1 , class T2 >
void	AlltoAllv (T1 &pSendData, T2 &pSendDataSizeMap, T2 &pSendDataOffsetMap, T1 &pRecvData, T2 &pRecvDataSizeMap, T2 &pRecvDataOffsetMap)
template<class T >
void	AllGather (T &pSendData, T &pRecvData)
template<class T >
void	AllGatherv (T &pSendData, T &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)
template<class T >
void	AllGatherv (T &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)
template<class T >
void	Bcast (T &pData, int pRoot)
template<class T >
T	Gather (int rootProc, T &val)
template<class T >
T	Scatter (int rootProc, T &pData)
template<class T >
void	DistGraphCreateAdjacent (T &sources, T &sourceweights, int reorder)
template<class T1 , class T2 >
void	NeighborAlltoAllv (T1 &pSendData, T2 &pSendDataSizeMap, T2 &pSendDataOffsetMap, T1 &pRecvData, T2 &pRecvDataSizeMap, T2 &pRecvDataOffsetMap)
template<class T >
void	Irsend (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	Isend (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	SendInit (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	Irecv (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	RecvInit (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
void	StartAll (const CommRequestSharedPtr &request)
void	WaitAll (const CommRequestSharedPtr &request)
CommRequestSharedPtr	CreateRequest (int num)
CommSharedPtr	CommCreateIf (int flag)
	If the flag is non-zero create a new communicator. More...
void	SplitComm (int pRows, int pColumns, int pTime=1)
	Splits this communicator into a grid of size pRows*pColumns and creates row and column communicators. By default the communicator is a single row. More...
CommSharedPtr	GetRowComm ()
	Retrieve the row communicator to which this process belongs. More...
CommSharedPtr	GetColumnComm ()
	Retrieve the column communicator to which this process belongs. More...
CommSharedPtr	GetTimeComm ()
	Retrieve the time communicator to which this process belongs. More...
CommSharedPtr	GetSpaceComm ()
	Retrieve the space communicator to which this process belongs. More...
bool	RemoveExistingFiles ()
std::pair< CommSharedPtr, CommSharedPtr >	SplitCommNode ()

Protected Member Functions
	Comm ()
virtual void	v_Finalise ()=0
virtual int	v_GetRank ()=0
virtual bool	v_TreatAsRankZero ()=0
virtual bool	v_IsSerial ()=0
virtual std::tuple< int, int, int >	v_GetVersion ()=0
virtual void	v_Block ()=0
virtual NekDouble	v_Wtime ()=0
virtual void	v_Send (void *buf, int count, CommDataType dt, int dest)=0
virtual void	v_Recv (void *buf, int count, CommDataType dt, int source)=0
virtual void	v_SendRecv (void *sendbuf, int sendcount, CommDataType sendtype, int dest, void *recvbuf, int recvcount, CommDataType recvtype, int source)=0
virtual void	v_AllReduce (void *buf, int count, CommDataType dt, enum ReduceOperator pOp)=0
virtual void	v_AlltoAll (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype)=0
virtual void	v_AlltoAllv (void *sendbuf, int sendcounts[], int sensdispls[], CommDataType sendtype, void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype)=0
virtual void	v_AllGather (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype)=0
virtual void	v_AllGatherv (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype)=0
virtual void	v_AllGatherv (void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype)=0
virtual void	v_Bcast (void *buffer, int count, CommDataType dt, int root)=0
virtual void	v_Gather (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype, int root)=0
virtual void	v_Scatter (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype, int root)=0
virtual void	v_DistGraphCreateAdjacent (int indegree, const int sources[], const int sourceweights[], int reorder)=0
virtual void	v_NeighborAlltoAllv (void *sendbuf, int sendcounts[], int sdispls[], CommDataType sendtype, void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype)=0
virtual void	v_Irsend (void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc)=0
virtual void	v_Isend (void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc)=0
virtual void	v_SendInit (void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc)=0
virtual void	v_Irecv (void *buf, int count, CommDataType dt, int source, CommRequestSharedPtr request, int loc)=0
virtual void	v_RecvInit (void *buf, int count, CommDataType dt, int source, CommRequestSharedPtr request, int loc)=0
virtual void	v_StartAll (CommRequestSharedPtr request)=0
virtual void	v_WaitAll (CommRequestSharedPtr request)=0
virtual CommRequestSharedPtr	v_CreateRequest (int num)=0
virtual void	v_SplitComm (int pRows, int pColumns, int pTime)=0
virtual CommSharedPtr	v_CommCreateIf (int flag)=0
virtual std::pair< CommSharedPtr, CommSharedPtr >	v_SplitCommNode ()

Protected Attributes
int	m_size
	Number of processes. More...
std::string	m_type
	Type of communication. More...
CommSharedPtr	m_commRow
	Row communicator. More...
CommSharedPtr	m_commColumn
	Column communicator. More...
CommSharedPtr	m_commTime
CommSharedPtr	m_commSpace

Detailed Description

Base communications class.

Definition at line 87 of file Comm.h.

Constructor & Destructor Documentation

Comm() [1/2]

Nektar::LibUtilities::Comm::Comm	(	int	narg,
		char *	arg[]
	)

Definition at line 39 of file Communication/Comm.cpp.

{
}

~Comm()

Nektar::LibUtilities::Comm::~Comm ( )

virtual

Definition at line 53 of file Communication/Comm.cpp.

{
}

Comm() [2/2]

Nektar::LibUtilities::Comm::Comm ( )

protected

Definition at line 46 of file Communication/Comm.cpp.

{
}

Member Function Documentation

AllGather()

template<class T >

void Nektar::LibUtilities::Comm::AllGather	(	T &	pSendData,
		T &	pRecvData
	)

Definition at line 425 of file Comm.h.

{
    BOOST_STATIC_ASSERT_MSG(
        CommDataTypeTraits<T>::IsVector,
        "AllGather only valid with Array or vector arguments.");
 
    int sendSize = CommDataTypeTraits<T>::GetCount(pSendData);
    int recvSize = sendSize;
 
    pRecvData = T(recvSize * GetSize());
 
    v_AllGather(CommDataTypeTraits<T>::GetPointer(pSendData), sendSize,
                CommDataTypeTraits<T>::GetDataType(),
                CommDataTypeTraits<T>::GetPointer(pRecvData), recvSize,
                CommDataTypeTraits<T>::GetDataType());
}

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), GetSize(), and v_AllGather().

AllGatherv() [1/2]

template<class T >

void Nektar::LibUtilities::Comm::AllGatherv	(	T &	pRecvData,
		Array< OneD, int > &	pRecvDataSizeMap,
		Array< OneD, int > &	pRecvDataOffsetMap
	)

Definition at line 467 of file Comm.h.

{
    BOOST_STATIC_ASSERT_MSG(
        CommDataTypeTraits<T>::IsVector,
        "AllGatherv only valid with Array or vector arguments.");
 
    v_AllGatherv(CommDataTypeTraits<T>::GetPointer(pRecvData),
                 pRecvDataSizeMap.data(), pRecvDataOffsetMap.data(),
                 CommDataTypeTraits<T>::GetDataType());
}

References v_AllGatherv().

AllGatherv() [2/2]

template<class T >

void Nektar::LibUtilities::Comm::AllGatherv	(	T &	pSendData,
		T &	pRecvData,
		Array< OneD, int > &	pRecvDataSizeMap,
		Array< OneD, int > &	pRecvDataOffsetMap
	)

Definition at line 446 of file Comm.h.

{
    BOOST_STATIC_ASSERT_MSG(
        CommDataTypeTraits<T>::IsVector,
        "AllGatherv only valid with Array or vector arguments.");
 
    int sendSize = CommDataTypeTraits<T>::GetCount(pSendData);
 
    v_AllGatherv(CommDataTypeTraits<T>::GetPointer(pSendData), sendSize,
                 CommDataTypeTraits<T>::GetDataType(),
                 CommDataTypeTraits<T>::GetPointer(pRecvData),
                 pRecvDataSizeMap.data(), pRecvDataOffsetMap.data(),
                 CommDataTypeTraits<T>::GetDataType());
}

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), and v_AllGatherv().

AllReduce()

template<class T >

void Nektar::LibUtilities::Comm::AllReduce	(	T &	pData,
		enum ReduceOperator	pOp
	)

Definition at line 370 of file Comm.h.

{
    v_AllReduce(CommDataTypeTraits<T>::GetPointer(pData),
                CommDataTypeTraits<T>::GetCount(pData),
                CommDataTypeTraits<T>::GetDataType(), pOp);
}

References v_AllReduce().

AlltoAll()

template<class T >

void Nektar::LibUtilities::Comm::AlltoAll	(	T &	pSendData,
		T &	pRecvData
	)

Definition at line 380 of file Comm.h.

{
    static_assert(CommDataTypeTraits<T>::IsVector,
                  "AlltoAll only valid with Array or vector arguments.");
    int sendSize = CommDataTypeTraits<T>::GetCount(pSendData);
    int recvSize = CommDataTypeTraits<T>::GetCount(pRecvData);
    ASSERTL0(sendSize == recvSize,
             "Send and Recv arrays have incompatible sizes in AlltoAll");
 
    int count = sendSize / GetSize();
    ASSERTL0(count * GetSize() == sendSize,
             "Array size incompatible with size of communicator");
 
    v_AlltoAll(CommDataTypeTraits<T>::GetPointer(pSendData), count,
               CommDataTypeTraits<T>::GetDataType(),
               CommDataTypeTraits<T>::GetPointer(pRecvData), count,
               CommDataTypeTraits<T>::GetDataType());
}

References ASSERTL0, Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), GetSize(), and v_AlltoAll().

AlltoAllv()

template<class T1 , class T2 >

void Nektar::LibUtilities::Comm::AlltoAllv	(	T1 &	pSendData,
		T2 &	pSendDataSizeMap,
		T2 &	pSendDataOffsetMap,
		T1 &	pRecvData,
		T2 &	pRecvDataSizeMap,
		T2 &	pRecvDataOffsetMap
	)

Definition at line 403 of file Comm.h.

{
    static_assert(CommDataTypeTraits<T1>::IsVector,
                  "AlltoAllv only valid with Array or vector arguments.");
    static_assert(std::is_same_v<T2, std::vector<int>> ||
                      std::is_same_v<T2, Array<OneD, int>>,
                  "Alltoallv size and offset maps should be integer vectors.");
    v_AlltoAllv(CommDataTypeTraits<T1>::GetPointer(pSendData),
                (int *)CommDataTypeTraits<T2>::GetPointer(pSendDataSizeMap),
                (int *)CommDataTypeTraits<T2>::GetPointer(pSendDataOffsetMap),
                CommDataTypeTraits<T1>::GetDataType(),
                CommDataTypeTraits<T1>::GetPointer(pRecvData),
                (int *)CommDataTypeTraits<T2>::GetPointer(pRecvDataSizeMap),
                (int *)CommDataTypeTraits<T2>::GetPointer(pRecvDataOffsetMap),
                CommDataTypeTraits<T1>::GetDataType());
}

References v_AlltoAllv().

Bcast()

template<class T >

void Nektar::LibUtilities::Comm::Bcast	(	T &	pData,
		int	pRoot
	)

Definition at line 482 of file Comm.h.

{
    v_Bcast(CommDataTypeTraits<T>::GetPointer(pData),
            CommDataTypeTraits<T>::GetCount(pData),
            CommDataTypeTraits<T>::GetDataType(), pRoot);
}

References v_Bcast().

Block()

void Nektar::LibUtilities::Comm::Block ( )

inline

Block execution until all processes reach this point.

Definition at line 320 of file Comm.h.

{
    v_Block();
}

References v_Block().

CommCreateIf()

CommSharedPtr Nektar::LibUtilities::Comm::CommCreateIf ( int  flag )

inline

If the flag is non-zero create a new communicator.

Definition at line 722 of file Comm.h.

{
    return v_CommCreateIf(flag);
}

References v_CommCreateIf().

CreateRequest()

CommRequestSharedPtr Nektar::LibUtilities::Comm::CreateRequest ( int  num )

inline

Creates a number of CommRequests.

Parameters

num	Number of requests to generate in the communication request object

Returns

Communication request object

Definition at line 714 of file Comm.h.

{
    return v_CreateRequest(num);
}

References v_CreateRequest().

DistGraphCreateAdjacent()

template<class T >

void Nektar::LibUtilities::Comm::DistGraphCreateAdjacent	(	T &	sources,
		T &	sourceweights,
		int	reorder
	)

This replaces the current MPI communicator with a new one that also holds the distributed graph topology information. If reordering is enabled using this might break code where process/rank numbers are assumed to remain constant. This also assumes that the graph is bi-directional, so all sources are also destinations with equal weighting.

Parameters

sources	Ranks of processes for which the calling process is the destination/source
sourceweights	Weights of the corresponding edges into the calling process
reorder	Ranks may be reordered (true) or not (false)

Definition at line 544 of file Comm.h.

{
    static_assert(
        CommDataTypeTraits<T>::IsVector,
        "DistGraphCreateAdjacent only valid with Array or vector arguments.");
 
    ASSERTL0(CommDataTypeTraits<T>::GetCount(sources) ==
                 CommDataTypeTraits<T>::GetCount(sourceweights),
             "Sources and weights array sizes don't match");
 
    int indegree = CommDataTypeTraits<T>::GetCount(sources);
 
    v_DistGraphCreateAdjacent(
        indegree, (const int *)CommDataTypeTraits<T>::GetPointer(sources),
        (const int *)CommDataTypeTraits<T>::GetPointer(sourceweights), reorder);
}

References ASSERTL0, Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), and v_DistGraphCreateAdjacent().

Finalise()

void Nektar::LibUtilities::Comm::Finalise ( )

inline

Definition at line 256 of file Comm.h.

{
    v_Finalise();
}

References v_Finalise().

Gather()

template<class T >

T Nektar::LibUtilities::Comm::Gather	(	int	rootProc,
		T &	val
	)

Concatenate all the input arrays, in rank order, onto the process with rank == rootProc

Definition at line 493 of file Comm.h.

{
    static_assert(CommDataTypeTraits<T>::IsVector,
                  "Gather only valid with Array or vector arguments.");
    bool amRoot  = (GetRank() == rootProc);
    unsigned nEl = CommDataTypeTraits<T>::GetCount(val);
 
    unsigned nOut = amRoot ? GetSize() * nEl : 0;
    T ans(nOut);
    void *recvbuf = amRoot ? CommDataTypeTraits<T>::GetPointer(ans) : nullptr;
 
    v_Gather(CommDataTypeTraits<T>::GetPointer(val), nEl,
             CommDataTypeTraits<T>::GetDataType(), recvbuf, nEl,
             CommDataTypeTraits<T>::GetDataType(), rootProc);
    return ans;
}

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), Nektar::LibUtilities::CommDataTypeTraits< T >::GetPointer(), GetRank(), GetSize(), and v_Gather().

GetColumnComm()

CommSharedPtr Nektar::LibUtilities::Comm::GetColumnComm ( )

inline

Retrieve the column communicator to which this process belongs.

Definition at line 756 of file Comm.h.

{
    if (!m_commColumn.get())
    {
        return shared_from_this();
    }
    else
    {
        return m_commColumn;
    }
}

References m_commColumn.

GetRank()

int Nektar::LibUtilities::Comm::GetRank ( )

inline

Definition at line 272 of file Comm.h.

{
    return v_GetRank();
}

References v_GetRank().

Referenced by Gather(), and Scatter().

GetRowComm()

CommSharedPtr Nektar::LibUtilities::Comm::GetRowComm ( )

inline

Retrieve the row communicator to which this process belongs.

Definition at line 740 of file Comm.h.

{
    if (!m_commRow.get())
    {
        return shared_from_this();
    }
    else
    {
        return m_commRow;
    }
}

References m_commRow.

GetSize()

int Nektar::LibUtilities::Comm::GetSize ( ) const

inline

Returns number of processes.

Definition at line 264 of file Comm.h.

{
    return m_size;
}

References m_size.

Referenced by AllGather(), AlltoAll(), Gather(), Scatter(), and Nektar::LibUtilities::CommMpi::v_AllReduce().

GetSpaceComm()

CommSharedPtr Nektar::LibUtilities::Comm::GetSpaceComm ( )

inline

Retrieve the space communicator to which this process belongs.

Definition at line 788 of file Comm.h.

{
    if (!m_commSpace.get())
    {
        return shared_from_this();
    }
    else
    {
        return m_commSpace;
    }
}

References m_commSpace.

GetTimeComm()

CommSharedPtr Nektar::LibUtilities::Comm::GetTimeComm ( )

inline

Retrieve the time communicator to which this process belongs.

Definition at line 772 of file Comm.h.

{
    if (!m_commTime.get())
    {
        return shared_from_this();
    }
    else
    {
        return m_commTime;
    }
}

References m_commTime.

GetType()

const std::string & Nektar::LibUtilities::Comm::GetType ( ) const

inline

Definition at line 280 of file Comm.h.

{
    return m_type;
}

References m_type.

GetVersion()

std::tuple< int, int, int > Nektar::LibUtilities::Comm::GetVersion ( )

inline

Returns

tuple of {major, minor, patch} version numbers

Definition at line 312 of file Comm.h.

{
    return v_GetVersion();
}

References v_GetVersion().

Irecv()

template<class T >

void Nektar::LibUtilities::Comm::Irecv	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Begins a nonblocking receive

Parameters

pProc	Rank of source
pData	Array/vector to place incoming data in to
count	Number of elements to receive in to pData
request	Communication request object
loc	Location in request to use

Definition at line 663 of file Comm.h.

{
    v_Irecv(CommDataTypeTraits<T>::GetPointer(pData), count,
            CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References CG_Iterations::loc, and v_Irecv().

Irsend()

template<class T >

void Nektar::LibUtilities::Comm::Irsend	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Starts a ready-mode nonblocking send

Parameters

pProc	Rank of destination
pData	Array/vector to send
count	Number of elements to send in pData
request	Communication request object
loc	Location in request to use

Definition at line 612 of file Comm.h.

{
    v_Irsend(CommDataTypeTraits<T>::GetPointer(pData), count,
             CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References CG_Iterations::loc, and v_Irsend().

Isend()

template<class T >

void Nektar::LibUtilities::Comm::Isend	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Starts a nonblocking send

Parameters

pProc	Rank of destination
pData	Array/vector to send
count	Number of elements to send in pData
request	Communication request object
loc	Location in request to use

Definition at line 629 of file Comm.h.

{
    v_Isend(CommDataTypeTraits<T>::GetPointer(pData), count,
            CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References CG_Iterations::loc, and v_Isend().

IsParallelInTime()

bool Nektar::LibUtilities::Comm::IsParallelInTime ( )

inline

Definition at line 304 of file Comm.h.

{
    return m_commTime.get();
}

References m_commTime.

IsSerial()

bool Nektar::LibUtilities::Comm::IsSerial ( )

inline

Definition at line 296 of file Comm.h.

{
    return v_IsSerial();
}

References v_IsSerial().

NeighborAlltoAllv()

template<class T1 , class T2 >

void Nektar::LibUtilities::Comm::NeighborAlltoAllv	(	T1 &	pSendData,
		T2 &	pSendDataSizeMap,
		T2 &	pSendDataOffsetMap,
		T1 &	pRecvData,
		T2 &	pRecvDataSizeMap,
		T2 &	pRecvDataOffsetMap
	)

Sends data to neighboring processes in a virtual topology communicator. All processes send different amounts of data to, and receive different amounts of data from, all neighbors

Parameters

pSendData	Array/vector to send to neighbors
pSendDataSizeMap	Array/vector where entry i specifies the number of elements to send to neighbor i
pSendDataOffsetMap	Array/vector where entry i specifies the displacement (offset from pSendData) from which to send data to neighbor i
pRecvData	Array/vector to place incoming data in to
pRecvDataSizeMap	Array/vector where entry i specifies the number of elements to receive from neighbor i
pRecvDataOffsetMap	Array/vector where entry i specifies the displacement (offset from pRecvData) from which to receive data from neighbor i

Definition at line 580 of file Comm.h.

{
    static_assert(
        CommDataTypeTraits<T1>::IsVector,
        "NeighbourAlltoAllv only valid with Array or vector arguments.");
    static_assert(
        std::is_same_v<T2, std::vector<int>> ||
            std::is_same_v<T2, Array<OneD, int>>,
        "NeighborAllToAllv size and offset maps should be integer vectors.");
    v_NeighborAlltoAllv(
        CommDataTypeTraits<T1>::GetPointer(pSendData),
        (int *)CommDataTypeTraits<T2>::GetPointer(pSendDataSizeMap),
        (int *)CommDataTypeTraits<T2>::GetPointer(pSendDataOffsetMap),
        CommDataTypeTraits<T1>::GetDataType(),
        CommDataTypeTraits<T1>::GetPointer(pRecvData),
        (int *)CommDataTypeTraits<T2>::GetPointer(pRecvDataSizeMap),
        (int *)CommDataTypeTraits<T2>::GetPointer(pRecvDataOffsetMap),
        CommDataTypeTraits<T1>::GetDataType());
}

References v_NeighborAlltoAllv().

Recv()

template<class T >

void Nektar::LibUtilities::Comm::Recv	(	int	pProc,
		T &	pData
	)

Definition at line 346 of file Comm.h.

{
    v_Recv(CommDataTypeTraits<T>::GetPointer(pData),
           CommDataTypeTraits<T>::GetCount(pData),
           CommDataTypeTraits<T>::GetDataType(), pProc);
}

References v_Recv().

RecvInit()

template<class T >

void Nektar::LibUtilities::Comm::RecvInit	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Create a persistent request for a receive

Parameters

pProc	Rank of source
pData	Array/vector to place incoming data in to
count	Number of elements to receive in to pData
request	Communication request object
loc	Location in request to use

Definition at line 680 of file Comm.h.

{
    v_RecvInit(CommDataTypeTraits<T>::GetPointer(pData), count,
               CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References CG_Iterations::loc, and v_RecvInit().

RemoveExistingFiles()

bool Nektar::LibUtilities::Comm::RemoveExistingFiles ( )

inline

Definition at line 803 of file Comm.h.

{
    return true;
}

Scatter()

template<class T >

T Nektar::LibUtilities::Comm::Scatter	(	int	rootProc,
		T &	pData
	)

Scatter pData across ranks in chunks of len(pData)/num_ranks

Definition at line 513 of file Comm.h.

{
    static_assert(CommDataTypeTraits<T>::IsVector,
                  "Scatter only valid with Array or vector arguments.");
 
    bool amRoot  = (GetRank() == rootProc);
    unsigned nEl = CommDataTypeTraits<T>::GetCount(pData) / GetSize();
 
    void *sendbuf = amRoot ? CommDataTypeTraits<T>::GetPointer(pData) : nullptr;
    T ans(nEl);
 
    v_Scatter(sendbuf, nEl, CommDataTypeTraits<T>::GetDataType(),
              CommDataTypeTraits<T>::GetPointer(ans), nEl,
              CommDataTypeTraits<T>::GetDataType(), rootProc);
    return ans;
}

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), Nektar::LibUtilities::CommDataTypeTraits< T >::GetPointer(), GetRank(), GetSize(), and v_Scatter().

Send()

template<class T >

void Nektar::LibUtilities::Comm::Send	(	int	pProc,
		T &	pData
	)

Definition at line 336 of file Comm.h.

{
    v_Send(CommDataTypeTraits<T>::GetPointer(pData),
           CommDataTypeTraits<T>::GetCount(pData),
           CommDataTypeTraits<T>::GetDataType(), pProc);
}

References v_Send().

SendInit()

template<class T >

void Nektar::LibUtilities::Comm::SendInit	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Creates a persistent request for a send

Parameters

pProc	Rank of destination
pData	Array/vector to send
count	Number of elements to send in pData
request	Communication request object
loc	Location in request to use

Definition at line 646 of file Comm.h.

{
    v_SendInit(CommDataTypeTraits<T>::GetPointer(pData), count,
               CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References CG_Iterations::loc, and v_SendInit().

SendRecv()

template<class T >

void Nektar::LibUtilities::Comm::SendRecv	(	int	pSendProc,
		T &	pSendData,
		int	pRecvProc,
		T &	pRecvData
	)

Definition at line 357 of file Comm.h.

{
    v_SendRecv(CommDataTypeTraits<T>::GetPointer(pSendData),
               CommDataTypeTraits<T>::GetCount(pSendData),
               CommDataTypeTraits<T>::GetDataType(), pSendProc,
               CommDataTypeTraits<T>::GetPointer(pRecvData),
               CommDataTypeTraits<T>::GetCount(pRecvData),
               CommDataTypeTraits<T>::GetDataType(), pRecvProc);
}

References v_SendRecv().

SplitComm()

void Nektar::LibUtilities::Comm::SplitComm ( int  pRows, int  pColumns, int  pTime = 1  )

inline

Splits this communicator into a grid of size pRows*pColumns and creates row and column communicators. By default the communicator is a single row.

Definition at line 732 of file Comm.h.

{
    v_SplitComm(pRows, pColumns, pTime);
}

References v_SplitComm().

SplitCommNode()

std::pair< CommSharedPtr, CommSharedPtr > Nektar::LibUtilities::Comm::SplitCommNode ( )

inline

Definition at line 811 of file Comm.h.

{
    return v_SplitCommNode();
}

References v_SplitCommNode().

StartAll()

void Nektar::LibUtilities::Comm::StartAll ( const CommRequestSharedPtr &  request )

inline

Starts a collection of persistent requests

Parameters

request

Communication request object

Definition at line 692 of file Comm.h.

{
    v_StartAll(request);
}

References v_StartAll().

TreatAsRankZero()

bool Nektar::LibUtilities::Comm::TreatAsRankZero ( )

inline

Definition at line 288 of file Comm.h.

{
    return v_TreatAsRankZero();
}

References v_TreatAsRankZero().

v_AllGather()

virtual void Nektar::LibUtilities::Comm::v_AllGather ( void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by AllGather().

v_AllGatherv() [1/2]

virtual void Nektar::LibUtilities::Comm::v_AllGatherv ( void *  recvbuf, int  recvcounts[], int  rdispls[], CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

v_AllGatherv() [2/2]

virtual void Nektar::LibUtilities::Comm::v_AllGatherv ( void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcounts[], int  rdispls[], CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by AllGatherv().

v_AllReduce()

virtual void Nektar::LibUtilities::Comm::v_AllReduce ( void *  buf, int  count, CommDataType  dt, enum ReduceOperator  pOp  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by AllReduce().

v_AlltoAll()

virtual void Nektar::LibUtilities::Comm::v_AlltoAll ( void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by AlltoAll().

v_AlltoAllv()

virtual void Nektar::LibUtilities::Comm::v_AlltoAllv ( void *  sendbuf, int  sendcounts[], int  sensdispls[], CommDataType  sendtype, void *  recvbuf, int  recvcounts[], int  rdispls[], CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by AlltoAllv().

v_Bcast()

virtual void Nektar::LibUtilities::Comm::v_Bcast ( void *  buffer, int  count, CommDataType  dt, int  root  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Bcast().

v_Block()

virtual void Nektar::LibUtilities::Comm::v_Block ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Block().

v_CommCreateIf()

virtual CommSharedPtr Nektar::LibUtilities::Comm::v_CommCreateIf ( int  flag )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by CommCreateIf().

v_CreateRequest()

virtual CommRequestSharedPtr Nektar::LibUtilities::Comm::v_CreateRequest ( int  num )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by CreateRequest().

v_DistGraphCreateAdjacent()

virtual void Nektar::LibUtilities::Comm::v_DistGraphCreateAdjacent ( int  indegree, const int  sources[], const int  sourceweights[], int  reorder  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by DistGraphCreateAdjacent().

v_Finalise()

virtual void Nektar::LibUtilities::Comm::v_Finalise ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, Nektar::LibUtilities::CommCwipi, and Nektar::LibUtilities::CommMpi.

Referenced by Finalise().

v_Gather()

virtual void Nektar::LibUtilities::Comm::v_Gather ( void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype, int  root  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Gather().

v_GetRank()

virtual int Nektar::LibUtilities::Comm::v_GetRank ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, Nektar::LibUtilities::CommSerial, and Nektar::FieldUtils::FieldConvertComm.

Referenced by GetRank().

v_GetVersion()

virtual std::tuple< int, int, int > Nektar::LibUtilities::Comm::v_GetVersion ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by GetVersion().

v_Irecv()

virtual void Nektar::LibUtilities::Comm::v_Irecv ( void *  buf, int  count, CommDataType  dt, int  source, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Irecv().

v_Irsend()

virtual void Nektar::LibUtilities::Comm::v_Irsend ( void *  buf, int  count, CommDataType  dt, int  dest, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Irsend().

v_Isend()

virtual void Nektar::LibUtilities::Comm::v_Isend ( void *  buf, int  count, CommDataType  dt, int  dest, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Isend().

v_IsSerial()

virtual bool Nektar::LibUtilities::Comm::v_IsSerial ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, Nektar::LibUtilities::CommSerial, and Nektar::FieldUtils::FieldConvertComm.

Referenced by IsSerial().

v_NeighborAlltoAllv()

virtual void Nektar::LibUtilities::Comm::v_NeighborAlltoAllv ( void *  sendbuf, int  sendcounts[], int  sdispls[], CommDataType  sendtype, void *  recvbuf, int  recvcounts[], int  rdispls[], CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by NeighborAlltoAllv().

v_Recv()

virtual void Nektar::LibUtilities::Comm::v_Recv ( void *  buf, int  count, CommDataType  dt, int  source  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Recv().

v_RecvInit()

virtual void Nektar::LibUtilities::Comm::v_RecvInit ( void *  buf, int  count, CommDataType  dt, int  source, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by RecvInit().

v_Scatter()

virtual void Nektar::LibUtilities::Comm::v_Scatter ( void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype, int  root  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Scatter().

v_Send()

virtual void Nektar::LibUtilities::Comm::v_Send ( void *  buf, int  count, CommDataType  dt, int  dest  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Send().

v_SendInit()

virtual void Nektar::LibUtilities::Comm::v_SendInit ( void *  buf, int  count, CommDataType  dt, int  dest, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by SendInit().

v_SendRecv()

virtual void Nektar::LibUtilities::Comm::v_SendRecv ( void *  sendbuf, int  sendcount, CommDataType  sendtype, int  dest, void *  recvbuf, int  recvcount, CommDataType  recvtype, int  source  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by SendRecv().

v_SplitComm()

virtual void Nektar::LibUtilities::Comm::v_SplitComm ( int  pRows, int  pColumns, int  pTime  )

protectedpure virtual

Implemented in Nektar::FieldUtils::FieldConvertComm, Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by SplitComm().

v_SplitCommNode()

std::pair< CommSharedPtr, CommSharedPtr > Nektar::LibUtilities::Comm::v_SplitCommNode ( )

protectedvirtual

Reimplemented in Nektar::LibUtilities::CommMpi.

Definition at line 69 of file Communication/Comm.cpp.

{
    return std::make_pair(CommSharedPtr(), CommSharedPtr());
}

Referenced by SplitCommNode().

v_StartAll()

virtual void Nektar::LibUtilities::Comm::v_StartAll ( CommRequestSharedPtr  request )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by StartAll().

v_TreatAsRankZero()

virtual bool Nektar::LibUtilities::Comm::v_TreatAsRankZero ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, Nektar::LibUtilities::CommSerial, and Nektar::FieldUtils::FieldConvertComm.

Referenced by TreatAsRankZero().

v_WaitAll()

virtual void Nektar::LibUtilities::Comm::v_WaitAll ( CommRequestSharedPtr  request )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by WaitAll().

v_Wtime()

virtual NekDouble Nektar::LibUtilities::Comm::v_Wtime ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Wtime().

WaitAll()

void Nektar::LibUtilities::Comm::WaitAll ( const CommRequestSharedPtr &  request )

inline

Waits for all CommRequests in the request object to complete.

Parameters

request

Communication request object

Definition at line 702 of file Comm.h.

{
    v_WaitAll(request);
}

References v_WaitAll().

Wtime()

double Nektar::LibUtilities::Comm::Wtime ( )

inline

Return the time in seconds.

Definition at line 328 of file Comm.h.

{
    return v_Wtime();
}

References v_Wtime().

Member Data Documentation

m_commColumn

CommSharedPtr Nektar::LibUtilities::Comm::m_commColumn

protected

Column communicator.

Definition at line 178 of file Comm.h.

Referenced by GetColumnComm(), Nektar::FieldUtils::FieldConvertComm::v_SplitComm(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

m_commRow

CommSharedPtr Nektar::LibUtilities::Comm::m_commRow

protected

Row communicator.

Definition at line 177 of file Comm.h.

Referenced by GetRowComm(), Nektar::FieldUtils::FieldConvertComm::v_SplitComm(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

m_commSpace

CommSharedPtr Nektar::LibUtilities::Comm::m_commSpace

protected

Definition at line 180 of file Comm.h.

Referenced by GetSpaceComm(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

m_commTime

CommSharedPtr Nektar::LibUtilities::Comm::m_commTime

protected

Definition at line 179 of file Comm.h.

Referenced by GetTimeComm(), IsParallelInTime(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

m_size

int Nektar::LibUtilities::Comm::m_size

protected

Number of processes.

Definition at line 175 of file Comm.h.

Referenced by Nektar::LibUtilities::CommCwipi::CommCwipi(), Nektar::LibUtilities::CommMpi::CommMpi(), Nektar::LibUtilities::CommSerial::CommSerial(), Nektar::FieldUtils::FieldConvertComm::FieldConvertComm(), GetSize(), Nektar::LibUtilities::CommMpi::v_IsSerial(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

m_type

std::string Nektar::LibUtilities::Comm::m_type

protected

Type of communication.

Definition at line 176 of file Comm.h.

Referenced by Nektar::LibUtilities::CommCwipi::CommCwipi(), Nektar::LibUtilities::CommMpi::CommMpi(), Nektar::LibUtilities::CommSerial::CommSerial(), Nektar::FieldUtils::FieldConvertComm::FieldConvertComm(), and GetType().