Nektar::LibUtilities::Comm Class Reference

Base communications class. More...

#include <Comm.h>

Inheritance diagram for Nektar::LibUtilities::Comm:

Public Member Functions
	Comm (int narg, char *arg[])
virtual	~Comm ()
void	Finalise ()
int	GetSize () const
	Returns number of processes.
int	GetRank ()
const std::string &	GetType () const
bool	TreatAsRankZero ()
bool	IsSerial ()
bool	IsParallelInTime ()
std::tuple< int, int, int >	GetVersion ()
void	Block ()
	Block execution until all processes reach this point.
NekDouble	Wtime ()
	Return the time in seconds.
template<class T >
void	Send (int pProc, T &pData)
template<class T >
void	Recv (int pProc, T &pData)
template<class T >
void	SendRecv (int pSendProc, T &pSendData, int pRecvProc, T &pRecvData)
template<class T >
void	AllReduce (T &pData, enum ReduceOperator pOp)
template<class T >
void	AlltoAll (T &pSendData, T &pRecvData)
template<class T1 , class T2 >
void	AlltoAllv (T1 &pSendData, T2 &pSendDataSizeMap, T2 &pSendDataOffsetMap, T1 &pRecvData, T2 &pRecvDataSizeMap, T2 &pRecvDataOffsetMap)
template<class T >
void	AllGather (T &pSendData, T &pRecvData)
template<class T >
void	AllGatherv (T &pSendData, T &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)
template<class T >
void	AllGatherv (T &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)
template<class T >
void	Bcast (T &pData, int pRoot)
template<class T >
T	Gather (int rootProc, T &val)
template<class T >
T	Scatter (int rootProc, T &pData)
template<class T >
void	DistGraphCreateAdjacent (T &sources, T &sourceweights, int reorder)
template<class T1 , class T2 >
void	NeighborAlltoAllv (T1 &pSendData, T2 &pSendDataSizeMap, T2 &pSendDataOffsetMap, T1 &pRecvData, T2 &pRecvDataSizeMap, T2 &pRecvDataOffsetMap)
template<class T >
void	Irsend (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	Isend (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	SendInit (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	Irecv (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	RecvInit (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
void	StartAll (const CommRequestSharedPtr &request)
void	WaitAll (const CommRequestSharedPtr &request)
CommRequestSharedPtr	CreateRequest (int num)
CommSharedPtr	CommCreateIf (int flag)
	If the flag is non-zero create a new communicator.
void	SplitComm (int pRows, int pColumns, int pTime=0)
	Splits this communicator into a grid of size pRows*pColumns and creates row and column communicators. By default the communicator is a single row.
CommSharedPtr	GetRowComm ()
	Retrieve the row communicator to which this process belongs.
CommSharedPtr	GetColumnComm ()
	Retrieve the column communicator to which this process belongs.
CommSharedPtr	GetTimeComm ()
	Retrieve the time communicator to which this process belongs.
CommSharedPtr	GetSpaceComm ()
	Retrieve the space communicator to which this process belongs.
bool	RemoveExistingFiles ()
std::pair< CommSharedPtr, CommSharedPtr >	SplitCommNode ()

Protected Member Functions
	Comm ()
virtual void	v_Finalise ()=0
virtual int	v_GetRank ()=0
virtual bool	v_TreatAsRankZero ()=0
virtual bool	v_IsSerial ()=0
virtual std::tuple< int, int, int >	v_GetVersion ()=0
virtual void	v_Block ()=0
virtual NekDouble	v_Wtime ()=0
virtual void	v_Send (const void *buf, int count, CommDataType dt, int dest)=0
virtual void	v_Recv (void *buf, int count, CommDataType dt, int source)=0
virtual void	v_SendRecv (const void *sendbuf, int sendcount, CommDataType sendtype, int dest, void *recvbuf, int recvcount, CommDataType recvtype, int source)=0
virtual void	v_AllReduce (void *buf, int count, CommDataType dt, enum ReduceOperator pOp)=0
virtual void	v_AlltoAll (const void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype)=0
virtual void	v_AlltoAllv (const void *sendbuf, const int *sendcounts, const int *senddispls, CommDataType sendtype, void *recvbuf, const int *recvcounts, const int *recvdispls, CommDataType recvtype)=0
virtual void	v_AllGather (const void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype)=0
virtual void	v_AllGatherv (const void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, const int *recvcounts, const int *recvdispls, CommDataType recvtype)=0
virtual void	v_AllGatherv (void *recvbuf, const int *recvcounts, const int *recvdispls, CommDataType recvtype)=0
virtual void	v_Bcast (void *buffer, int count, CommDataType dt, int root)=0
virtual void	v_Gather (const void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype, int root)=0
virtual void	v_Scatter (const void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype, int root)=0
virtual void	v_DistGraphCreateAdjacent (int indegree, const int *sources, const int *sourceweights, int reorder)=0
virtual void	v_NeighborAlltoAllv (const void *sendbuf, const int *sendcounts, const int *senddispls, CommDataType sendtype, void *recvbuf, const int *recvcounts, const int *recvdispls, CommDataType recvtype)=0
virtual void	v_Irsend (const void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc)=0
virtual void	v_Isend (const void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc)=0
virtual void	v_SendInit (const void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc)=0
virtual void	v_Irecv (void *buf, int count, CommDataType dt, int source, CommRequestSharedPtr request, int loc)=0
virtual void	v_RecvInit (void *buf, int count, CommDataType dt, int source, CommRequestSharedPtr request, int loc)=0
virtual void	v_StartAll (CommRequestSharedPtr request)=0
virtual void	v_WaitAll (CommRequestSharedPtr request)=0
virtual CommRequestSharedPtr	v_CreateRequest (int num)=0
virtual void	v_SplitComm (int pRows, int pColumns, int pTime)=0
virtual CommSharedPtr	v_CommCreateIf (int flag)=0
virtual std::pair< CommSharedPtr, CommSharedPtr >	v_SplitCommNode ()

Protected Attributes
int	m_size
	Number of processes.
std::string	m_type
	Type of communication.
CommSharedPtr	m_commRow
	Row communicator.
CommSharedPtr	m_commColumn
	Column communicator.
CommSharedPtr	m_commTime
CommSharedPtr	m_commSpace

Detailed Description

Base communications class.

Definition at line 87 of file Comm.h.

Constructor & Destructor Documentation

Comm() [1/2]

Nektar::LibUtilities::Comm::Comm	(	int	narg,
		char *	arg[]
	)

Definition at line 39 of file Communication/Comm.cpp.

{
}

~Comm()

Nektar::LibUtilities::Comm::~Comm ( )

virtual

Definition at line 53 of file Communication/Comm.cpp.

{
}

Comm() [2/2]

Nektar::LibUtilities::Comm::Comm ( )

protected

Definition at line 46 of file Communication/Comm.cpp.

{
}

Member Function Documentation

AllGather()

template<class T >

void Nektar::LibUtilities::Comm::AllGather	(	T &	pSendData,
		T &	pRecvData
	)

Definition at line 430 of file Comm.h.

{
    BOOST_STATIC_ASSERT_MSG(
        CommDataTypeTraits<T>::IsVector,
        "AllGather only valid with Array or vector arguments.");
 
    int sendSize = CommDataTypeTraits<T>::GetCount(pSendData);
    int recvSize = sendSize;
 
    pRecvData = T(recvSize * GetSize());
 
    v_AllGather(CommDataTypeTraits<T>::GetPointer(pSendData), sendSize,
                CommDataTypeTraits<T>::GetDataType(),
                CommDataTypeTraits<T>::GetPointer(pRecvData), recvSize,
                CommDataTypeTraits<T>::GetDataType());
}

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), GetSize(), and v_AllGather().

AllGatherv() [1/2]

template<class T >

void Nektar::LibUtilities::Comm::AllGatherv	(	T &	pRecvData,
		Array< OneD, int > &	pRecvDataSizeMap,
		Array< OneD, int > &	pRecvDataOffsetMap
	)

Definition at line 472 of file Comm.h.

{
    BOOST_STATIC_ASSERT_MSG(
        CommDataTypeTraits<T>::IsVector,
        "AllGatherv only valid with Array or vector arguments.");
 
    v_AllGatherv(CommDataTypeTraits<T>::GetPointer(pRecvData),
                 pRecvDataSizeMap.data(), pRecvDataOffsetMap.data(),
                 CommDataTypeTraits<T>::GetDataType());
}

References v_AllGatherv().

AllGatherv() [2/2]

template<class T >

void Nektar::LibUtilities::Comm::AllGatherv	(	T &	pSendData,
		T &	pRecvData,
		Array< OneD, int > &	pRecvDataSizeMap,
		Array< OneD, int > &	pRecvDataOffsetMap
	)

Definition at line 451 of file Comm.h.

{
    BOOST_STATIC_ASSERT_MSG(
        CommDataTypeTraits<T>::IsVector,
        "AllGatherv only valid with Array or vector arguments.");
 
    int sendSize = CommDataTypeTraits<T>::GetCount(pSendData);
 
    v_AllGatherv(CommDataTypeTraits<T>::GetPointer(pSendData), sendSize,
                 CommDataTypeTraits<T>::GetDataType(),
                 CommDataTypeTraits<T>::GetPointer(pRecvData),
                 pRecvDataSizeMap.data(), pRecvDataOffsetMap.data(),
                 CommDataTypeTraits<T>::GetDataType());
}

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), and v_AllGatherv().

AllReduce()

template<class T >

void Nektar::LibUtilities::Comm::AllReduce	(	T &	pData,
		enum ReduceOperator	pOp
	)

Definition at line 375 of file Comm.h.

{
    v_AllReduce(CommDataTypeTraits<T>::GetPointer(pData),
                CommDataTypeTraits<T>::GetCount(pData),
                CommDataTypeTraits<T>::GetDataType(), pOp);
}

References v_AllReduce().

AlltoAll()

template<class T >

void Nektar::LibUtilities::Comm::AlltoAll	(	T &	pSendData,
		T &	pRecvData
	)

Definition at line 385 of file Comm.h.

{
    static_assert(CommDataTypeTraits<T>::IsVector,
                  "AlltoAll only valid with Array or vector arguments.");
    int sendSize = CommDataTypeTraits<T>::GetCount(pSendData);
    int recvSize = CommDataTypeTraits<T>::GetCount(pRecvData);
    ASSERTL0(sendSize == recvSize,
             "Send and Recv arrays have incompatible sizes in AlltoAll");
 
    int count = sendSize / GetSize();
    ASSERTL0(count * GetSize() == sendSize,
             "Array size incompatible with size of communicator");
 
    v_AlltoAll(CommDataTypeTraits<T>::GetPointer(pSendData), count,
               CommDataTypeTraits<T>::GetDataType(),
               CommDataTypeTraits<T>::GetPointer(pRecvData), count,
               CommDataTypeTraits<T>::GetDataType());
}

References ASSERTL0, Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), GetSize(), and v_AlltoAll().

AlltoAllv()

template<class T1 , class T2 >

void Nektar::LibUtilities::Comm::AlltoAllv	(	T1 &	pSendData,
		T2 &	pSendDataSizeMap,
		T2 &	pSendDataOffsetMap,
		T1 &	pRecvData,
		T2 &	pRecvDataSizeMap,
		T2 &	pRecvDataOffsetMap
	)

Definition at line 408 of file Comm.h.

{
    static_assert(CommDataTypeTraits<T1>::IsVector,
                  "AlltoAllv only valid with Array or vector arguments.");
    static_assert(std::is_same_v<T2, std::vector<int>> ||
                      std::is_same_v<T2, Array<OneD, int>>,
                  "Alltoallv size and offset maps should be integer vectors.");
    v_AlltoAllv(CommDataTypeTraits<T1>::GetPointer(pSendData),
                (int *)CommDataTypeTraits<T2>::GetPointer(pSendDataSizeMap),
                (int *)CommDataTypeTraits<T2>::GetPointer(pSendDataOffsetMap),
                CommDataTypeTraits<T1>::GetDataType(),
                CommDataTypeTraits<T1>::GetPointer(pRecvData),
                (int *)CommDataTypeTraits<T2>::GetPointer(pRecvDataSizeMap),
                (int *)CommDataTypeTraits<T2>::GetPointer(pRecvDataOffsetMap),
                CommDataTypeTraits<T1>::GetDataType());
}

References v_AlltoAllv().

Bcast()

template<class T >

void Nektar::LibUtilities::Comm::Bcast	(	T &	pData,
		int	pRoot
	)

Definition at line 487 of file Comm.h.

{
    v_Bcast(CommDataTypeTraits<T>::GetPointer(pData),
            CommDataTypeTraits<T>::GetCount(pData),
            CommDataTypeTraits<T>::GetDataType(), pRoot);
}

References v_Bcast().

Block()

void Nektar::LibUtilities::Comm::Block ( )

inline

Block execution until all processes reach this point.

Definition at line 325 of file Comm.h.

{
    v_Block();
}

References v_Block().

CommCreateIf()

CommSharedPtr Nektar::LibUtilities::Comm::CommCreateIf ( int  flag )

inline

If the flag is non-zero create a new communicator.

Definition at line 728 of file Comm.h.

{
    return v_CommCreateIf(flag);
}

References v_CommCreateIf().

CreateRequest()

CommRequestSharedPtr Nektar::LibUtilities::Comm::CreateRequest ( int  num )

inline

Creates a number of CommRequests.

Parameters

num	Number of requests to generate in the communication request object

Returns

Communication request object

Definition at line 720 of file Comm.h.

{
    return v_CreateRequest(num);
}

References v_CreateRequest().

DistGraphCreateAdjacent()

template<class T >

void Nektar::LibUtilities::Comm::DistGraphCreateAdjacent	(	T &	sources,
		T &	sourceweights,
		int	reorder
	)

This replaces the current MPI communicator with a new one that also holds the distributed graph topology information. If reordering is enabled using this might break code where process/rank numbers are assumed to remain constant. This also assumes that the graph is bi-directional, so all sources are also destinations with equal weighting.

Parameters

sources	Ranks of processes for which the calling process is the destination/source
sourceweights	Weights of the corresponding edges into the calling process
reorder	Ranks may be reordered (true) or not (false)

Definition at line 550 of file Comm.h.

{
    static_assert(
        CommDataTypeTraits<T>::IsVector,
        "DistGraphCreateAdjacent only valid with Array or vector arguments.");
 
    ASSERTL0(CommDataTypeTraits<T>::GetCount(sources) ==
                 CommDataTypeTraits<T>::GetCount(sourceweights),
             "Sources and weights array sizes don't match");
 
    int indegree = CommDataTypeTraits<T>::GetCount(sources);
 
    v_DistGraphCreateAdjacent(
        indegree, (const int *)CommDataTypeTraits<T>::GetPointer(sources),
        (const int *)CommDataTypeTraits<T>::GetPointer(sourceweights), reorder);
}

References ASSERTL0, Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), and v_DistGraphCreateAdjacent().

Finalise()

void Nektar::LibUtilities::Comm::Finalise ( )

inline

Definition at line 261 of file Comm.h.

{
    v_Finalise();
}

References v_Finalise().

Gather()

template<class T >

T Nektar::LibUtilities::Comm::Gather	(	int	rootProc,
		T &	val
	)

Concatenate all the input arrays, in rank order, onto the process with rank == rootProc

Definition at line 498 of file Comm.h.

{
    static_assert(CommDataTypeTraits<T>::IsVector,
                  "Gather only valid with Array or vector arguments.");
    bool amRoot  = (GetRank() == rootProc);
    unsigned nEl = CommDataTypeTraits<T>::GetCount(val);
 
    unsigned nOut = amRoot ? GetSize() * nEl : 0;
    T ans(nOut);
    void *recvbuf = amRoot ? CommDataTypeTraits<T>::GetPointer(ans) : nullptr;
 
    v_Gather(CommDataTypeTraits<T>::GetPointer(val), nEl,
             CommDataTypeTraits<T>::GetDataType(), recvbuf, nEl,
             CommDataTypeTraits<T>::GetDataType(), rootProc);
    return ans;
}

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), Nektar::LibUtilities::CommDataTypeTraits< T >::GetPointer(), GetRank(), GetSize(), and v_Gather().

GetColumnComm()

CommSharedPtr Nektar::LibUtilities::Comm::GetColumnComm ( )

inline

Retrieve the column communicator to which this process belongs.

Definition at line 762 of file Comm.h.

{
    if (!m_commColumn.get())
    {
        return shared_from_this();
    }
    else
    {
        return m_commColumn;
    }
}

References m_commColumn.

GetRank()

int Nektar::LibUtilities::Comm::GetRank ( )

inline

Definition at line 277 of file Comm.h.

{
    return v_GetRank();
}

References v_GetRank().

Referenced by export_Comm(), Gather(), and Scatter().

GetRowComm()

CommSharedPtr Nektar::LibUtilities::Comm::GetRowComm ( )

inline

Retrieve the row communicator to which this process belongs.

Definition at line 746 of file Comm.h.

{
    if (!m_commRow.get())
    {
        return shared_from_this();
    }
    else
    {
        return m_commRow;
    }
}

References m_commRow.

GetSize()

int Nektar::LibUtilities::Comm::GetSize ( ) const

inline

Returns number of processes.

Definition at line 269 of file Comm.h.

{
    return m_size;
}

References m_size.

Referenced by AllGather(), AlltoAll(), export_Comm(), Gather(), Scatter(), and Nektar::LibUtilities::CommMpi::v_AllReduce().

GetSpaceComm()

CommSharedPtr Nektar::LibUtilities::Comm::GetSpaceComm ( )

inline

Retrieve the space communicator to which this process belongs.

Definition at line 794 of file Comm.h.

{
    if (!m_commSpace.get())
    {
        return shared_from_this();
    }
    else
    {
        return m_commSpace;
    }
}

References m_commSpace.

GetTimeComm()

CommSharedPtr Nektar::LibUtilities::Comm::GetTimeComm ( )

inline

Retrieve the time communicator to which this process belongs.

Definition at line 778 of file Comm.h.

{
    if (!m_commTime.get())
    {
        return shared_from_this();
    }
    else
    {
        return m_commTime;
    }
}

References m_commTime.

GetType()

const std::string & Nektar::LibUtilities::Comm::GetType ( ) const

inline

Definition at line 285 of file Comm.h.

{
    return m_type;
}

References m_type.

Referenced by export_Comm().

GetVersion()

std::tuple< int, int, int > Nektar::LibUtilities::Comm::GetVersion ( )

inline

Returns

tuple of {major, minor, patch} version numbers

Definition at line 317 of file Comm.h.

{
    return v_GetVersion();
}

References v_GetVersion().

Irecv()

template<class T >

void Nektar::LibUtilities::Comm::Irecv	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Begins a nonblocking receive

Parameters

pProc	Rank of source
pData	Array/vector to place incoming data in to
count	Number of elements to receive in to pData
request	Communication request object
loc	Location in request to use

Definition at line 669 of file Comm.h.

{
    v_Irecv(CommDataTypeTraits<T>::GetPointer(pData), count,
            CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References v_Irecv().

Irsend()

template<class T >

void Nektar::LibUtilities::Comm::Irsend	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Starts a ready-mode nonblocking send

Parameters

pProc	Rank of destination
pData	Array/vector to send
count	Number of elements to send in pData
request	Communication request object
loc	Location in request to use

Definition at line 618 of file Comm.h.

{
    v_Irsend(CommDataTypeTraits<T>::GetPointer(pData), count,
             CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References v_Irsend().

Isend()

template<class T >

void Nektar::LibUtilities::Comm::Isend	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Starts a nonblocking send

Parameters

pProc	Rank of destination
pData	Array/vector to send
count	Number of elements to send in pData
request	Communication request object
loc	Location in request to use

Definition at line 635 of file Comm.h.

{
    v_Isend(CommDataTypeTraits<T>::GetPointer(pData), count,
            CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References v_Isend().

IsParallelInTime()

bool Nektar::LibUtilities::Comm::IsParallelInTime ( )

inline

Definition at line 309 of file Comm.h.

{
    return m_commTime.get();
}

References m_commTime.

IsSerial()

bool Nektar::LibUtilities::Comm::IsSerial ( )

inline

Definition at line 301 of file Comm.h.

{
    return v_IsSerial();
}

References v_IsSerial().

NeighborAlltoAllv()

template<class T1 , class T2 >

void Nektar::LibUtilities::Comm::NeighborAlltoAllv	(	T1 &	pSendData,
		T2 &	pSendDataSizeMap,
		T2 &	pSendDataOffsetMap,
		T1 &	pRecvData,
		T2 &	pRecvDataSizeMap,
		T2 &	pRecvDataOffsetMap
	)

Sends data to neighboring processes in a virtual topology communicator. All processes send different amounts of data to, and receive different amounts of data from, all neighbors

Parameters

pSendData	Array/vector to send to neighbors
pSendDataSizeMap	Array/vector where entry i specifies the number of elements to send to neighbor i
pSendDataOffsetMap	Array/vector where entry i specifies the displacement (offset from pSendData) from which to send data to neighbor i
pRecvData	Array/vector to place incoming data in to
pRecvDataSizeMap	Array/vector where entry i specifies the number of elements to receive from neighbor i
pRecvDataOffsetMap	Array/vector where entry i specifies the displacement (offset from pRecvData) from which to receive data from neighbor i

Definition at line 586 of file Comm.h.

{
    static_assert(
        CommDataTypeTraits<T1>::IsVector,
        "NeighbourAlltoAllv only valid with Array or vector arguments.");
    static_assert(
        std::is_same_v<T2, std::vector<int>> ||
            std::is_same_v<T2, Array<OneD, int>>,
        "NeighborAllToAllv size and offset maps should be integer vectors.");
    v_NeighborAlltoAllv(
        CommDataTypeTraits<T1>::GetPointer(pSendData),
        (int *)CommDataTypeTraits<T2>::GetPointer(pSendDataSizeMap),
        (int *)CommDataTypeTraits<T2>::GetPointer(pSendDataOffsetMap),
        CommDataTypeTraits<T1>::GetDataType(),
        CommDataTypeTraits<T1>::GetPointer(pRecvData),
        (int *)CommDataTypeTraits<T2>::GetPointer(pRecvDataSizeMap),
        (int *)CommDataTypeTraits<T2>::GetPointer(pRecvDataOffsetMap),
        CommDataTypeTraits<T1>::GetDataType());
}

References v_NeighborAlltoAllv().

Recv()

template<class T >

void Nektar::LibUtilities::Comm::Recv	(	int	pProc,
		T &	pData
	)

Definition at line 351 of file Comm.h.

{
    v_Recv(CommDataTypeTraits<T>::GetPointer(pData),
           CommDataTypeTraits<T>::GetCount(pData),
           CommDataTypeTraits<T>::GetDataType(), pProc);
}

References v_Recv().

RecvInit()

template<class T >

void Nektar::LibUtilities::Comm::RecvInit	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Create a persistent request for a receive

Parameters

pProc	Rank of source
pData	Array/vector to place incoming data in to
count	Number of elements to receive in to pData
request	Communication request object
loc	Location in request to use

Definition at line 686 of file Comm.h.

{
    v_RecvInit(CommDataTypeTraits<T>::GetPointer(pData), count,
               CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References v_RecvInit().

RemoveExistingFiles()

bool Nektar::LibUtilities::Comm::RemoveExistingFiles ( )

inline

Definition at line 809 of file Comm.h.

{
    return true;
}

Scatter()

template<class T >

T Nektar::LibUtilities::Comm::Scatter	(	int	rootProc,
		T &	pData
	)

Scatter pData across ranks in chunks of len(pData)/num_ranks

Definition at line 518 of file Comm.h.

{
    static_assert(CommDataTypeTraits<T>::IsVector,
                  "Scatter only valid with Array or vector arguments.");
 
    bool amRoot  = (GetRank() == rootProc);
    unsigned nEl = CommDataTypeTraits<T>::GetCount(pData) / GetSize();
 
    const void *sendbuf =
        amRoot ? CommDataTypeTraits<T>::GetPointer(pData) : nullptr;
    T ans(nEl);
 
    v_Scatter(sendbuf, nEl, CommDataTypeTraits<T>::GetDataType(),
              CommDataTypeTraits<T>::GetPointer(ans), nEl,
              CommDataTypeTraits<T>::GetDataType(), rootProc);
    return ans;
}

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), Nektar::LibUtilities::CommDataTypeTraits< T >::GetPointer(), GetRank(), GetSize(), and v_Scatter().

Send()

template<class T >

void Nektar::LibUtilities::Comm::Send	(	int	pProc,
		T &	pData
	)

Definition at line 341 of file Comm.h.

{
    v_Send(CommDataTypeTraits<T>::GetPointer(pData),
           CommDataTypeTraits<T>::GetCount(pData),
           CommDataTypeTraits<T>::GetDataType(), pProc);
}

References v_Send().

SendInit()

template<class T >

void Nektar::LibUtilities::Comm::SendInit	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Creates a persistent request for a send

Parameters

pProc	Rank of destination
pData	Array/vector to send
count	Number of elements to send in pData
request	Communication request object
loc	Location in request to use

Definition at line 652 of file Comm.h.

{
    v_SendInit(CommDataTypeTraits<T>::GetPointer(pData), count,
               CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References v_SendInit().

SendRecv()

template<class T >

void Nektar::LibUtilities::Comm::SendRecv	(	int	pSendProc,
		T &	pSendData,
		int	pRecvProc,
		T &	pRecvData
	)

Definition at line 362 of file Comm.h.

{
    v_SendRecv(CommDataTypeTraits<T>::GetPointer(pSendData),
               CommDataTypeTraits<T>::GetCount(pSendData),
               CommDataTypeTraits<T>::GetDataType(), pSendProc,
               CommDataTypeTraits<T>::GetPointer(pRecvData),
               CommDataTypeTraits<T>::GetCount(pRecvData),
               CommDataTypeTraits<T>::GetDataType(), pRecvProc);
}

References v_SendRecv().

SplitComm()

void Nektar::LibUtilities::Comm::SplitComm ( int  pRows, int  pColumns, int  pTime = 0  )

inline

Splits this communicator into a grid of size pRows*pColumns and creates row and column communicators. By default the communicator is a single row.

Definition at line 738 of file Comm.h.

{
    v_SplitComm(pRows, pColumns, pTime);
}

References v_SplitComm().

SplitCommNode()

std::pair< CommSharedPtr, CommSharedPtr > Nektar::LibUtilities::Comm::SplitCommNode ( )

inline

Definition at line 817 of file Comm.h.

{
    return v_SplitCommNode();
}

References v_SplitCommNode().

StartAll()

void Nektar::LibUtilities::Comm::StartAll ( const CommRequestSharedPtr &  request )

inline

Starts a collection of persistent requests

Parameters

request

Communication request object

Definition at line 698 of file Comm.h.

{
    v_StartAll(request);
}

References v_StartAll().

TreatAsRankZero()

bool Nektar::LibUtilities::Comm::TreatAsRankZero ( )

inline

Definition at line 293 of file Comm.h.

{
    return v_TreatAsRankZero();
}

References v_TreatAsRankZero().

v_AllGather()

virtual void Nektar::LibUtilities::Comm::v_AllGather ( const void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by AllGather().

v_AllGatherv() [1/2]

virtual void Nektar::LibUtilities::Comm::v_AllGatherv ( const void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, const int *  recvcounts, const int *  recvdispls, CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by AllGatherv(), and AllGatherv().

v_AllGatherv() [2/2]

virtual void Nektar::LibUtilities::Comm::v_AllGatherv ( void *  recvbuf, const int *  recvcounts, const int *  recvdispls, CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

v_AllReduce()

virtual void Nektar::LibUtilities::Comm::v_AllReduce ( void *  buf, int  count, CommDataType  dt, enum ReduceOperator  pOp  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by AllReduce().

v_AlltoAll()

virtual void Nektar::LibUtilities::Comm::v_AlltoAll ( const void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by AlltoAll().

v_AlltoAllv()

virtual void Nektar::LibUtilities::Comm::v_AlltoAllv ( const void *  sendbuf, const int *  sendcounts, const int *  senddispls, CommDataType  sendtype, void *  recvbuf, const int *  recvcounts, const int *  recvdispls, CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by AlltoAllv().

v_Bcast()

virtual void Nektar::LibUtilities::Comm::v_Bcast ( void *  buffer, int  count, CommDataType  dt, int  root  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Bcast().

v_Block()

virtual void Nektar::LibUtilities::Comm::v_Block ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Block().

v_CommCreateIf()

virtual CommSharedPtr Nektar::LibUtilities::Comm::v_CommCreateIf ( int  flag )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by CommCreateIf().

v_CreateRequest()

virtual CommRequestSharedPtr Nektar::LibUtilities::Comm::v_CreateRequest ( int  num )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by CreateRequest().

v_DistGraphCreateAdjacent()

virtual void Nektar::LibUtilities::Comm::v_DistGraphCreateAdjacent ( int  indegree, const int *  sources, const int *  sourceweights, int  reorder  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by DistGraphCreateAdjacent().

v_Finalise()

virtual void Nektar::LibUtilities::Comm::v_Finalise ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, Nektar::LibUtilities::CommCwipi, and Nektar::LibUtilities::CommMpi.

Referenced by Finalise().

v_Gather()

virtual void Nektar::LibUtilities::Comm::v_Gather ( const void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype, int  root  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Gather().

v_GetRank()

virtual int Nektar::LibUtilities::Comm::v_GetRank ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, Nektar::LibUtilities::CommSerial, and Nektar::FieldUtils::FieldConvertComm.

Referenced by GetRank().

v_GetVersion()

virtual std::tuple< int, int, int > Nektar::LibUtilities::Comm::v_GetVersion ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by GetVersion().

v_Irecv()

virtual void Nektar::LibUtilities::Comm::v_Irecv ( void *  buf, int  count, CommDataType  dt, int  source, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Irecv().

v_Irsend()

virtual void Nektar::LibUtilities::Comm::v_Irsend ( const void *  buf, int  count, CommDataType  dt, int  dest, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Irsend().

v_Isend()

virtual void Nektar::LibUtilities::Comm::v_Isend ( const void *  buf, int  count, CommDataType  dt, int  dest, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Isend().

v_IsSerial()

virtual bool Nektar::LibUtilities::Comm::v_IsSerial ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, Nektar::LibUtilities::CommSerial, and Nektar::FieldUtils::FieldConvertComm.

Referenced by IsSerial().

v_NeighborAlltoAllv()

virtual void Nektar::LibUtilities::Comm::v_NeighborAlltoAllv ( const void *  sendbuf, const int *  sendcounts, const int *  senddispls, CommDataType  sendtype, void *  recvbuf, const int *  recvcounts, const int *  recvdispls, CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by NeighborAlltoAllv().

v_Recv()

virtual void Nektar::LibUtilities::Comm::v_Recv ( void *  buf, int  count, CommDataType  dt, int  source  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Recv().

v_RecvInit()

virtual void Nektar::LibUtilities::Comm::v_RecvInit ( void *  buf, int  count, CommDataType  dt, int  source, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by RecvInit().

v_Scatter()

virtual void Nektar::LibUtilities::Comm::v_Scatter ( const void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype, int  root  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Scatter().

v_Send()

virtual void Nektar::LibUtilities::Comm::v_Send ( const void *  buf, int  count, CommDataType  dt, int  dest  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Send().

v_SendInit()

virtual void Nektar::LibUtilities::Comm::v_SendInit ( const void *  buf, int  count, CommDataType  dt, int  dest, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by SendInit().

v_SendRecv()

virtual void Nektar::LibUtilities::Comm::v_SendRecv ( const void *  sendbuf, int  sendcount, CommDataType  sendtype, int  dest, void *  recvbuf, int  recvcount, CommDataType  recvtype, int  source  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by SendRecv().

v_SplitComm()

virtual void Nektar::LibUtilities::Comm::v_SplitComm ( int  pRows, int  pColumns, int  pTime  )

protectedpure virtual

Implemented in Nektar::FieldUtils::FieldConvertComm, Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by SplitComm().

v_SplitCommNode()

std::pair< CommSharedPtr, CommSharedPtr > Nektar::LibUtilities::Comm::v_SplitCommNode ( )

protectedvirtual

Reimplemented in Nektar::LibUtilities::CommMpi.

Definition at line 69 of file Communication/Comm.cpp.

{
    return std::make_pair(CommSharedPtr(), CommSharedPtr());
}

Referenced by SplitCommNode().

v_StartAll()

virtual void Nektar::LibUtilities::Comm::v_StartAll ( CommRequestSharedPtr  request )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by StartAll().

v_TreatAsRankZero()

virtual bool Nektar::LibUtilities::Comm::v_TreatAsRankZero ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, Nektar::LibUtilities::CommSerial, and Nektar::FieldUtils::FieldConvertComm.

Referenced by TreatAsRankZero().

v_WaitAll()

virtual void Nektar::LibUtilities::Comm::v_WaitAll ( CommRequestSharedPtr  request )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by WaitAll().

v_Wtime()

virtual NekDouble Nektar::LibUtilities::Comm::v_Wtime ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Wtime().

WaitAll()

void Nektar::LibUtilities::Comm::WaitAll ( const CommRequestSharedPtr &  request )

inline

Waits for all CommRequests in the request object to complete.

Parameters

request

Communication request object

Definition at line 708 of file Comm.h.

{
    v_WaitAll(request);
}

References v_WaitAll().

Wtime()

double Nektar::LibUtilities::Comm::Wtime ( )

inline

Return the time in seconds.

Definition at line 333 of file Comm.h.

{
    return v_Wtime();
}

References v_Wtime().

Member Data Documentation

m_commColumn

CommSharedPtr Nektar::LibUtilities::Comm::m_commColumn

protected

Column communicator.

Definition at line 178 of file Comm.h.

Referenced by GetColumnComm(), Nektar::FieldUtils::FieldConvertComm::v_SplitComm(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

m_commRow

CommSharedPtr Nektar::LibUtilities::Comm::m_commRow

protected

Row communicator.

Definition at line 177 of file Comm.h.

Referenced by GetRowComm(), Nektar::FieldUtils::FieldConvertComm::v_SplitComm(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

m_commSpace

CommSharedPtr Nektar::LibUtilities::Comm::m_commSpace

protected

Definition at line 180 of file Comm.h.

Referenced by GetSpaceComm(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

m_commTime

CommSharedPtr Nektar::LibUtilities::Comm::m_commTime

protected

Definition at line 179 of file Comm.h.

Referenced by GetTimeComm(), IsParallelInTime(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

m_size

int Nektar::LibUtilities::Comm::m_size

protected

Number of processes.

Definition at line 175 of file Comm.h.

Referenced by Nektar::LibUtilities::CommCwipi::CommCwipi(), Nektar::LibUtilities::CommMpi::CommMpi(), Nektar::LibUtilities::CommMpi::CommMpi(), Nektar::LibUtilities::CommSerial::CommSerial(), Nektar::FieldUtils::FieldConvertComm::FieldConvertComm(), Nektar::FieldUtils::FieldConvertComm::FieldConvertComm(), GetSize(), Nektar::LibUtilities::CommMpi::v_IsSerial(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

m_type

std::string Nektar::LibUtilities::Comm::m_type

protected

Type of communication.

Definition at line 176 of file Comm.h.

Referenced by Nektar::LibUtilities::CommCwipi::CommCwipi(), Nektar::LibUtilities::CommMpi::CommMpi(), Nektar::LibUtilities::CommMpi::CommMpi(), Nektar::LibUtilities::CommSerial::CommSerial(), Nektar::FieldUtils::FieldConvertComm::FieldConvertComm(), Nektar::FieldUtils::FieldConvertComm::FieldConvertComm(), and GetType().