Nektar::LibUtilities::Comm Class Reference

Base communications class. More...

#include <Comm.h>

Inheritance diagram for Nektar::LibUtilities::Comm:

Public Member Functions
	Comm (int narg, char *arg[])
virtual	~Comm ()
void	Finalise ()
int	GetSize () const
	Returns number of processes. More...
int	GetRank ()
const std::string &	GetType () const
void	Block ()
	Block execution until all processes reach this point. More...
NekDouble	Wtime ()
	Return the time in seconds. More...
template<class T >
void	Send (int pProc, T &pData)
template<class T >
void	Recv (int pProc, T &pData)
template<class T >
void	SendRecv (int pSendProc, T &pSendData, int pRecvProc, T &pRecvData)
template<class T >
void	SendRecvReplace (int pSendProc, int pRecvProc, T &pData)
template<class T >
void	AllReduce (T &pData, enum ReduceOperator pOp)
template<class T >
void	AlltoAll (T &pSendData, T &pRecvData)
template<class T1 , class T2 >
void	AlltoAllv (T1 &pSendData, T2 &pSendDataSizeMap, T2 &pSendDataOffsetMap, T1 &pRecvData, T2 &pRecvDataSizeMap, T2 &pRecvDataOffsetMap)
template<class T >
void	AllGather (T &pSendData, T &pRecvData)
template<class T >
void	AllGatherv (T &pSendData, T &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)
template<class T >
void	AllGatherv (T &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)
template<class T >
void	Bcast (T &pData, int pRoot)
template<class T >
void	Exscan (T &pData, enum ReduceOperator pOp, T &ans)
template<class T >
T	Gather (int rootProc, T &val)
template<class T >
T	Scatter (int rootProc, T &pData)
template<class T >
void	DistGraphCreateAdjacent (T &sources, T &sourceweights, int reorder)
template<class T1 , class T2 >
void	NeighborAlltoAllv (T1 &pSendData, T2 &pSendDataSizeMap, T2 &pSendDataOffsetMap, T1 &pRecvData, T2 &pRecvDataSizeMap, T2 &pRecvDataOffsetMap)
template<class T >
void	Irsend (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	Isend (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	SendInit (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	Irecv (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	RecvInit (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
void	StartAll (const CommRequestSharedPtr &request)
void	WaitAll (const CommRequestSharedPtr &request)
CommRequestSharedPtr	CreateRequest (int num)
CommSharedPtr	CommCreateIf (int flag)
	If the flag is non-zero create a new communicator. More...
void	SplitComm (int pRows, int pColumns, int pTime=1)
	Splits this communicator into a grid of size pRows*pColumns and creates row and column communicators. By default the communicator is a single row. More...
CommSharedPtr	GetRowComm ()
	Retrieve the row communicator to which this process belongs. More...
CommSharedPtr	GetColumnComm ()
	Retrieve the column communicator to which this process belongs. More...
CommSharedPtr	GetTimeComm ()
	Retrieve the time communicator to which this process belongs. More...
CommSharedPtr	GetSpaceComm ()
	Retrieve the space communicator to which this process belongs. More...
bool	TreatAsRankZero ()
bool	IsSerial ()
std::tuple< int, int, int >	GetVersion ()
bool	RemoveExistingFiles ()
std::pair< CommSharedPtr, CommSharedPtr >	SplitCommNode ()

Protected Member Functions
	Comm ()
virtual void	v_Finalise ()=0
virtual int	v_GetRank ()=0
virtual void	v_Block ()=0
virtual NekDouble	v_Wtime ()=0
virtual void	v_Send (void *buf, int count, CommDataType dt, int dest)=0
virtual void	v_Recv (void *buf, int count, CommDataType dt, int source)=0
virtual void	v_SendRecv (void *sendbuf, int sendcount, CommDataType sendtype, int dest, void *recvbuf, int recvcount, CommDataType recvtype, int source)=0
virtual void	v_SendRecvReplace (void *buf, int count, CommDataType dt, int pSendProc, int pRecvProc)=0
virtual void	v_AllReduce (void *buf, int count, CommDataType dt, enum ReduceOperator pOp)=0
virtual void	v_AlltoAll (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype)=0
virtual void	v_AlltoAllv (void *sendbuf, int sendcounts[], int sensdispls[], CommDataType sendtype, void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype)=0
virtual void	v_AllGather (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype)=0
virtual void	v_AllGatherv (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype)=0
virtual void	v_AllGatherv (void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype)=0
virtual void	v_Bcast (void *buffer, int count, CommDataType dt, int root)=0
virtual void	v_Exscan (Array< OneD, unsigned long long > &pData, enum ReduceOperator pOp, Array< OneD, unsigned long long > &ans)=0
virtual void	v_Gather (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype, int root)=0
virtual void	v_Scatter (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype, int root)=0
virtual CommSharedPtr	v_CommCreateIf (int flag)=0
virtual void	v_DistGraphCreateAdjacent (int indegree, const int sources[], const int sourceweights[], int reorder)=0
virtual void	v_NeighborAlltoAllv (void *sendbuf, int sendcounts[], int sdispls[], CommDataType sendtype, void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype)=0
virtual void	v_Irsend (void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc)=0
virtual void	v_Isend (void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc)=0
virtual void	v_SendInit (void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc)=0
virtual void	v_Irecv (void *buf, int count, CommDataType dt, int source, CommRequestSharedPtr request, int loc)=0
virtual void	v_RecvInit (void *buf, int count, CommDataType dt, int source, CommRequestSharedPtr request, int loc)=0
virtual void	v_StartAll (CommRequestSharedPtr request)=0
virtual void	v_WaitAll (CommRequestSharedPtr request)=0
virtual CommRequestSharedPtr	v_CreateRequest (int num)=0
virtual void	v_SplitComm (int pRows, int pColumns, int pTime)=0
virtual bool	v_TreatAsRankZero ()=0
virtual bool	v_IsSerial ()=0
virtual std::tuple< int, int, int >	v_GetVersion ()=0
virtual bool	v_RemoveExistingFiles ()
virtual std::pair< CommSharedPtr, CommSharedPtr >	v_SplitCommNode ()

Protected Attributes
int	m_size
	Number of processes. More...
std::string	m_type
	Type of communication. More...
CommSharedPtr	m_commRow
	Row communicator. More...
CommSharedPtr	m_commColumn
	Column communicator. More...
CommSharedPtr	m_commTime
CommSharedPtr	m_commSpace

Detailed Description

Base communications class.

Definition at line 89 of file Comm.h.

Constructor & Destructor Documentation

Comm() [1/2]

Nektar::LibUtilities::Comm::Comm	(	int	narg,
		char *	arg[]
	)

Definition at line 43 of file Communication/Comm.cpp.

{
    boost::ignore_unused(narg, arg);
}

~Comm()

Nektar::LibUtilities::Comm::~Comm ( )

virtual

Definition at line 52 of file Communication/Comm.cpp.

{
}

Comm() [2/2]

Nektar::LibUtilities::Comm::Comm ( )

protected

Definition at line 48 of file Communication/Comm.cpp.

{
}

Member Function Documentation

AllGather()

template<class T >

void Nektar::LibUtilities::Comm::AllGather	(	T &	pSendData,
		T &	pRecvData
	)

Definition at line 408 of file Comm.h.

{
    BOOST_STATIC_ASSERT_MSG(
        CommDataTypeTraits<T>::IsVector,
        "AllGather only valid with Array or vector arguments.");
 
    int sendSize = CommDataTypeTraits<T>::GetCount(pSendData);
    int recvSize = sendSize;
 
    pRecvData = T(recvSize * GetSize());
 
    v_AllGather(CommDataTypeTraits<T>::GetPointer(pSendData), sendSize,
                CommDataTypeTraits<T>::GetDataType(),
                CommDataTypeTraits<T>::GetPointer(pRecvData), recvSize,
                CommDataTypeTraits<T>::GetDataType());
}

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), GetSize(), and v_AllGather().

AllGatherv() [1/2]

template<class T >

void Nektar::LibUtilities::Comm::AllGatherv	(	T &	pRecvData,
		Array< OneD, int > &	pRecvDataSizeMap,
		Array< OneD, int > &	pRecvDataOffsetMap
	)

Definition at line 450 of file Comm.h.

{
    BOOST_STATIC_ASSERT_MSG(
        CommDataTypeTraits<T>::IsVector,
        "AllGatherv only valid with Array or vector arguments.");
 
    v_AllGatherv(CommDataTypeTraits<T>::GetPointer(pRecvData),
                 pRecvDataSizeMap.get(), pRecvDataOffsetMap.get(),
                 CommDataTypeTraits<T>::GetDataType());
}

References v_AllGatherv().

AllGatherv() [2/2]

template<class T >

void Nektar::LibUtilities::Comm::AllGatherv	(	T &	pSendData,
		T &	pRecvData,
		Array< OneD, int > &	pRecvDataSizeMap,
		Array< OneD, int > &	pRecvDataOffsetMap
	)

Definition at line 429 of file Comm.h.

{
    BOOST_STATIC_ASSERT_MSG(
        CommDataTypeTraits<T>::IsVector,
        "AllGatherv only valid with Array or vector arguments.");
 
    int sendSize = CommDataTypeTraits<T>::GetCount(pSendData);
 
    v_AllGatherv(CommDataTypeTraits<T>::GetPointer(pSendData), sendSize,
                 CommDataTypeTraits<T>::GetDataType(),
                 CommDataTypeTraits<T>::GetPointer(pRecvData),
                 pRecvDataSizeMap.get(), pRecvDataOffsetMap.get(),
                 CommDataTypeTraits<T>::GetDataType());
}

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), and v_AllGatherv().

AllReduce()

template<class T >

void Nektar::LibUtilities::Comm::AllReduce	(	T &	pData,
		enum ReduceOperator	pOp
	)

Definition at line 359 of file Comm.h.

{
    v_AllReduce(CommDataTypeTraits<T>::GetPointer(pData),
                CommDataTypeTraits<T>::GetCount(pData),
                CommDataTypeTraits<T>::GetDataType(), pOp);
}

References v_AllReduce().

AlltoAll()

template<class T >

void Nektar::LibUtilities::Comm::AlltoAll	(	T &	pSendData,
		T &	pRecvData
	)

Definition at line 366 of file Comm.h.

{
    static_assert(CommDataTypeTraits<T>::IsVector,
                  "AlltoAll only valid with Array or vector arguments.");
    int sendSize = CommDataTypeTraits<T>::GetCount(pSendData);
    int recvSize = CommDataTypeTraits<T>::GetCount(pRecvData);
    ASSERTL0(sendSize == recvSize,
             "Send and Recv arrays have incompatible sizes in AlltoAll");
 
    int count = sendSize / GetSize();
    ASSERTL0(count * GetSize() == sendSize,
             "Array size incompatible with size of communicator");
 
    v_AlltoAll(CommDataTypeTraits<T>::GetPointer(pSendData), count,
               CommDataTypeTraits<T>::GetDataType(),
               CommDataTypeTraits<T>::GetPointer(pRecvData), count,
               CommDataTypeTraits<T>::GetDataType());
}

References ASSERTL0, Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), GetSize(), and v_AlltoAll().

AlltoAllv()

template<class T1 , class T2 >

void Nektar::LibUtilities::Comm::AlltoAllv	(	T1 &	pSendData,
		T2 &	pSendDataSizeMap,
		T2 &	pSendDataOffsetMap,
		T1 &	pRecvData,
		T2 &	pRecvDataSizeMap,
		T2 &	pRecvDataOffsetMap
	)

Definition at line 389 of file Comm.h.

{
    static_assert(CommDataTypeTraits<T1>::IsVector,
                  "AlltoAllv only valid with Array or vector arguments.");
    static_assert(std::is_same<T2, std::vector<int>>::value ||
                      std::is_same<T2, Array<OneD, int>>::value,
                  "Alltoallv size and offset maps should be integer vectors.");
    v_AlltoAllv(CommDataTypeTraits<T1>::GetPointer(pSendData),
                (int *)CommDataTypeTraits<T2>::GetPointer(pSendDataSizeMap),
                (int *)CommDataTypeTraits<T2>::GetPointer(pSendDataOffsetMap),
                CommDataTypeTraits<T1>::GetDataType(),
                CommDataTypeTraits<T1>::GetPointer(pRecvData),
                (int *)CommDataTypeTraits<T2>::GetPointer(pRecvDataSizeMap),
                (int *)CommDataTypeTraits<T2>::GetPointer(pRecvDataOffsetMap),
                CommDataTypeTraits<T1>::GetDataType());
}

References v_AlltoAllv().

Bcast()

template<class T >

void Nektar::LibUtilities::Comm::Bcast	(	T &	pData,
		int	pRoot
	)

Definition at line 465 of file Comm.h.

{
    v_Bcast(CommDataTypeTraits<T>::GetPointer(pData),
            CommDataTypeTraits<T>::GetCount(pData),
            CommDataTypeTraits<T>::GetDataType(), pRoot);
}

References v_Bcast().

Block()

void Nektar::LibUtilities::Comm::Block ( )

inline

Block execution until all processes reach this point.

Definition at line 303 of file Comm.h.

{
    v_Block();
}

References v_Block().

CommCreateIf()

CommSharedPtr Nektar::LibUtilities::Comm::CommCreateIf ( int  flag )

inline

If the flag is non-zero create a new communicator.

Definition at line 716 of file Comm.h.

{
    return v_CommCreateIf(flag);
}

References v_CommCreateIf().

CreateRequest()

CommRequestSharedPtr Nektar::LibUtilities::Comm::CreateRequest ( int  num )

inline

Creates a number of CommRequests.

Parameters

num	Number of requests to generate in the communication request object

Returns

Communication request object

Definition at line 708 of file Comm.h.

{
    return v_CreateRequest(num);
}

References v_CreateRequest().

DistGraphCreateAdjacent()

template<class T >

void Nektar::LibUtilities::Comm::DistGraphCreateAdjacent	(	T &	sources,
		T &	sourceweights,
		int	reorder
	)

This replaces the current MPI communicator with a new one that also holds the distributed graph topology information. If reordering is enabled using this might break code where process/rank numbers are assumed to remain constant. This also assumes that the graph is bi-directional, so all sources are also destinations with equal weighting.

Parameters

sources	Ranks of processes for which the calling process is the destination/source
sourceweights	Weights of the corresponding edges into the calling process
reorder	Ranks may be reordered (true) or not (false)

Definition at line 538 of file Comm.h.

{
    static_assert(
        CommDataTypeTraits<T>::IsVector,
        "DistGraphCreateAdjacent only valid with Array or vector arguments.");
 
    ASSERTL0(CommDataTypeTraits<T>::GetCount(sources) ==
                 CommDataTypeTraits<T>::GetCount(sourceweights),
             "Sources and weights array sizes don't match");
 
    int indegree = CommDataTypeTraits<T>::GetCount(sources);
 
    v_DistGraphCreateAdjacent(
        indegree, (const int *)CommDataTypeTraits<T>::GetPointer(sources),
        (const int *)CommDataTypeTraits<T>::GetPointer(sourceweights), reorder);
}

References ASSERTL0, Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), and v_DistGraphCreateAdjacent().

Exscan()

template<class T >

void Nektar::LibUtilities::Comm::Exscan	(	T &	pData,
		enum ReduceOperator	pOp,
		T &	ans
	)

Definition at line 473 of file Comm.h.

{
    ASSERTL0(CommDataTypeTraits<T>::GetCount(pData) ==
                 CommDataTypeTraits<T>::GetCount(ans),
             "Input and output array sizes don't match");
    v_Exscan(CommDataTypeTraits<T>::GetPointer(pData),
             CommDataTypeTraits<T>::GetPointer(ans),
             CommDataTypeTraits<T>::GetCount(pData),
             CommDataTypeTraits<T>::GetDataType(), pOp);
}

References ASSERTL0, and v_Exscan().

Finalise()

void Nektar::LibUtilities::Comm::Finalise ( )

inline

Definition at line 271 of file Comm.h.

{
    v_Finalise();
}

References v_Finalise().

Gather()

template<class T >

T Nektar::LibUtilities::Comm::Gather	(	int	rootProc,
		T &	val
	)

Concatenate all the input arrays, in rank order, onto the process with rank == rootProc

Definition at line 488 of file Comm.h.

{
    static_assert(CommDataTypeTraits<T>::IsVector,
                  "Gather only valid with Array or vector arguments.");
    bool amRoot  = (GetRank() == rootProc);
    unsigned nEl = CommDataTypeTraits<T>::GetCount(val);
 
    unsigned nOut = amRoot ? GetSize() * nEl : 0;
    T ans(nOut);
    void *recvbuf = amRoot ? CommDataTypeTraits<T>::GetPointer(ans) : NULL;
 
    v_Gather(CommDataTypeTraits<T>::GetPointer(val), nEl,
             CommDataTypeTraits<T>::GetDataType(), recvbuf, nEl,
             CommDataTypeTraits<T>::GetDataType(), rootProc);
    return ans;
}

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), Nektar::LibUtilities::CommDataTypeTraits< T >::GetPointer(), GetRank(), GetSize(), and v_Gather().

GetColumnComm()

CommSharedPtr Nektar::LibUtilities::Comm::GetColumnComm ( )

inline

Retrieve the column communicator to which this process belongs.

Definition at line 750 of file Comm.h.

{
    if (!m_commColumn.get())
    {
        return shared_from_this();
    }
    else
    {
        return m_commColumn;
    }
}

References m_commColumn.

GetRank()

int Nektar::LibUtilities::Comm::GetRank ( )

inline

Definition at line 287 of file Comm.h.

{
    return v_GetRank();
}

References v_GetRank().

Referenced by export_Comm(), Gather(), and Scatter().

GetRowComm()

CommSharedPtr Nektar::LibUtilities::Comm::GetRowComm ( )

inline

Retrieve the row communicator to which this process belongs.

Definition at line 734 of file Comm.h.

{
    if (!m_commRow.get())
    {
        return shared_from_this();
    }
    else
    {
        return m_commRow;
    }
}

References m_commRow.

GetSize()

int Nektar::LibUtilities::Comm::GetSize ( ) const

inline

Returns number of processes.

Definition at line 279 of file Comm.h.

{
    return m_size;
}

References m_size.

Referenced by AllGather(), AlltoAll(), export_Comm(), Gather(), Scatter(), and Nektar::LibUtilities::CommMpi::v_AllReduce().

GetSpaceComm()

CommSharedPtr Nektar::LibUtilities::Comm::GetSpaceComm ( )

inline

Retrieve the space communicator to which this process belongs.

Definition at line 782 of file Comm.h.

{
    if (!m_commSpace.get())
    {
        return shared_from_this();
    }
    else
    {
        return m_commSpace;
    }
}

References m_commSpace.

GetTimeComm()

CommSharedPtr Nektar::LibUtilities::Comm::GetTimeComm ( )

inline

Retrieve the time communicator to which this process belongs.

Definition at line 766 of file Comm.h.

{
    if (!m_commTime.get())
    {
        return shared_from_this();
    }
    else
    {
        return m_commTime;
    }
}

References m_commTime.

GetType()

const std::string & Nektar::LibUtilities::Comm::GetType ( ) const

inline

Definition at line 295 of file Comm.h.

{
    return m_type;
}

References m_type.

Referenced by export_Comm().

GetVersion()

std::tuple< int, int, int > Nektar::LibUtilities::Comm::GetVersion ( )

inline

Returns

tuple of {major, minor, patch} version numbers

Definition at line 807 of file Comm.h.

{
    return v_GetVersion();
}

References v_GetVersion().

Irecv()

template<class T >

void Nektar::LibUtilities::Comm::Irecv	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Begins a nonblocking receive

Parameters

pProc	Rank of source
pData	Array/vector to place incoming data in to
count	Number of elements to receive in to pData
request	Communication request object
loc	Location in request to use

Definition at line 657 of file Comm.h.

{
    v_Irecv(CommDataTypeTraits<T>::GetPointer(pData), count,
            CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References CG_Iterations::loc, and v_Irecv().

Irsend()

template<class T >

void Nektar::LibUtilities::Comm::Irsend	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Starts a ready-mode nonblocking send

Parameters

pProc	Rank of destination
pData	Array/vector to send
count	Number of elements to send in pData
request	Communication request object
loc	Location in request to use

Definition at line 606 of file Comm.h.

{
    v_Irsend(CommDataTypeTraits<T>::GetPointer(pData), count,
             CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References CG_Iterations::loc, and v_Irsend().

Isend()

template<class T >

void Nektar::LibUtilities::Comm::Isend	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Starts a nonblocking send

Parameters

pProc	Rank of destination
pData	Array/vector to send
count	Number of elements to send in pData
request	Communication request object
loc	Location in request to use

Definition at line 623 of file Comm.h.

{
    v_Isend(CommDataTypeTraits<T>::GetPointer(pData), count,
            CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References CG_Iterations::loc, and v_Isend().

IsSerial()

bool Nektar::LibUtilities::Comm::IsSerial ( )

inline

Definition at line 799 of file Comm.h.

{
    return v_IsSerial();
}

References v_IsSerial().

NeighborAlltoAllv()

template<class T1 , class T2 >

void Nektar::LibUtilities::Comm::NeighborAlltoAllv	(	T1 &	pSendData,
		T2 &	pSendDataSizeMap,
		T2 &	pSendDataOffsetMap,
		T1 &	pRecvData,
		T2 &	pRecvDataSizeMap,
		T2 &	pRecvDataOffsetMap
	)

Sends data to neighboring processes in a virtual topology communicator. All processes send different amounts of data to, and receive different amounts of data from, all neighbors

Parameters

pSendData	Array/vector to send to neighbors
pSendDataSizeMap	Array/vector where entry i specifies the number of elements to send to neighbor i
pSendDataOffsetMap	Array/vector where entry i specifies the displacement (offset from pSendData) from which to send data to neighbor i
pRecvData	Array/vector to place incoming data in to
pRecvDataSizeMap	Array/vector where entry i specifies the number of elements to receive from neighbor i
pRecvDataOffsetMap	Array/vector where entry i specifies the displacement (offset from pRecvData) from which to receive data from neighbor i

Definition at line 574 of file Comm.h.

{
    static_assert(
        CommDataTypeTraits<T1>::IsVector,
        "NeighbourAlltoAllv only valid with Array or vector arguments.");
    static_assert(
        std::is_same<T2, std::vector<int>>::value ||
            std::is_same<T2, Array<OneD, int>>::value,
        "NeighborAllToAllv size and offset maps should be integer vectors.");
    v_NeighborAlltoAllv(
        CommDataTypeTraits<T1>::GetPointer(pSendData),
        (int *)CommDataTypeTraits<T2>::GetPointer(pSendDataSizeMap),
        (int *)CommDataTypeTraits<T2>::GetPointer(pSendDataOffsetMap),
        CommDataTypeTraits<T1>::GetDataType(),
        CommDataTypeTraits<T1>::GetPointer(pRecvData),
        (int *)CommDataTypeTraits<T2>::GetPointer(pRecvDataSizeMap),
        (int *)CommDataTypeTraits<T2>::GetPointer(pRecvDataOffsetMap),
        CommDataTypeTraits<T1>::GetDataType());
}

References v_NeighborAlltoAllv().

Recv()

template<class T >

void Nektar::LibUtilities::Comm::Recv	(	int	pProc,
		T &	pData
	)

Definition at line 323 of file Comm.h.

{
    v_Recv(CommDataTypeTraits<T>::GetPointer(pData),
           CommDataTypeTraits<T>::GetCount(pData),
           CommDataTypeTraits<T>::GetDataType(), pProc);
}

References v_Recv().

RecvInit()

template<class T >

void Nektar::LibUtilities::Comm::RecvInit	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Create a persistent request for a receive

Parameters

pProc	Rank of source
pData	Array/vector to place incoming data in to
count	Number of elements to receive in to pData
request	Communication request object
loc	Location in request to use

Definition at line 674 of file Comm.h.

{
    v_RecvInit(CommDataTypeTraits<T>::GetPointer(pData), count,
               CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References CG_Iterations::loc, and v_RecvInit().

RemoveExistingFiles()

bool Nektar::LibUtilities::Comm::RemoveExistingFiles ( )

inline

Definition at line 812 of file Comm.h.

{
    return v_RemoveExistingFiles();
}

References v_RemoveExistingFiles().

Scatter()

template<class T >

T Nektar::LibUtilities::Comm::Scatter	(	int	rootProc,
		T &	pData
	)

Scatter pData across ranks in chunks of len(pData)/num_ranks

Definition at line 507 of file Comm.h.

{
    static_assert(CommDataTypeTraits<T>::IsVector,
                  "Scatter only valid with Array or vector arguments.");
 
    bool amRoot  = (GetRank() == rootProc);
    unsigned nEl = CommDataTypeTraits<T>::GetCount(pData) / GetSize();
 
    void *sendbuf = amRoot ? CommDataTypeTraits<T>::GetPointer(pData) : NULL;
    T ans(nEl);
 
    v_Scatter(sendbuf, nEl, CommDataTypeTraits<T>::GetDataType(),
              CommDataTypeTraits<T>::GetPointer(ans), nEl,
              CommDataTypeTraits<T>::GetDataType(), rootProc);
    return ans;
}

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), Nektar::LibUtilities::CommDataTypeTraits< T >::GetPointer(), GetRank(), GetSize(), and v_Scatter().

Send()

template<class T >

void Nektar::LibUtilities::Comm::Send	(	int	pProc,
		T &	pData
	)

Definition at line 316 of file Comm.h.

{
    v_Send(CommDataTypeTraits<T>::GetPointer(pData),
           CommDataTypeTraits<T>::GetCount(pData),
           CommDataTypeTraits<T>::GetDataType(), pProc);
}

References v_Send().

SendInit()

template<class T >

void Nektar::LibUtilities::Comm::SendInit	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Creates a persistent request for a send

Parameters

pProc	Rank of destination
pData	Array/vector to send
count	Number of elements to send in pData
request	Communication request object
loc	Location in request to use

Definition at line 640 of file Comm.h.

{
    v_SendInit(CommDataTypeTraits<T>::GetPointer(pData), count,
               CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References CG_Iterations::loc, and v_SendInit().

SendRecv()

template<class T >

void Nektar::LibUtilities::Comm::SendRecv	(	int	pSendProc,
		T &	pSendData,
		int	pRecvProc,
		T &	pRecvData
	)

Definition at line 334 of file Comm.h.

{
    v_SendRecv(CommDataTypeTraits<T>::GetPointer(pSendData),
               CommDataTypeTraits<T>::GetCount(pSendData),
               CommDataTypeTraits<T>::GetDataType(), pSendProc,
               CommDataTypeTraits<T>::GetPointer(pRecvData),
               CommDataTypeTraits<T>::GetCount(pRecvData),
               CommDataTypeTraits<T>::GetDataType(), pRecvProc);
}

References v_SendRecv().

SendRecvReplace()

template<class T >

void Nektar::LibUtilities::Comm::SendRecvReplace	(	int	pSendProc,
		int	pRecvProc,
		T &	pData
	)

Definition at line 348 of file Comm.h.

{
    v_SendRecvReplace(CommDataTypeTraits<T>::GetPointer(pData),
                      CommDataTypeTraits<T>::GetCount(pData),
                      CommDataTypeTraits<T>::GetDataType(), pSendProc,
                      pRecvProc);
}

References v_SendRecvReplace().

SplitComm()

void Nektar::LibUtilities::Comm::SplitComm ( int  pRows, int  pColumns, int  pTime = 1  )

inline

Splits this communicator into a grid of size pRows*pColumns and creates row and column communicators. By default the communicator is a single row.

Definition at line 726 of file Comm.h.

{
    v_SplitComm(pRows, pColumns, pTime);
}

References v_SplitComm().

SplitCommNode()

std::pair< CommSharedPtr, CommSharedPtr > Nektar::LibUtilities::Comm::SplitCommNode ( )

inline

Definition at line 817 of file Comm.h.

{
    return v_SplitCommNode();
}

References v_SplitCommNode().

StartAll()

void Nektar::LibUtilities::Comm::StartAll ( const CommRequestSharedPtr &  request )

inline

Starts a collection of persistent requests

Parameters

request

Communication request object

Definition at line 686 of file Comm.h.

{
    v_StartAll(request);
}

References v_StartAll().

TreatAsRankZero()

bool Nektar::LibUtilities::Comm::TreatAsRankZero ( )

inline

Definition at line 794 of file Comm.h.

{
    return v_TreatAsRankZero();
}

References v_TreatAsRankZero().

v_AllGather()

virtual void Nektar::LibUtilities::Comm::v_AllGather ( void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by AllGather().

v_AllGatherv() [1/2]

virtual void Nektar::LibUtilities::Comm::v_AllGatherv ( void *  recvbuf, int  recvcounts[], int  rdispls[], CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

v_AllGatherv() [2/2]

virtual void Nektar::LibUtilities::Comm::v_AllGatherv ( void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcounts[], int  rdispls[], CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by AllGatherv().

v_AllReduce()

virtual void Nektar::LibUtilities::Comm::v_AllReduce ( void *  buf, int  count, CommDataType  dt, enum ReduceOperator  pOp  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by AllReduce().

v_AlltoAll()

virtual void Nektar::LibUtilities::Comm::v_AlltoAll ( void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by AlltoAll().

v_AlltoAllv()

virtual void Nektar::LibUtilities::Comm::v_AlltoAllv ( void *  sendbuf, int  sendcounts[], int  sensdispls[], CommDataType  sendtype, void *  recvbuf, int  recvcounts[], int  rdispls[], CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by AlltoAllv().

v_Bcast()

virtual void Nektar::LibUtilities::Comm::v_Bcast ( void *  buffer, int  count, CommDataType  dt, int  root  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by Bcast().

v_Block()

virtual void Nektar::LibUtilities::Comm::v_Block ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by Block().

v_CommCreateIf()

virtual CommSharedPtr Nektar::LibUtilities::Comm::v_CommCreateIf ( int  flag )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by CommCreateIf().

v_CreateRequest()

virtual CommRequestSharedPtr Nektar::LibUtilities::Comm::v_CreateRequest ( int  num )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by CreateRequest().

v_DistGraphCreateAdjacent()

virtual void Nektar::LibUtilities::Comm::v_DistGraphCreateAdjacent ( int  indegree, const int  sources[], const int  sourceweights[], int  reorder  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by DistGraphCreateAdjacent().

v_Exscan()

virtual void Nektar::LibUtilities::Comm::v_Exscan ( Array< OneD, unsigned long long > &  pData, enum ReduceOperator  pOp, Array< OneD, unsigned long long > &  ans  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Exscan().

v_Finalise()

virtual void Nektar::LibUtilities::Comm::v_Finalise ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommCwipi.

Referenced by Finalise().

v_Gather()

virtual void Nektar::LibUtilities::Comm::v_Gather ( void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype, int  root  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by Gather().

v_GetRank()

virtual int Nektar::LibUtilities::Comm::v_GetRank ( )

protectedpure virtual

Implemented in Nektar::FieldUtils::FieldConvertComm, Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by GetRank().

v_GetVersion()

virtual std::tuple<int, int, int> Nektar::LibUtilities::Comm::v_GetVersion ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by GetVersion().

v_Irecv()

virtual void Nektar::LibUtilities::Comm::v_Irecv ( void *  buf, int  count, CommDataType  dt, int  source, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by Irecv().

v_Irsend()

virtual void Nektar::LibUtilities::Comm::v_Irsend ( void *  buf, int  count, CommDataType  dt, int  dest, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by Irsend().

v_Isend()

virtual void Nektar::LibUtilities::Comm::v_Isend ( void *  buf, int  count, CommDataType  dt, int  dest, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Isend().

v_IsSerial()

virtual bool Nektar::LibUtilities::Comm::v_IsSerial ( )

protectedpure virtual

Implemented in Nektar::FieldUtils::FieldConvertComm, Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by IsSerial().

v_NeighborAlltoAllv()

virtual void Nektar::LibUtilities::Comm::v_NeighborAlltoAllv ( void *  sendbuf, int  sendcounts[], int  sdispls[], CommDataType  sendtype, void *  recvbuf, int  recvcounts[], int  rdispls[], CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by NeighborAlltoAllv().

v_Recv()

virtual void Nektar::LibUtilities::Comm::v_Recv ( void *  buf, int  count, CommDataType  dt, int  source  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by Recv().

v_RecvInit()

virtual void Nektar::LibUtilities::Comm::v_RecvInit ( void *  buf, int  count, CommDataType  dt, int  source, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by RecvInit().

v_RemoveExistingFiles()

bool Nektar::LibUtilities::Comm::v_RemoveExistingFiles ( void  )

protectedvirtual

Reimplemented in Nektar::FieldUtils::FieldConvertComm.

Definition at line 57 of file Communication/Comm.cpp.

{
    return true;
}

Referenced by RemoveExistingFiles().

v_Scatter()

virtual void Nektar::LibUtilities::Comm::v_Scatter ( void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype, int  root  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by Scatter().

v_Send()

virtual void Nektar::LibUtilities::Comm::v_Send ( void *  buf, int  count, CommDataType  dt, int  dest  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by Send().

v_SendInit()

virtual void Nektar::LibUtilities::Comm::v_SendInit ( void *  buf, int  count, CommDataType  dt, int  dest, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by SendInit().

v_SendRecv()

virtual void Nektar::LibUtilities::Comm::v_SendRecv ( void *  sendbuf, int  sendcount, CommDataType  sendtype, int  dest, void *  recvbuf, int  recvcount, CommDataType  recvtype, int  source  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by SendRecv().

v_SendRecvReplace()

virtual void Nektar::LibUtilities::Comm::v_SendRecvReplace ( void *  buf, int  count, CommDataType  dt, int  pSendProc, int  pRecvProc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by SendRecvReplace().

v_SplitComm()

virtual void Nektar::LibUtilities::Comm::v_SplitComm ( int  pRows, int  pColumns, int  pTime  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, Nektar::LibUtilities::CommMpi, and Nektar::FieldUtils::FieldConvertComm.

Referenced by SplitComm().

v_SplitCommNode()

std::pair< CommSharedPtr, CommSharedPtr > Nektar::LibUtilities::Comm::v_SplitCommNode ( )

protectedvirtual

Reimplemented in Nektar::LibUtilities::CommMpi.

Definition at line 62 of file Communication/Comm.cpp.

{
    return std::make_pair(CommSharedPtr(), CommSharedPtr());
}

Referenced by SplitCommNode().

v_StartAll()

virtual void Nektar::LibUtilities::Comm::v_StartAll ( CommRequestSharedPtr  request )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by StartAll().

v_TreatAsRankZero()

virtual bool Nektar::LibUtilities::Comm::v_TreatAsRankZero ( )

protectedpure virtual

Implemented in Nektar::FieldUtils::FieldConvertComm, Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by TreatAsRankZero().

v_WaitAll()

virtual void Nektar::LibUtilities::Comm::v_WaitAll ( CommRequestSharedPtr  request )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by WaitAll().

v_Wtime()

virtual NekDouble Nektar::LibUtilities::Comm::v_Wtime ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by Wtime().

WaitAll()

void Nektar::LibUtilities::Comm::WaitAll ( const CommRequestSharedPtr &  request )

inline

Waits for all CommRequests in the request object to complete.

Parameters

request

Communication request object

Definition at line 696 of file Comm.h.

{
    v_WaitAll(request);
}

References v_WaitAll().

Wtime()

double Nektar::LibUtilities::Comm::Wtime ( )

inline

Return the time in seconds.

Definition at line 311 of file Comm.h.

{
    return v_Wtime();
}

References v_Wtime().

Member Data Documentation

m_commColumn

CommSharedPtr Nektar::LibUtilities::Comm::m_commColumn

protected

Column communicator.

Definition at line 184 of file Comm.h.

Referenced by GetColumnComm(), Nektar::FieldUtils::FieldConvertComm::v_SplitComm(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

m_commRow

CommSharedPtr Nektar::LibUtilities::Comm::m_commRow

protected

Row communicator.

Definition at line 183 of file Comm.h.

Referenced by GetRowComm(), Nektar::FieldUtils::FieldConvertComm::v_SplitComm(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

m_commSpace

CommSharedPtr Nektar::LibUtilities::Comm::m_commSpace

protected

Definition at line 186 of file Comm.h.

Referenced by GetSpaceComm(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

m_commTime

CommSharedPtr Nektar::LibUtilities::Comm::m_commTime

protected

Definition at line 185 of file Comm.h.

Referenced by GetTimeComm(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

m_size

int Nektar::LibUtilities::Comm::m_size

protected

Number of processes.

Definition at line 181 of file Comm.h.

Referenced by Nektar::LibUtilities::CommCwipi::CommCwipi(), Nektar::LibUtilities::CommMpi::CommMpi(), Nektar::LibUtilities::CommSerial::CommSerial(), Nektar::FieldUtils::FieldConvertComm::FieldConvertComm(), GetSize(), Nektar::LibUtilities::CommMpi::v_IsSerial(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

m_type

std::string Nektar::LibUtilities::Comm::m_type

protected

Type of communication.

Definition at line 182 of file Comm.h.

Referenced by Nektar::LibUtilities::CommCwipi::CommCwipi(), Nektar::LibUtilities::CommMpi::CommMpi(), Nektar::LibUtilities::CommSerial::CommSerial(), Nektar::FieldUtils::FieldConvertComm::FieldConvertComm(), and GetType().