Nektar::LibUtilities::Comm Class Reference

Base communications class. More...

#include <Comm.h>

Inheritance diagram for Nektar::LibUtilities::Comm:

Public Member Functions
	Comm (int narg, char *arg[])
virtual	~Comm ()
void	Finalise ()
int	GetSize () const
	Returns number of processes. More...
int	GetRank ()
const std::string &	GetType () const
void	Block ()
	Block execution until all processes reach this point. More...
NekDouble	Wtime ()
	Return the time in seconds. More...
template<class T >
void	Send (int pProc, T &pData)
template<class T >
void	Recv (int pProc, T &pData)
template<class T >
void	SendRecv (int pSendProc, T &pSendData, int pRecvProc, T &pRecvData)
template<class T >
void	SendRecvReplace (int pSendProc, int pRecvProc, T &pData)
template<class T >
void	AllReduce (T &pData, enum ReduceOperator pOp)
template<class T >
void	AlltoAll (T &pSendData, T &pRecvData)
template<class T1 , class T2 >
void	AlltoAllv (T1 &pSendData, T2 &pSendDataSizeMap, T2 &pSendDataOffsetMap, T1 &pRecvData, T2 &pRecvDataSizeMap, T2 &pRecvDataOffsetMap)
template<class T >
void	AllGather (T &pSendData, T &pRecvData)
template<class T >
void	AllGatherv (T &pSendData, T &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)
template<class T >
void	AllGatherv (T &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)
template<class T >
void	Bcast (T &pData, int pRoot)
template<class T >
void	Exscan (T &pData, enum ReduceOperator pOp, T &ans)
template<class T >
T	Gather (int rootProc, T &val)
template<class T >
T	Scatter (int rootProc, T &pData)
template<class T >
void	DistGraphCreateAdjacent (T &sources, T &sourceweights, int reorder)
template<class T1 , class T2 >
void	NeighborAlltoAllv (T1 &pSendData, T2 &pSendDataSizeMap, T2 &pSendDataOffsetMap, T1 &pRecvData, T2 &pRecvDataSizeMap, T2 &pRecvDataOffsetMap)
template<class T >
void	Irsend (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	SendInit (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	Irecv (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
template<class T >
void	RecvInit (int pProc, T &pData, int count, const CommRequestSharedPtr &request, int loc)
void	StartAll (const CommRequestSharedPtr &request)
void	WaitAll (const CommRequestSharedPtr &request)
CommRequestSharedPtr	CreateRequest (int num)
CommSharedPtr	CommCreateIf (int flag)
	If the flag is non-zero create a new communicator. More...
void	SplitComm (int pRows, int pColumns)
	Splits this communicator into a grid of size pRows*pColumns and creates row and column communicators. By default the communicator is a single row. More...
CommSharedPtr	GetRowComm ()
	Retrieve the row communicator to which this process belongs. More...
CommSharedPtr	GetColumnComm ()
	Retrieve the column communicator to which this process belongs. More...
bool	TreatAsRankZero ()
bool	IsSerial ()
std::tuple< int, int, int >	GetVersion ()
bool	RemoveExistingFiles ()
std::pair< CommSharedPtr, CommSharedPtr >	SplitCommNode ()

Protected Member Functions
	Comm ()
virtual void	v_Finalise ()=0
virtual int	v_GetRank ()=0
virtual void	v_Block ()=0
virtual NekDouble	v_Wtime ()=0
virtual void	v_Send (void *buf, int count, CommDataType dt, int dest)=0
virtual void	v_Recv (void *buf, int count, CommDataType dt, int source)=0
virtual void	v_SendRecv (void *sendbuf, int sendcount, CommDataType sendtype, int dest, void *recvbuf, int recvcount, CommDataType recvtype, int source)=0
virtual void	v_SendRecvReplace (void *buf, int count, CommDataType dt, int pSendProc, int pRecvProc)=0
virtual void	v_AllReduce (void *buf, int count, CommDataType dt, enum ReduceOperator pOp)=0
virtual void	v_AlltoAll (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype)=0
virtual void	v_AlltoAllv (void *sendbuf, int sendcounts[], int sensdispls[], CommDataType sendtype, void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype)=0
virtual void	v_AllGather (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype)=0
virtual void	v_AllGatherv (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype)=0
virtual void	v_AllGatherv (void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype)=0
virtual void	v_Bcast (void *buffer, int count, CommDataType dt, int root)=0
virtual void	v_Exscan (Array< OneD, unsigned long long > &pData, enum ReduceOperator pOp, Array< OneD, unsigned long long > &ans)=0
virtual void	v_Gather (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype, int root)=0
virtual void	v_Scatter (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype, int root)=0
virtual CommSharedPtr	v_CommCreateIf (int flag)=0
virtual void	v_DistGraphCreateAdjacent (int indegree, const int sources[], const int sourceweights[], int reorder)=0
virtual void	v_NeighborAlltoAllv (void *sendbuf, int sendcounts[], int sdispls[], CommDataType sendtype, void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype)=0
virtual void	v_Irsend (void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc)=0
virtual void	v_SendInit (void *buf, int count, CommDataType dt, int dest, CommRequestSharedPtr request, int loc)=0
virtual void	v_Irecv (void *buf, int count, CommDataType dt, int source, CommRequestSharedPtr request, int loc)=0
virtual void	v_RecvInit (void *buf, int count, CommDataType dt, int source, CommRequestSharedPtr request, int loc)=0
virtual void	v_StartAll (CommRequestSharedPtr request)=0
virtual void	v_WaitAll (CommRequestSharedPtr request)=0
virtual CommRequestSharedPtr	v_CreateRequest (int num)=0
virtual void	v_SplitComm (int pRows, int pColumns)=0
virtual bool	v_TreatAsRankZero ()=0
virtual bool	v_IsSerial ()=0
virtual std::tuple< int, int, int >	v_GetVersion ()=0
virtual bool	v_RemoveExistingFiles ()
virtual std::pair< CommSharedPtr, CommSharedPtr >	v_SplitCommNode ()

Protected Attributes
int	m_size
	Number of processes. More...
std::string	m_type
	Type of communication. More...
CommSharedPtr	m_commRow
	Row communicator. More...
CommSharedPtr	m_commColumn
	Column communicator. More...

Detailed Description

Base communications class.

Definition at line 89 of file Comm.h.

Constructor & Destructor Documentation

Comm() [1/2]

Nektar::LibUtilities::Comm::Comm	(	int	narg,
		char *	arg[]
	)

Definition at line 43 of file Communication/Comm.cpp.

{
    boost::ignore_unused(narg, arg);
}

~Comm()

Nektar::LibUtilities::Comm::~Comm ( )

virtual

Definition at line 52 of file Communication/Comm.cpp.

{
}

Comm() [2/2]

Nektar::LibUtilities::Comm::Comm ( )

protected

Definition at line 48 of file Communication/Comm.cpp.

{
}

Member Function Documentation

AllGather()

template<class T >

void Nektar::LibUtilities::Comm::AllGather	(	T &	pSendData,
		T &	pRecvData
	)

Definition at line 398 of file Comm.h.

{
    BOOST_STATIC_ASSERT_MSG(
        CommDataTypeTraits<T>::IsVector,
        "AllGather only valid with Array or vector arguments.");
 
    int sendSize = CommDataTypeTraits<T>::GetCount(pSendData);
    int recvSize = sendSize;
 
    pRecvData = T(recvSize * GetSize());
 
    v_AllGather(CommDataTypeTraits<T>::GetPointer(pSendData), sendSize,
                CommDataTypeTraits<T>::GetDataType(),
                CommDataTypeTraits<T>::GetPointer(pRecvData), recvSize,
                CommDataTypeTraits<T>::GetDataType());
}

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), GetSize(), and v_AllGather().

AllGatherv() [1/2]

template<class T >

void Nektar::LibUtilities::Comm::AllGatherv	(	T &	pRecvData,
		Array< OneD, int > &	pRecvDataSizeMap,
		Array< OneD, int > &	pRecvDataOffsetMap
	)

Definition at line 440 of file Comm.h.

{
    BOOST_STATIC_ASSERT_MSG(
        CommDataTypeTraits<T>::IsVector,
        "AllGatherv only valid with Array or vector arguments.");
 
    v_AllGatherv(CommDataTypeTraits<T>::GetPointer(pRecvData),
                 pRecvDataSizeMap.get(), pRecvDataOffsetMap.get(),
                 CommDataTypeTraits<T>::GetDataType());
}

References v_AllGatherv().

AllGatherv() [2/2]

template<class T >

void Nektar::LibUtilities::Comm::AllGatherv	(	T &	pSendData,
		T &	pRecvData,
		Array< OneD, int > &	pRecvDataSizeMap,
		Array< OneD, int > &	pRecvDataOffsetMap
	)

Definition at line 419 of file Comm.h.

{
    BOOST_STATIC_ASSERT_MSG(
        CommDataTypeTraits<T>::IsVector,
        "AllGatherv only valid with Array or vector arguments.");
 
    int sendSize = CommDataTypeTraits<T>::GetCount(pSendData);
 
    v_AllGatherv(CommDataTypeTraits<T>::GetPointer(pSendData), sendSize,
                 CommDataTypeTraits<T>::GetDataType(),
                 CommDataTypeTraits<T>::GetPointer(pRecvData),
                 pRecvDataSizeMap.get(), pRecvDataOffsetMap.get(),
                 CommDataTypeTraits<T>::GetDataType());
}

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), and v_AllGatherv().

AllReduce()

template<class T >

void Nektar::LibUtilities::Comm::AllReduce	(	T &	pData,
		enum ReduceOperator	pOp
	)

Definition at line 349 of file Comm.h.

{
    v_AllReduce(CommDataTypeTraits<T>::GetPointer(pData),
                CommDataTypeTraits<T>::GetCount(pData),
                CommDataTypeTraits<T>::GetDataType(), pOp);
}

References v_AllReduce().

AlltoAll()

template<class T >

void Nektar::LibUtilities::Comm::AlltoAll	(	T &	pSendData,
		T &	pRecvData
	)

Definition at line 356 of file Comm.h.

{
    static_assert(CommDataTypeTraits<T>::IsVector,
                  "AlltoAll only valid with Array or vector arguments.");
    int sendSize = CommDataTypeTraits<T>::GetCount(pSendData);
    int recvSize = CommDataTypeTraits<T>::GetCount(pRecvData);
    ASSERTL0(sendSize == recvSize,
             "Send and Recv arrays have incompatible sizes in AlltoAll");
 
    int count = sendSize / GetSize();
    ASSERTL0(count * GetSize() == sendSize,
             "Array size incompatible with size of communicator");
 
    v_AlltoAll(CommDataTypeTraits<T>::GetPointer(pSendData), count,
               CommDataTypeTraits<T>::GetDataType(),
               CommDataTypeTraits<T>::GetPointer(pRecvData), count,
               CommDataTypeTraits<T>::GetDataType());
}

References ASSERTL0, Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), GetSize(), and v_AlltoAll().

AlltoAllv()

template<class T1 , class T2 >

void Nektar::LibUtilities::Comm::AlltoAllv	(	T1 &	pSendData,
		T2 &	pSendDataSizeMap,
		T2 &	pSendDataOffsetMap,
		T1 &	pRecvData,
		T2 &	pRecvDataSizeMap,
		T2 &	pRecvDataOffsetMap
	)

Definition at line 379 of file Comm.h.

{
    static_assert(CommDataTypeTraits<T1>::IsVector,
                  "AlltoAllv only valid with Array or vector arguments.");
    static_assert(std::is_same<T2, std::vector<int>>::value ||
                      std::is_same<T2, Array<OneD, int>>::value,
                  "Alltoallv size and offset maps should be integer vectors.");
    v_AlltoAllv(CommDataTypeTraits<T1>::GetPointer(pSendData),
                (int *)CommDataTypeTraits<T2>::GetPointer(pSendDataSizeMap),
                (int *)CommDataTypeTraits<T2>::GetPointer(pSendDataOffsetMap),
                CommDataTypeTraits<T1>::GetDataType(),
                CommDataTypeTraits<T1>::GetPointer(pRecvData),
                (int *)CommDataTypeTraits<T2>::GetPointer(pRecvDataSizeMap),
                (int *)CommDataTypeTraits<T2>::GetPointer(pRecvDataOffsetMap),
                CommDataTypeTraits<T1>::GetDataType());
}

References v_AlltoAllv().

Bcast()

template<class T >

void Nektar::LibUtilities::Comm::Bcast	(	T &	pData,
		int	pRoot
	)

Definition at line 455 of file Comm.h.

{
    v_Bcast(CommDataTypeTraits<T>::GetPointer(pData),
            CommDataTypeTraits<T>::GetCount(pData),
            CommDataTypeTraits<T>::GetDataType(), pRoot);
}

References v_Bcast().

Block()

void Nektar::LibUtilities::Comm::Block ( )

inline

Block execution until all processes reach this point.

Definition at line 293 of file Comm.h.

{
    v_Block();
}

References v_Block().

CommCreateIf()

CommSharedPtr Nektar::LibUtilities::Comm::CommCreateIf ( int  flag )

inline

If the flag is non-zero create a new communicator.

Definition at line 689 of file Comm.h.

{
    return v_CommCreateIf(flag);
}

References v_CommCreateIf().

CreateRequest()

CommRequestSharedPtr Nektar::LibUtilities::Comm::CreateRequest ( int  num )

inline

Creates a number of CommRequests.

Parameters

num	Number of requests to generate in the communication request object

Returns

Communication request object

Definition at line 681 of file Comm.h.

{
    return v_CreateRequest(num);
}

References v_CreateRequest().

DistGraphCreateAdjacent()

template<class T >

void Nektar::LibUtilities::Comm::DistGraphCreateAdjacent	(	T &	sources,
		T &	sourceweights,
		int	reorder
	)

This replaces the current MPI communicator with a new one that also holds the distributed graph topology information. If reordering is enabled using this might break code where process/rank numbers are assumed to remain constant. This also assumes that the graph is bi-directional, so all sources are also destinations with equal weighting.

Parameters

sources	Ranks of processes for which the calling process is the destination/source
sourceweights	Weights of the corresponding edges into the calling process
reorder	Ranks may be reordered (true) or not (false)

Definition at line 528 of file Comm.h.

{
    static_assert(
        CommDataTypeTraits<T>::IsVector,
        "DistGraphCreateAdjacent only valid with Array or vector arguments.");
 
    ASSERTL0(CommDataTypeTraits<T>::GetCount(sources) ==
                 CommDataTypeTraits<T>::GetCount(sourceweights),
             "Sources and weights array sizes don't match");
 
    int indegree = CommDataTypeTraits<T>::GetCount(sources);
 
    v_DistGraphCreateAdjacent(
        indegree, (const int *)CommDataTypeTraits<T>::GetPointer(sources),
        (const int *)CommDataTypeTraits<T>::GetPointer(sourceweights), reorder);
}

References ASSERTL0, Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), and v_DistGraphCreateAdjacent().

Exscan()

template<class T >

void Nektar::LibUtilities::Comm::Exscan	(	T &	pData,
		enum ReduceOperator	pOp,
		T &	ans
	)

Definition at line 463 of file Comm.h.

{
    ASSERTL0(CommDataTypeTraits<T>::GetCount(pData) ==
                 CommDataTypeTraits<T>::GetCount(ans),
             "Input and output array sizes don't match");
    v_Exscan(CommDataTypeTraits<T>::GetPointer(pData),
             CommDataTypeTraits<T>::GetPointer(ans),
             CommDataTypeTraits<T>::GetCount(pData),
             CommDataTypeTraits<T>::GetDataType(), pOp);
}

References ASSERTL0, and v_Exscan().

Finalise()

void Nektar::LibUtilities::Comm::Finalise ( )

inline

Definition at line 261 of file Comm.h.

{
    v_Finalise();
}

References v_Finalise().

Gather()

template<class T >

T Nektar::LibUtilities::Comm::Gather	(	int	rootProc,
		T &	val
	)

Concatenate all the input arrays, in rank order, onto the process with rank == rootProc

Definition at line 478 of file Comm.h.

{
    static_assert(CommDataTypeTraits<T>::IsVector,
                  "Gather only valid with Array or vector arguments.");
    bool amRoot  = (GetRank() == rootProc);
    unsigned nEl = CommDataTypeTraits<T>::GetCount(val);
 
    unsigned nOut = amRoot ? GetSize() * nEl : 0;
    T ans(nOut);
    void *recvbuf = amRoot ? CommDataTypeTraits<T>::GetPointer(ans) : NULL;
 
    v_Gather(CommDataTypeTraits<T>::GetPointer(val), nEl,
             CommDataTypeTraits<T>::GetDataType(), recvbuf, nEl,
             CommDataTypeTraits<T>::GetDataType(), rootProc);
    return ans;
}

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), Nektar::LibUtilities::CommDataTypeTraits< T >::GetPointer(), GetRank(), GetSize(), and v_Gather().

GetColumnComm()

CommSharedPtr Nektar::LibUtilities::Comm::GetColumnComm ( )

inline

Retrieve the column communicator to which this process belongs.

Definition at line 723 of file Comm.h.

{
    if (!m_commColumn.get())
    {
        return shared_from_this();
    }
    else
    {
        return m_commColumn;
    }
}

References m_commColumn.

GetRank()

int Nektar::LibUtilities::Comm::GetRank ( )

inline

Definition at line 277 of file Comm.h.

{
    return v_GetRank();
}

References v_GetRank().

Referenced by export_Comm(), Gather(), and Scatter().

GetRowComm()

CommSharedPtr Nektar::LibUtilities::Comm::GetRowComm ( )

inline

Retrieve the row communicator to which this process belongs.

Definition at line 707 of file Comm.h.

{
    if (!m_commRow.get())
    {
        return shared_from_this();
    }
    else
    {
        return m_commRow;
    }
}

References m_commRow.

GetSize()

int Nektar::LibUtilities::Comm::GetSize ( ) const

inline

Returns number of processes.

Definition at line 269 of file Comm.h.

{
    return m_size;
}

References m_size.

Referenced by AllGather(), AlltoAll(), export_Comm(), Gather(), Scatter(), and Nektar::LibUtilities::CommMpi::v_AllReduce().

GetType()

const std::string & Nektar::LibUtilities::Comm::GetType ( ) const

inline

Definition at line 285 of file Comm.h.

{
    return m_type;
}

References m_type.

Referenced by export_Comm().

GetVersion()

std::tuple< int, int, int > Nektar::LibUtilities::Comm::GetVersion ( )

inline

Returns

tuple of {major, minor, patch} version numbers

Definition at line 748 of file Comm.h.

{
    return v_GetVersion();
}

References v_GetVersion().

Irecv()

template<class T >

void Nektar::LibUtilities::Comm::Irecv	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Begins a nonblocking receive

Parameters

pProc	Rank of source
pData	Array/vector to place incoming data in to
count	Number of elements to receive in to pData
request	Communication request object
loc	Location in request to use

Definition at line 630 of file Comm.h.

{
    v_Irecv(CommDataTypeTraits<T>::GetPointer(pData), count,
            CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References CG_Iterations::loc, and v_Irecv().

Irsend()

template<class T >

void Nektar::LibUtilities::Comm::Irsend	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Starts a ready-mode nonblocking send

Parameters

pProc	Rank of destination
pData	Array/vector to send
count	Number of elements to send in pData
request	Communication request object
loc	Location in request to use

Definition at line 596 of file Comm.h.

{
    v_Irsend(CommDataTypeTraits<T>::GetPointer(pData), count,
             CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References CG_Iterations::loc, and v_Irsend().

IsSerial()

bool Nektar::LibUtilities::Comm::IsSerial ( )

inline

Definition at line 740 of file Comm.h.

{
    return v_IsSerial();
}

References v_IsSerial().

NeighborAlltoAllv()

template<class T1 , class T2 >

void Nektar::LibUtilities::Comm::NeighborAlltoAllv	(	T1 &	pSendData,
		T2 &	pSendDataSizeMap,
		T2 &	pSendDataOffsetMap,
		T1 &	pRecvData,
		T2 &	pRecvDataSizeMap,
		T2 &	pRecvDataOffsetMap
	)

Sends data to neighboring processes in a virtual topology communicator. All processes send different amounts of data to, and receive different amounts of data from, all neighbors

Parameters

pSendData	Array/vector to send to neighbors
pSendDataSizeMap	Array/vector where entry i specifies the number of elements to send to neighbor i
pSendDataOffsetMap	Array/vector where entry i specifies the displacement (offset from pSendData) from which to send data to neighbor i
pRecvData	Array/vector to place incoming data in to
pRecvDataSizeMap	Array/vector where entry i specifies the number of elements to receive from neighbor i
pRecvDataOffsetMap	Array/vector where entry i specifies the displacement (offset from pRecvData) from which to receive data from neighbor i

Definition at line 564 of file Comm.h.

{
    static_assert(
        CommDataTypeTraits<T1>::IsVector,
        "NeighbourAlltoAllv only valid with Array or vector arguments.");
    static_assert(
        std::is_same<T2, std::vector<int>>::value ||
            std::is_same<T2, Array<OneD, int>>::value,
        "NeighborAllToAllv size and offset maps should be integer vectors.");
    v_NeighborAlltoAllv(
        CommDataTypeTraits<T1>::GetPointer(pSendData),
        (int *)CommDataTypeTraits<T2>::GetPointer(pSendDataSizeMap),
        (int *)CommDataTypeTraits<T2>::GetPointer(pSendDataOffsetMap),
        CommDataTypeTraits<T1>::GetDataType(),
        CommDataTypeTraits<T1>::GetPointer(pRecvData),
        (int *)CommDataTypeTraits<T2>::GetPointer(pRecvDataSizeMap),
        (int *)CommDataTypeTraits<T2>::GetPointer(pRecvDataOffsetMap),
        CommDataTypeTraits<T1>::GetDataType());
}

References v_NeighborAlltoAllv().

Recv()

template<class T >

void Nektar::LibUtilities::Comm::Recv	(	int	pProc,
		T &	pData
	)

Definition at line 313 of file Comm.h.

{
    v_Recv(CommDataTypeTraits<T>::GetPointer(pData),
           CommDataTypeTraits<T>::GetCount(pData),
           CommDataTypeTraits<T>::GetDataType(), pProc);
}

References v_Recv().

RecvInit()

template<class T >

void Nektar::LibUtilities::Comm::RecvInit	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Create a persistent request for a receive

Parameters

pProc	Rank of source
pData	Array/vector to place incoming data in to
count	Number of elements to receive in to pData
request	Communication request object
loc	Location in request to use

Definition at line 647 of file Comm.h.

{
    v_RecvInit(CommDataTypeTraits<T>::GetPointer(pData), count,
               CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References CG_Iterations::loc, and v_RecvInit().

RemoveExistingFiles()

bool Nektar::LibUtilities::Comm::RemoveExistingFiles ( )

inline

Definition at line 753 of file Comm.h.

{
    return v_RemoveExistingFiles();
}

References v_RemoveExistingFiles().

Scatter()

template<class T >

T Nektar::LibUtilities::Comm::Scatter	(	int	rootProc,
		T &	pData
	)

Scatter pData across ranks in chunks of len(pData)/num_ranks

Definition at line 497 of file Comm.h.

{
    static_assert(CommDataTypeTraits<T>::IsVector,
                  "Scatter only valid with Array or vector arguments.");
 
    bool amRoot  = (GetRank() == rootProc);
    unsigned nEl = CommDataTypeTraits<T>::GetCount(pData) / GetSize();
 
    void *sendbuf = amRoot ? CommDataTypeTraits<T>::GetPointer(pData) : NULL;
    T ans(nEl);
 
    v_Scatter(sendbuf, nEl, CommDataTypeTraits<T>::GetDataType(),
              CommDataTypeTraits<T>::GetPointer(ans), nEl,
              CommDataTypeTraits<T>::GetDataType(), rootProc);
    return ans;
}

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), Nektar::LibUtilities::CommDataTypeTraits< T >::GetPointer(), GetRank(), GetSize(), and v_Scatter().

Send()

template<class T >

void Nektar::LibUtilities::Comm::Send	(	int	pProc,
		T &	pData
	)

Definition at line 306 of file Comm.h.

{
    v_Send(CommDataTypeTraits<T>::GetPointer(pData),
           CommDataTypeTraits<T>::GetCount(pData),
           CommDataTypeTraits<T>::GetDataType(), pProc);
}

References v_Send().

SendInit()

template<class T >

void Nektar::LibUtilities::Comm::SendInit	(	int	pProc,
		T &	pData,
		int	count,
		const CommRequestSharedPtr &	request,
		int	loc
	)

Creates a persistent request for a send

Parameters

pProc	Rank of destination
pData	Array/vector to send
count	Number of elements to send in pData
request	Communication request object
loc	Location in request to use

Definition at line 613 of file Comm.h.

{
    v_SendInit(CommDataTypeTraits<T>::GetPointer(pData), count,
               CommDataTypeTraits<T>::GetDataType(), pProc, request, loc);
}

References CG_Iterations::loc, and v_SendInit().

SendRecv()

template<class T >

void Nektar::LibUtilities::Comm::SendRecv	(	int	pSendProc,
		T &	pSendData,
		int	pRecvProc,
		T &	pRecvData
	)

Definition at line 324 of file Comm.h.

{
    v_SendRecv(CommDataTypeTraits<T>::GetPointer(pSendData),
               CommDataTypeTraits<T>::GetCount(pSendData),
               CommDataTypeTraits<T>::GetDataType(), pSendProc,
               CommDataTypeTraits<T>::GetPointer(pRecvData),
               CommDataTypeTraits<T>::GetCount(pRecvData),
               CommDataTypeTraits<T>::GetDataType(), pRecvProc);
}

References v_SendRecv().

SendRecvReplace()

template<class T >

void Nektar::LibUtilities::Comm::SendRecvReplace	(	int	pSendProc,
		int	pRecvProc,
		T &	pData
	)

Definition at line 338 of file Comm.h.

{
    v_SendRecvReplace(CommDataTypeTraits<T>::GetPointer(pData),
                      CommDataTypeTraits<T>::GetCount(pData),
                      CommDataTypeTraits<T>::GetDataType(), pSendProc,
                      pRecvProc);
}

References v_SendRecvReplace().

SplitComm()

void Nektar::LibUtilities::Comm::SplitComm ( int  pRows, int  pColumns  )

inline

Splits this communicator into a grid of size pRows*pColumns and creates row and column communicators. By default the communicator is a single row.

Definition at line 699 of file Comm.h.

{
    v_SplitComm(pRows, pColumns);
}

References v_SplitComm().

SplitCommNode()

std::pair< CommSharedPtr, CommSharedPtr > Nektar::LibUtilities::Comm::SplitCommNode ( )

inline

Definition at line 758 of file Comm.h.

{
    return v_SplitCommNode();
}

References v_SplitCommNode().

StartAll()

void Nektar::LibUtilities::Comm::StartAll ( const CommRequestSharedPtr &  request )

inline

Starts a collection of persistent requests

Parameters

request

Communication request object

Definition at line 659 of file Comm.h.

{
    v_StartAll(request);
}

References v_StartAll().

TreatAsRankZero()

bool Nektar::LibUtilities::Comm::TreatAsRankZero ( )

inline

Definition at line 735 of file Comm.h.

{
    return v_TreatAsRankZero();
}

References v_TreatAsRankZero().

v_AllGather()

virtual void Nektar::LibUtilities::Comm::v_AllGather ( void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by AllGather().

v_AllGatherv() [1/2]

virtual void Nektar::LibUtilities::Comm::v_AllGatherv ( void *  recvbuf, int  recvcounts[], int  rdispls[], CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

v_AllGatherv() [2/2]

virtual void Nektar::LibUtilities::Comm::v_AllGatherv ( void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcounts[], int  rdispls[], CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by AllGatherv().

v_AllReduce()

virtual void Nektar::LibUtilities::Comm::v_AllReduce ( void *  buf, int  count, CommDataType  dt, enum ReduceOperator  pOp  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by AllReduce().

v_AlltoAll()

virtual void Nektar::LibUtilities::Comm::v_AlltoAll ( void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by AlltoAll().

v_AlltoAllv()

virtual void Nektar::LibUtilities::Comm::v_AlltoAllv ( void *  sendbuf, int  sendcounts[], int  sensdispls[], CommDataType  sendtype, void *  recvbuf, int  recvcounts[], int  rdispls[], CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by AlltoAllv().

v_Bcast()

virtual void Nektar::LibUtilities::Comm::v_Bcast ( void *  buffer, int  count, CommDataType  dt, int  root  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by Bcast().

v_Block()

virtual void Nektar::LibUtilities::Comm::v_Block ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by Block().

v_CommCreateIf()

virtual CommSharedPtr Nektar::LibUtilities::Comm::v_CommCreateIf ( int  flag )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by CommCreateIf().

v_CreateRequest()

virtual CommRequestSharedPtr Nektar::LibUtilities::Comm::v_CreateRequest ( int  num )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by CreateRequest().

v_DistGraphCreateAdjacent()

virtual void Nektar::LibUtilities::Comm::v_DistGraphCreateAdjacent ( int  indegree, const int  sources[], const int  sourceweights[], int  reorder  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by DistGraphCreateAdjacent().

v_Exscan()

virtual void Nektar::LibUtilities::Comm::v_Exscan ( Array< OneD, unsigned long long > &  pData, enum ReduceOperator  pOp, Array< OneD, unsigned long long > &  ans  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Exscan().

v_Finalise()

virtual void Nektar::LibUtilities::Comm::v_Finalise ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommCwipi.

Referenced by Finalise().

v_Gather()

virtual void Nektar::LibUtilities::Comm::v_Gather ( void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype, int  root  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by Gather().

v_GetRank()

virtual int Nektar::LibUtilities::Comm::v_GetRank ( )

protectedpure virtual

Implemented in Nektar::FieldUtils::FieldConvertComm, Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by GetRank().

v_GetVersion()

virtual std::tuple<int, int, int> Nektar::LibUtilities::Comm::v_GetVersion ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by GetVersion().

v_Irecv()

virtual void Nektar::LibUtilities::Comm::v_Irecv ( void *  buf, int  count, CommDataType  dt, int  source, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by Irecv().

v_Irsend()

virtual void Nektar::LibUtilities::Comm::v_Irsend ( void *  buf, int  count, CommDataType  dt, int  dest, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by Irsend().

v_IsSerial()

virtual bool Nektar::LibUtilities::Comm::v_IsSerial ( )

protectedpure virtual

Implemented in Nektar::FieldUtils::FieldConvertComm, Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by IsSerial().

v_NeighborAlltoAllv()

virtual void Nektar::LibUtilities::Comm::v_NeighborAlltoAllv ( void *  sendbuf, int  sendcounts[], int  sdispls[], CommDataType  sendtype, void *  recvbuf, int  recvcounts[], int  rdispls[], CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by NeighborAlltoAllv().

v_Recv()

virtual void Nektar::LibUtilities::Comm::v_Recv ( void *  buf, int  count, CommDataType  dt, int  source  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by Recv().

v_RecvInit()

virtual void Nektar::LibUtilities::Comm::v_RecvInit ( void *  buf, int  count, CommDataType  dt, int  source, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by RecvInit().

v_RemoveExistingFiles()

bool Nektar::LibUtilities::Comm::v_RemoveExistingFiles ( void  )

protectedvirtual

Reimplemented in Nektar::FieldUtils::FieldConvertComm.

Definition at line 57 of file Communication/Comm.cpp.

{
    return true;
}

Referenced by RemoveExistingFiles().

v_Scatter()

virtual void Nektar::LibUtilities::Comm::v_Scatter ( void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype, int  root  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by Scatter().

v_Send()

virtual void Nektar::LibUtilities::Comm::v_Send ( void *  buf, int  count, CommDataType  dt, int  dest  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by Send().

v_SendInit()

virtual void Nektar::LibUtilities::Comm::v_SendInit ( void *  buf, int  count, CommDataType  dt, int  dest, CommRequestSharedPtr  request, int  loc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by SendInit().

v_SendRecv()

virtual void Nektar::LibUtilities::Comm::v_SendRecv ( void *  sendbuf, int  sendcount, CommDataType  sendtype, int  dest, void *  recvbuf, int  recvcount, CommDataType  recvtype, int  source  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by SendRecv().

v_SendRecvReplace()

virtual void Nektar::LibUtilities::Comm::v_SendRecvReplace ( void *  buf, int  count, CommDataType  dt, int  pSendProc, int  pRecvProc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by SendRecvReplace().

v_SplitComm()

virtual void Nektar::LibUtilities::Comm::v_SplitComm ( int  pRows, int  pColumns  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, Nektar::LibUtilities::CommMpi, and Nektar::FieldUtils::FieldConvertComm.

Referenced by SplitComm().

v_SplitCommNode()

std::pair< CommSharedPtr, CommSharedPtr > Nektar::LibUtilities::Comm::v_SplitCommNode ( )

protectedvirtual

Reimplemented in Nektar::LibUtilities::CommMpi.

Definition at line 62 of file Communication/Comm.cpp.

{
    return std::make_pair(CommSharedPtr(), CommSharedPtr());
}

Referenced by SplitCommNode().

v_StartAll()

virtual void Nektar::LibUtilities::Comm::v_StartAll ( CommRequestSharedPtr  request )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by StartAll().

v_TreatAsRankZero()

virtual bool Nektar::LibUtilities::Comm::v_TreatAsRankZero ( )

protectedpure virtual

Implemented in Nektar::FieldUtils::FieldConvertComm, Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by TreatAsRankZero().

v_WaitAll()

virtual void Nektar::LibUtilities::Comm::v_WaitAll ( CommRequestSharedPtr  request )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by WaitAll().

v_Wtime()

virtual NekDouble Nektar::LibUtilities::Comm::v_Wtime ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by Wtime().

WaitAll()

void Nektar::LibUtilities::Comm::WaitAll ( const CommRequestSharedPtr &  request )

inline

Waits for all CommRequests in the request object to complete.

Parameters

request

Communication request object

Definition at line 669 of file Comm.h.

{
    v_WaitAll(request);
}

References v_WaitAll().

Wtime()

double Nektar::LibUtilities::Comm::Wtime ( )

inline

Return the time in seconds.

Definition at line 301 of file Comm.h.

{
    return v_Wtime();
}

References v_Wtime().

Member Data Documentation

m_commColumn

CommSharedPtr Nektar::LibUtilities::Comm::m_commColumn

protected

Column communicator.

Definition at line 178 of file Comm.h.

Referenced by GetColumnComm(), Nektar::FieldUtils::FieldConvertComm::v_SplitComm(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

m_commRow

CommSharedPtr Nektar::LibUtilities::Comm::m_commRow

protected

Row communicator.

Definition at line 177 of file Comm.h.

Referenced by GetRowComm(), Nektar::FieldUtils::FieldConvertComm::v_SplitComm(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

m_size

int Nektar::LibUtilities::Comm::m_size

protected

Number of processes.

Definition at line 175 of file Comm.h.

Referenced by Nektar::LibUtilities::CommCwipi::CommCwipi(), Nektar::LibUtilities::CommMpi::CommMpi(), Nektar::LibUtilities::CommSerial::CommSerial(), Nektar::FieldUtils::FieldConvertComm::FieldConvertComm(), GetSize(), Nektar::LibUtilities::CommMpi::v_IsSerial(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

m_type

std::string Nektar::LibUtilities::Comm::m_type

protected

Type of communication.

Definition at line 176 of file Comm.h.

Referenced by Nektar::LibUtilities::CommCwipi::CommCwipi(), Nektar::LibUtilities::CommMpi::CommMpi(), Nektar::LibUtilities::CommSerial::CommSerial(), Nektar::FieldUtils::FieldConvertComm::FieldConvertComm(), and GetType().