Nektar::LibUtilities::Comm Class Reference

Base communications class. More...

#include <Comm.h>

Inheritance diagram for Nektar::LibUtilities::Comm:

Collaboration diagram for Nektar::LibUtilities::Comm:

Public Member Functions
	Comm (int narg, char *arg[])
virtual	~Comm ()
void	Finalise ()
int	GetSize ()
	Returns number of processes. More...
int	GetRank ()
const std::string &	GetType () const
void	Block ()
	Block execution until all processes reach this point. More...
NekDouble	Wtime ()
	Return the time in seconds. More...
template<class T >
void	Send (int pProc, T &pData)
template<class T >
void	Recv (int pProc, T &pData)
template<class T >
void	SendRecv (int pSendProc, T &pSendData, int pRecvProc, T &pRecvData)
template<class T >
void	SendRecvReplace (int pSendProc, int pRecvProc, T &pData)
template<class T >
void	AllReduce (T &pData, enum ReduceOperator pOp)
template<class T >
void	AlltoAll (T &pSendData, T &pRecvData)
template<class T >
void	AlltoAllv (Array< OneD, T > &pSendData, Array< OneD, int > &pSendDataSizeMap, Array< OneD, int > &pSendDataOffsetMap, Array< OneD, T > &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)
template<class T >
void	Bcast (T &data, int rootProc)
template<class T >
void	Exscan (T &pData, const enum ReduceOperator pOp, T &ans)
template<class T >
T	Gather (const int rootProc, T &val)
template<class T >
T	Scatter (const int rootProc, T &pData)
CommSharedPtr	CommCreateIf (int flag)
	If the flag is non-zero create a new communicator. More...
void	SplitComm (int pRows, int pColumns)
	Splits this communicator into a grid of size pRows*pColumns and creates row and column communicators. By default the communicator is a single row. More...
CommSharedPtr	GetRowComm ()
	Retrieve the row communicator to which this process belongs. More...
CommSharedPtr	GetColumnComm ()
	Retrieve the column communicator to which this process belongs. More...
bool	TreatAsRankZero (void)
bool	RemoveExistingFiles (void)

Protected Member Functions
	Comm ()
virtual void	v_Finalise ()=0
virtual int	v_GetRank ()=0
virtual void	v_Block ()=0
virtual NekDouble	v_Wtime ()=0
virtual void	v_Send (void *buf, int count, CommDataType dt, int dest)=0
virtual void	v_Recv (void *buf, int count, CommDataType dt, int source)=0
virtual void	v_SendRecv (void *sendbuf, int sendcount, CommDataType sendtype, int dest, void *recvbuf, int recvcount, CommDataType recvtype, int source)=0
virtual void	v_SendRecvReplace (void *buf, int count, CommDataType dt, int pSendProc, int pRecvProc)=0
virtual void	v_AllReduce (void *buf, int count, CommDataType dt, enum ReduceOperator pOp)=0
virtual void	v_AlltoAll (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype)=0
virtual void	v_AlltoAllv (void *sendbuf, int sendcounts[], int sensdispls[], CommDataType sendtype, void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype)=0
virtual void	v_Bcast (void *buffer, int count, CommDataType dt, int root)=0
virtual void	v_Exscan (Array< OneD, unsigned long long > &pData, const enum ReduceOperator pOp, Array< OneD, unsigned long long > &ans)=0
virtual void	v_Gather (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype, int root)=0
virtual void	v_Scatter (void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype, int root)=0
virtual CommSharedPtr	v_CommCreateIf (int flag)=0
virtual void	v_SplitComm (int pRows, int pColumns)=0
virtual bool	v_TreatAsRankZero (void)=0
virtual bool	v_RemoveExistingFiles (void)

Protected Attributes
int	m_size
	Number of processes. More...
std::string	m_type
	Type of communication. More...
CommSharedPtr	m_commRow
	Row communicator. More...
CommSharedPtr	m_commColumn
	Column communicator. More...

Detailed Description

Base communications class.

Definition at line 75 of file Comm.h.

Constructor & Destructor Documentation

Nektar::LibUtilities::Comm::Comm	(	int	narg,
		char *	arg[]
	)

Definition at line 43 of file Comm.cpp.

{
}

Nektar::LibUtilities::Comm::~Comm ( )

virtual

Definition at line 51 of file Comm.cpp.

{
}

Nektar::LibUtilities::Comm::Comm ( )

protected

Definition at line 47 of file Comm.cpp.

{
}

Member Function Documentation

template<class T >

void Nektar::LibUtilities::Comm::AllReduce	(	T &	pData,
		enum ReduceOperator	pOp
	)

Definition at line 268 of file Comm.h.

References v_AllReduce().

{
    v_AllReduce(CommDataTypeTraits<T>::GetPointer(pData),
                CommDataTypeTraits<T>::GetCount(pData),
                CommDataTypeTraits<T>::GetDataType(), pOp);
}

template<class T >

void Nektar::LibUtilities::Comm::AlltoAll	(	T &	pSendData,
		T &	pRecvData
	)

Definition at line 275 of file Comm.h.

References ASSERTL0, Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), GetSize(), and v_AlltoAll().

{
    BOOST_STATIC_ASSERT_MSG(
        CommDataTypeTraits<T>::IsVector,
        "AlltoAll only valid with Array or vector arguments.");
    int sendSize = CommDataTypeTraits<T>::GetCount(pSendData);
    int recvSize = CommDataTypeTraits<T>::GetCount(pRecvData);
    ASSERTL0(sendSize == recvSize,
             "Send and Recv arrays have incompatible sizes in AlltoAll");

    int count = sendSize / GetSize();
    ASSERTL0(count * GetSize() == sendSize,
             "Array size incompatible with size of communicator");

    v_AlltoAll(CommDataTypeTraits<T>::GetPointer(pSendData), count,
               CommDataTypeTraits<T>::GetDataType(),
               CommDataTypeTraits<T>::GetPointer(pRecvData), count,
               CommDataTypeTraits<T>::GetDataType());
}

template<class T >

void Nektar::LibUtilities::Comm::AlltoAllv	(	Array< OneD, T > &	pSendData,
		Array< OneD, int > &	pSendDataSizeMap,
		Array< OneD, int > &	pSendDataOffsetMap,
		Array< OneD, T > &	pRecvData,
		Array< OneD, int > &	pRecvDataSizeMap,
		Array< OneD, int > &	pRecvDataOffsetMap
	)

Definition at line 299 of file Comm.h.

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetDataType(), and v_AlltoAllv().

{
    v_AlltoAllv(pSendData.get(), pSendDataSizeMap.get(),
                pSendDataOffsetMap.get(), CommDataTypeTraits<T>::GetDataType(),
                pRecvData.get(), pRecvDataSizeMap.get(),
                pRecvDataOffsetMap.get(), CommDataTypeTraits<T>::GetDataType());
}

template<class T >

void Nektar::LibUtilities::Comm::Bcast	(	T &	data,
		int	rootProc
	)

Definition at line 315 of file Comm.h.

References v_Bcast().

{
    v_Bcast(CommDataTypeTraits<T>::GetPointer(pData),
            CommDataTypeTraits<T>::GetCount(pData),
            CommDataTypeTraits<T>::GetDataType(), pRoot);
}

void Nektar::LibUtilities::Comm::Block ( )

inline

Block execution until all processes reach this point.

Definition at line 212 of file Comm.h.

References v_Block().

{
    v_Block();
}

CommSharedPtr Nektar::LibUtilities::Comm::CommCreateIf ( int  flag )

inline

If the flag is non-zero create a new communicator.

Definition at line 379 of file Comm.h.

References v_CommCreateIf().

{
    return v_CommCreateIf(flag);
}

template<class T >

void Nektar::LibUtilities::Comm::Exscan	(	T &	pData,
		const enum ReduceOperator	pOp,
		T &	ans
	)

Definition at line 323 of file Comm.h.

References ASSERTL0, and v_Exscan().

{
    ASSERTL0(CommDataTypeTraits<T>::GetCount(pData) ==
                 CommDataTypeTraits<T>::GetCount(ans),
             "Input and output array sizes don't match");
    v_Exscan(CommDataTypeTraits<T>::GetPointer(pData),
             CommDataTypeTraits<T>::GetPointer(ans),
             CommDataTypeTraits<T>::GetCount(pData),
             CommDataTypeTraits<T>::GetDataType(), pOp);
}

void Nektar::LibUtilities::Comm::Finalise ( )

inline

Definition at line 180 of file Comm.h.

References v_Finalise().

{
    v_Finalise();
}

template<class T >

T Nektar::LibUtilities::Comm::Gather	(	const int	rootProc,
		T &	val
	)

Concatenate all the input arrays, in rank order, onto the process with rank == rootProc

Definition at line 338 of file Comm.h.

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), Nektar::LibUtilities::CommDataTypeTraits< T >::GetPointer(), GetRank(), GetSize(), and v_Gather().

{
    BOOST_STATIC_ASSERT_MSG(
        CommDataTypeTraits<T>::IsVector,
        "Gather only valid with Array or vector arguments.");
    bool amRoot  = (GetRank() == rootProc);
    unsigned nEl = CommDataTypeTraits<T>::GetCount(val);

    unsigned nOut = amRoot ? GetSize() * nEl : 0;
    T ans(nOut);
    void *recvbuf = amRoot ? CommDataTypeTraits<T>::GetPointer(ans) : NULL;

    v_Gather(CommDataTypeTraits<T>::GetPointer(val), nEl,
             CommDataTypeTraits<T>::GetDataType(), recvbuf, nEl,
             CommDataTypeTraits<T>::GetDataType(), rootProc);
    return ans;
}

CommSharedPtr Nektar::LibUtilities::Comm::GetColumnComm ( )

inline

Retrieve the column communicator to which this process belongs.

Definition at line 413 of file Comm.h.

References m_commColumn.

{
    if (!m_commColumn.get())
    {
        return shared_from_this();
    }
    else
    {
        return m_commColumn;
    }
}

int Nektar::LibUtilities::Comm::GetRank ( )

inline

Definition at line 196 of file Comm.h.

References v_GetRank().

Referenced by Gather(), and Scatter().

{
    return v_GetRank();
}

CommSharedPtr Nektar::LibUtilities::Comm::GetRowComm ( )

inline

Retrieve the row communicator to which this process belongs.

Definition at line 397 of file Comm.h.

References m_commRow.

{
    if (!m_commRow.get())
    {
        return shared_from_this();
    }
    else
    {
        return m_commRow;
    }
}

int Nektar::LibUtilities::Comm::GetSize ( )

inline

Returns number of processes.

Definition at line 188 of file Comm.h.

References m_size.

Referenced by AlltoAll(), Gather(), Scatter(), and Nektar::LibUtilities::CommMpi::v_AllReduce().

{
    return m_size;
}

const std::string & Nektar::LibUtilities::Comm::GetType ( ) const

inline

Definition at line 204 of file Comm.h.

References m_type.

{
    return m_type;
}

template<class T >

void Nektar::LibUtilities::Comm::Recv	(	int	pProc,
		T &	pData
	)

Definition at line 232 of file Comm.h.

References v_Recv().

{
    v_Recv(CommDataTypeTraits<T>::GetPointer(pData),
           CommDataTypeTraits<T>::GetCount(pData),
           CommDataTypeTraits<T>::GetDataType(), pProc);
}

bool Nektar::LibUtilities::Comm::RemoveExistingFiles ( void  )

inline

Definition at line 430 of file Comm.h.

References v_RemoveExistingFiles().

{
    return v_RemoveExistingFiles();
}

template<class T >

T Nektar::LibUtilities::Comm::Scatter	(	const int	rootProc,
		T &	pData
	)

Scatter pData across ranks in chunks of len(pData)/num_ranks

Definition at line 358 of file Comm.h.

References Nektar::LibUtilities::CommDataTypeTraits< T >::GetCount(), Nektar::LibUtilities::CommDataTypeTraits< T >::GetPointer(), GetRank(), GetSize(), and v_Scatter().

{
    BOOST_STATIC_ASSERT_MSG(
        CommDataTypeTraits<T>::IsVector,
        "Scatter only valid with Array or vector arguments.");

    bool amRoot  = (GetRank() == rootProc);
    unsigned nEl = CommDataTypeTraits<T>::GetCount(pData) / GetSize();

    void *sendbuf = amRoot ? CommDataTypeTraits<T>::GetPointer(pData) : NULL;
    T ans(nEl);

    v_Scatter(sendbuf, nEl, CommDataTypeTraits<T>::GetDataType(),
              CommDataTypeTraits<T>::GetPointer(ans), nEl,
              CommDataTypeTraits<T>::GetDataType(), rootProc);
    return ans;
}

template<class T >

void Nektar::LibUtilities::Comm::Send	(	int	pProc,
		T &	pData
	)

Definition at line 225 of file Comm.h.

References v_Send().

{
    v_Send(CommDataTypeTraits<T>::GetPointer(pData),
           CommDataTypeTraits<T>::GetCount(pData),
           CommDataTypeTraits<T>::GetDataType(), pProc);
}

template<class T >

void Nektar::LibUtilities::Comm::SendRecv	(	int	pSendProc,
		T &	pSendData,
		int	pRecvProc,
		T &	pRecvData
	)

Definition at line 243 of file Comm.h.

References v_SendRecv().

{
    v_SendRecv(CommDataTypeTraits<T>::GetPointer(pSendData),
               CommDataTypeTraits<T>::GetCount(pSendData),
               CommDataTypeTraits<T>::GetDataType(), pSendProc,
               CommDataTypeTraits<T>::GetPointer(pRecvData),
               CommDataTypeTraits<T>::GetCount(pRecvData),
               CommDataTypeTraits<T>::GetDataType(), pRecvProc);
}

template<class T >

void Nektar::LibUtilities::Comm::SendRecvReplace	(	int	pSendProc,
		int	pRecvProc,
		T &	pData
	)

Definition at line 257 of file Comm.h.

References v_SendRecvReplace().

{
    v_SendRecvReplace(CommDataTypeTraits<T>::GetPointer(pData),
                      CommDataTypeTraits<T>::GetCount(pData),
                      CommDataTypeTraits<T>::GetDataType(), pSendProc,
                      pRecvProc);
}

void Nektar::LibUtilities::Comm::SplitComm ( int  pRows, int  pColumns  )

inline

Splits this communicator into a grid of size pRows*pColumns and creates row and column communicators. By default the communicator is a single row.

Definition at line 389 of file Comm.h.

References v_SplitComm().

{
    v_SplitComm(pRows, pColumns);
}

bool Nektar::LibUtilities::Comm::TreatAsRankZero ( void  )

inline

Definition at line 425 of file Comm.h.

References v_TreatAsRankZero().

{
    return v_TreatAsRankZero();
}

virtual void Nektar::LibUtilities::Comm::v_AllReduce ( void *  buf, int  count, CommDataType  dt, enum ReduceOperator  pOp  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by AllReduce().

virtual void Nektar::LibUtilities::Comm::v_AlltoAll ( void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by AlltoAll().

virtual void Nektar::LibUtilities::Comm::v_AlltoAllv ( void *  sendbuf, int  sendcounts[], int  sensdispls[], CommDataType  sendtype, void *  recvbuf, int  recvcounts[], int  rdispls[], CommDataType  recvtype  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by AlltoAllv().

virtual void Nektar::LibUtilities::Comm::v_Bcast ( void *  buffer, int  count, CommDataType  dt, int  root  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Bcast().

virtual void Nektar::LibUtilities::Comm::v_Block ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Block().

virtual CommSharedPtr Nektar::LibUtilities::Comm::v_CommCreateIf ( int  flag )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by CommCreateIf().

virtual void Nektar::LibUtilities::Comm::v_Exscan ( Array< OneD, unsigned long long > &  pData, const enum ReduceOperator  pOp, Array< OneD, unsigned long long > &  ans  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Exscan().

virtual void Nektar::LibUtilities::Comm::v_Finalise ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Finalise().

virtual void Nektar::LibUtilities::Comm::v_Gather ( void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype, int  root  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Gather().

virtual int Nektar::LibUtilities::Comm::v_GetRank ( )

protectedpure virtual

Implemented in Nektar::FieldUtils::FieldConvertComm, Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by GetRank().

virtual void Nektar::LibUtilities::Comm::v_Recv ( void *  buf, int  count, CommDataType  dt, int  source  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Recv().

bool Nektar::LibUtilities::Comm::v_RemoveExistingFiles ( void  )

protectedvirtual

Reimplemented in Nektar::FieldUtils::FieldConvertComm.

Definition at line 56 of file Comm.cpp.

Referenced by RemoveExistingFiles().

{
    return true;
}

virtual void Nektar::LibUtilities::Comm::v_Scatter ( void *  sendbuf, int  sendcount, CommDataType  sendtype, void *  recvbuf, int  recvcount, CommDataType  recvtype, int  root  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Scatter().

virtual void Nektar::LibUtilities::Comm::v_Send ( void *  buf, int  count, CommDataType  dt, int  dest  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Send().

virtual void Nektar::LibUtilities::Comm::v_SendRecv ( void *  sendbuf, int  sendcount, CommDataType  sendtype, int  dest, void *  recvbuf, int  recvcount, CommDataType  recvtype, int  source  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by SendRecv().

virtual void Nektar::LibUtilities::Comm::v_SendRecvReplace ( void *  buf, int  count, CommDataType  dt, int  pSendProc, int  pRecvProc  )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by SendRecvReplace().

virtual void Nektar::LibUtilities::Comm::v_SplitComm ( int  pRows, int  pColumns  )

protectedpure virtual

Implemented in Nektar::FieldUtils::FieldConvertComm, Nektar::LibUtilities::CommSerial, and Nektar::LibUtilities::CommMpi.

Referenced by SplitComm().

virtual bool Nektar::LibUtilities::Comm::v_TreatAsRankZero ( void  )

protectedpure virtual

Implemented in Nektar::FieldUtils::FieldConvertComm, Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by TreatAsRankZero().

virtual NekDouble Nektar::LibUtilities::Comm::v_Wtime ( )

protectedpure virtual

Implemented in Nektar::LibUtilities::CommMpi, and Nektar::LibUtilities::CommSerial.

Referenced by Wtime().

double Nektar::LibUtilities::Comm::Wtime ( )

inline

Return the time in seconds.

Definition at line 220 of file Comm.h.

References v_Wtime().

{
    return v_Wtime();
}

Member Data Documentation

CommSharedPtr Nektar::LibUtilities::Comm::m_commColumn

protected

Column communicator.

Definition at line 133 of file Comm.h.

Referenced by GetColumnComm(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

CommSharedPtr Nektar::LibUtilities::Comm::m_commRow

protected

Row communicator.

Definition at line 132 of file Comm.h.

Referenced by GetRowComm(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

int Nektar::LibUtilities::Comm::m_size

protected

Number of processes.

Definition at line 130 of file Comm.h.

Referenced by Nektar::LibUtilities::CommMpi::CommMpi(), Nektar::LibUtilities::CommSerial::CommSerial(), GetSize(), and Nektar::LibUtilities::CommMpi::v_SplitComm().

std::string Nektar::LibUtilities::Comm::m_type

protected

Type of communication.

Definition at line 131 of file Comm.h.

Referenced by Nektar::LibUtilities::CommMpi::CommMpi(), Nektar::LibUtilities::CommSerial::CommSerial(), and GetType().