Nektar++
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Comm.h
Go to the documentation of this file.
1 ///////////////////////////////////////////////////////////////////////////////
2 //
3 // File Comm.h
4 //
5 // For more information, please see: http://www.nektar.info
6 //
7 // The MIT License
8 //
9 // Copyright (c) 2006 Division of Applied Mathematics, Brown University (USA),
10 // Department of Aeronautics, Imperial College London (UK), and Scientific
11 // Computing and Imaging Institute, University of Utah (USA).
12 //
13 // License for the specific language governing rights and limitations under
14 // Permission is hereby granted, free of charge, to any person obtaining a
15 // copy of this software and associated documentation files (the "Software"),
16 // to deal in the Software without restriction, including without limitation
17 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
18 // and/or sell copies of the Software, and to permit persons to whom the
19 // Software is furnished to do so, subject to the following conditions:
20 //
21 // The above copyright notice and this permission notice shall be included
22 // in all copies or substantial portions of the Software.
23 //
24 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
25 // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
26 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
27 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
28 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
29 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
30 // DEALINGS IN THE SOFTWARE.
31 //
32 // Description: Base communication class
33 //
34 ///////////////////////////////////////////////////////////////////////////////
35 #ifndef NEKTAR_LIB_UTILITIES_COMM_H
36 #define NEKTAR_LIB_UTILITIES_COMM_H
37 
38 #include <vector>
39 
42 #include <boost/enable_shared_from_this.hpp>
43 #include <boost/static_assert.hpp>
44 
46 // namespace Nektar { template <typename Dim, typename DataType> class Array; }
49 
50 namespace Nektar
51 {
52 namespace LibUtilities
53 {
54 // Forward declarations
55 class Comm;
56 
57 /// Pointer to a Communicator object.
58 typedef boost::shared_ptr<Comm> CommSharedPtr;
59 
60 /// Datatype of the NekFactory used to instantiate classes derived from
61 /// the EquationSystem class.
63 
65 
66 /// Type of operation to perform in AllReduce.
68 {
72 };
73 
74 /// Base communications class
75 class Comm : public boost::enable_shared_from_this<Comm>
76 {
77 public:
78  LIB_UTILITIES_EXPORT Comm(int narg, char *arg[]);
79  LIB_UTILITIES_EXPORT virtual ~Comm();
80 
81  LIB_UTILITIES_EXPORT inline void Finalise();
82 
83  /// Returns number of processes
84  LIB_UTILITIES_EXPORT inline int GetSize();
85  LIB_UTILITIES_EXPORT inline int GetRank();
86  LIB_UTILITIES_EXPORT inline const std::string &GetType() const;
87 
88  /// Block execution until all processes reach this point
89  LIB_UTILITIES_EXPORT inline void Block();
90 
91  /// Return the time in seconds
93 
94  template <class T> void Send(int pProc, T &pData);
95  template <class T> void Recv(int pProc, T &pData);
96  template <class T>
97  void SendRecv(int pSendProc, T &pSendData, int pRecvProc, T &pRecvData);
98  template <class T>
99  void SendRecvReplace(int pSendProc, int pRecvProc, T &pData);
100 
101  template <class T> void AllReduce(T &pData, enum ReduceOperator pOp);
102 
103  template <class T> void AlltoAll(T &pSendData, T &pRecvData);
104  template <class T>
105  void AlltoAllv(Array<OneD, T> &pSendData,
106  Array<OneD, int> &pSendDataSizeMap,
107  Array<OneD, int> &pSendDataOffsetMap,
108  Array<OneD, T> &pRecvData,
109  Array<OneD, int> &pRecvDataSizeMap,
110  Array<OneD, int> &pRecvDataOffsetMap);
111 
112  template <class T> void Bcast(T &data, int rootProc);
113 
114  template <class T>
115  void Exscan(T &pData, const enum ReduceOperator pOp, T &ans);
116 
117  template <class T> T Gather(const int rootProc, T &val);
118  template <class T> T Scatter(const int rootProc, T &pData);
119 
120  LIB_UTILITIES_EXPORT inline CommSharedPtr CommCreateIf(int flag);
121 
122  LIB_UTILITIES_EXPORT inline void SplitComm(int pRows, int pColumns);
123  LIB_UTILITIES_EXPORT inline CommSharedPtr GetRowComm();
124  LIB_UTILITIES_EXPORT inline CommSharedPtr GetColumnComm();
125 
126  LIB_UTILITIES_EXPORT inline bool TreatAsRankZero(void);
127  LIB_UTILITIES_EXPORT inline bool RemoveExistingFiles(void);
128 
129 protected:
130  int m_size; ///< Number of processes
131  std::string m_type; ///< Type of communication
132  CommSharedPtr m_commRow; ///< Row communicator
133  CommSharedPtr m_commColumn; ///< Column communicator
134 
135  Comm();
136 
137  virtual void v_Finalise() = 0;
138  virtual int v_GetRank() = 0;
139  virtual void v_Block() = 0;
140  virtual NekDouble v_Wtime() = 0;
141  virtual void v_Send(void *buf, int count, CommDataType dt, int dest) = 0;
142  virtual void v_Recv(void *buf, int count, CommDataType dt, int source) = 0;
143  virtual void v_SendRecv(void *sendbuf, int sendcount, CommDataType sendtype,
144  int dest, void *recvbuf, int recvcount,
145  CommDataType recvtype, int source) = 0;
146  virtual void v_SendRecvReplace(void *buf, int count, CommDataType dt,
147  int pSendProc, int pRecvProc) = 0;
148  virtual void v_AllReduce(void *buf, int count, CommDataType dt,
149  enum ReduceOperator pOp) = 0;
150  virtual void v_AlltoAll(void *sendbuf, int sendcount, CommDataType sendtype,
151  void *recvbuf, int recvcount,
152  CommDataType recvtype) = 0;
153  virtual void v_AlltoAllv(void *sendbuf, int sendcounts[], int sensdispls[],
154  CommDataType sendtype, void *recvbuf,
155  int recvcounts[], int rdispls[],
156  CommDataType recvtype) = 0;
157  virtual void v_Bcast(void *buffer, int count, CommDataType dt,
158  int root) = 0;
159 
160  virtual void v_Exscan(Array<OneD, unsigned long long> &pData,
161  const enum ReduceOperator pOp,
163 
164  virtual void v_Gather(void *sendbuf, int sendcount, CommDataType sendtype,
165  void *recvbuf, int recvcount, CommDataType recvtype,
166  int root) = 0;
167  virtual void v_Scatter(void *sendbuf, int sendcount, CommDataType sendtype,
168  void *recvbuf, int recvcount, CommDataType recvtype,
169  int root) = 0;
170 
171  virtual CommSharedPtr v_CommCreateIf(int flag) = 0;
172  virtual void v_SplitComm(int pRows, int pColumns) = 0;
173  virtual bool v_TreatAsRankZero(void) = 0;
174  LIB_UTILITIES_EXPORT virtual bool v_RemoveExistingFiles(void);
175 };
176 
177 /**
178  *
179  */
180 inline void Comm::Finalise()
181 {
182  v_Finalise();
183 }
184 
185 /**
186  *
187  */
188 inline int Comm::GetSize()
189 {
190  return m_size;
191 }
192 
193 /**
194  *
195  */
196 inline int Comm::GetRank()
197 {
198  return v_GetRank();
199 }
200 
201 /**
202  *
203  */
204 inline const std::string &Comm::GetType() const
205 {
206  return m_type;
207 }
208 
209 /**
210  *
211  */
212 inline void Comm::Block()
213 {
214  v_Block();
215 }
216 
217 /**
218  *
219  */
220 inline double Comm::Wtime()
221 {
222  return v_Wtime();
223 }
224 
225 template <class T> void Comm::Send(int pProc, T &pData)
226 {
230 }
231 
232 template <class T> void Comm::Recv(int pProc, T &pData)
233 {
237 }
238 
239 /**
240  *
241  */
242 template <class T>
243 void Comm::SendRecv(int pSendProc, T &pSendData, int pRecvProc, T &pRecvData)
244 {
251 }
252 
253 /**
254  *
255  */
256 template <class T>
257 void Comm::SendRecvReplace(int pSendProc, int pRecvProc, T &pData)
258 {
262  pRecvProc);
263 }
264 
265 /**
266  *
267  */
268 template <class T> void Comm::AllReduce(T &pData, enum ReduceOperator pOp)
269 {
273 }
274 
275 template <class T> void Comm::AlltoAll(T &pSendData, T &pRecvData)
276 {
277  BOOST_STATIC_ASSERT_MSG(
279  "AlltoAll only valid with Array or vector arguments.");
280  int sendSize = CommDataTypeTraits<T>::GetCount(pSendData);
281  int recvSize = CommDataTypeTraits<T>::GetCount(pRecvData);
282  ASSERTL0(sendSize == recvSize,
283  "Send and Recv arrays have incompatible sizes in AlltoAll");
284 
285  int count = sendSize / GetSize();
286  ASSERTL0(count * GetSize() == sendSize,
287  "Array size incompatible with size of communicator");
288 
291  CommDataTypeTraits<T>::GetPointer(pRecvData), count,
293 }
294 
295 /**
296  *
297  */
298 template <class T>
300  Array<OneD, int> &pSendDataSizeMap,
301  Array<OneD, int> &pSendDataOffsetMap,
302  Array<OneD, T> &pRecvData,
303  Array<OneD, int> &pRecvDataSizeMap,
304  Array<OneD, int> &pRecvDataOffsetMap)
305 {
306  v_AlltoAllv(pSendData.get(), pSendDataSizeMap.get(),
307  pSendDataOffsetMap.get(), CommDataTypeTraits<T>::GetDataType(),
308  pRecvData.get(), pRecvDataSizeMap.get(),
309  pRecvDataOffsetMap.get(), CommDataTypeTraits<T>::GetDataType());
310 }
311 
312 /**
313  *
314  */
315 template <class T> void Comm::Bcast(T &pData, int pRoot)
316 {
320 }
321 
322 template <class T>
323 void Comm::Exscan(T &pData, const enum ReduceOperator pOp, T &ans)
324 {
327  "Input and output array sizes don't match");
332 }
333 
334 /**
335  * Concatenate all the input arrays, in rank order, onto the process with rank
336  * == rootProc
337  */
338 template <class T> T Comm::Gather(const int rootProc, T &val)
339 {
340  BOOST_STATIC_ASSERT_MSG(
342  "Gather only valid with Array or vector arguments.");
343  bool amRoot = (GetRank() == rootProc);
344  unsigned nEl = CommDataTypeTraits<T>::GetCount(val);
345 
346  unsigned nOut = amRoot ? GetSize() * nEl : 0;
347  T ans(nOut);
348  void *recvbuf = amRoot ? CommDataTypeTraits<T>::GetPointer(ans) : NULL;
349 
350  v_Gather(CommDataTypeTraits<T>::GetPointer(val), nEl,
351  CommDataTypeTraits<T>::GetDataType(), recvbuf, nEl,
353  return ans;
354 }
355 /**
356  * Scatter pData across ranks in chunks of len(pData)/num_ranks
357  */
358 template <class T> T Comm::Scatter(const int rootProc, T &pData)
359 {
360  BOOST_STATIC_ASSERT_MSG(
362  "Scatter only valid with Array or vector arguments.");
363 
364  bool amRoot = (GetRank() == rootProc);
365  unsigned nEl = CommDataTypeTraits<T>::GetCount(pData) / GetSize();
366 
367  void *sendbuf = amRoot ? CommDataTypeTraits<T>::GetPointer(pData) : NULL;
368  T ans(nEl);
369 
371  CommDataTypeTraits<T>::GetPointer(ans), nEl,
373  return ans;
374 }
375 
376 /**
377  * @brief If the flag is non-zero create a new communicator.
378  */
379 inline CommSharedPtr Comm::CommCreateIf(int flag)
380 {
381  return v_CommCreateIf(flag);
382 }
383 
384 /**
385  * @brief Splits this communicator into a grid of size pRows*pColumns
386  * and creates row and column communicators. By default the communicator
387  * is a single row.
388  */
389 inline void Comm::SplitComm(int pRows, int pColumns)
390 {
391  v_SplitComm(pRows, pColumns);
392 }
393 
394 /**
395  * @brief Retrieve the row communicator to which this process belongs.
396  */
397 inline CommSharedPtr Comm::GetRowComm()
398 {
399  if (!m_commRow.get())
400  {
401  return shared_from_this();
402  }
403  else
404  {
405  return m_commRow;
406  }
407 }
408 
409 /**
410  * @brief Retrieve the column communicator to which this process
411  * belongs.
412  */
413 inline CommSharedPtr Comm::GetColumnComm()
414 {
415  if (!m_commColumn.get())
416  {
417  return shared_from_this();
418  }
419  else
420  {
421  return m_commColumn;
422  }
423 }
424 
425 inline bool Comm::TreatAsRankZero(void)
426 {
427  return v_TreatAsRankZero();
428 }
429 
430 inline bool Comm::RemoveExistingFiles(void)
431 {
432  return v_RemoveExistingFiles();
433 }
434 }
435 }
436 
437 #endif
void Exscan(T &pData, const enum ReduceOperator pOp, T &ans)
Definition: Comm.h:323
void Recv(int pProc, T &pData)
Definition: Comm.h:232
void AllReduce(T &pData, enum ReduceOperator pOp)
Definition: Comm.h:268
#define ASSERTL0(condition, msg)
Definition: ErrorUtil.hpp:198
virtual void v_Recv(void *buf, int count, CommDataType dt, int source)=0
ReduceOperator
Type of operation to perform in AllReduce.
Definition: Comm.h:67
CommSharedPtr m_commColumn
Column communicator.
Definition: Comm.h:133
virtual void v_AllReduce(void *buf, int count, CommDataType dt, enum ReduceOperator pOp)=0
std::string m_type
Type of communication.
Definition: Comm.h:131
bool TreatAsRankZero(void)
Definition: Comm.h:425
array buffer
Definition: GsLib.hpp:60
virtual void v_AlltoAllv(void *sendbuf, int sendcounts[], int sensdispls[], CommDataType sendtype, void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype)=0
static int GetCount(const T &val)
Definition: CommDataType.h:96
void AlltoAllv(Array< OneD, T > &pSendData, Array< OneD, int > &pSendDataSizeMap, Array< OneD, int > &pSendDataOffsetMap, Array< OneD, T > &pRecvData, Array< OneD, int > &pRecvDataSizeMap, Array< OneD, int > &pRecvDataOffsetMap)
Definition: Comm.h:299
virtual void v_Send(void *buf, int count, CommDataType dt, int dest)=0
CommFactory & GetCommFactory()
Definition: Comm.cpp:61
CommSharedPtr m_commRow
Row communicator.
Definition: Comm.h:132
void AlltoAll(T &pSendData, T &pRecvData)
Definition: Comm.h:275
void SendRecv(int pSendProc, T &pSendData, int pRecvProc, T &pRecvData)
Definition: Comm.h:243
virtual void v_SendRecv(void *sendbuf, int sendcount, CommDataType sendtype, int dest, void *recvbuf, int recvcount, CommDataType recvtype, int source)=0
virtual bool v_RemoveExistingFiles(void)
Definition: Comm.cpp:56
virtual void v_Exscan(Array< OneD, unsigned long long > &pData, const enum ReduceOperator pOp, Array< OneD, unsigned long long > &ans)=0
virtual void v_Bcast(void *buffer, int count, CommDataType dt, int root)=0
NekDouble Wtime()
Return the time in seconds.
Definition: Comm.h:220
CommSharedPtr GetRowComm()
Retrieve the row communicator to which this process belongs.
Definition: Comm.h:397
boost::shared_ptr< Comm > CommSharedPtr
Pointer to a Communicator object.
Definition: Comm.h:55
virtual void v_Gather(void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype, int root)=0
virtual CommSharedPtr v_CommCreateIf(int flag)=0
T Scatter(const int rootProc, T &pData)
Definition: Comm.h:358
LibUtilities::NekFactory< std::string, Comm, int, char ** > CommFactory
Datatype of the NekFactory used to instantiate classes derived from the EquationSystem class...
Definition: Comm.h:62
virtual void v_Scatter(void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype, int root)=0
const std::string & GetType() const
Definition: Comm.h:204
#define LIB_UTILITIES_EXPORT
virtual int v_GetRank()=0
void Bcast(T &data, int rootProc)
Definition: Comm.h:315
virtual bool v_TreatAsRankZero(void)=0
virtual void v_Finalise()=0
void Send(int pProc, T &pData)
Definition: Comm.h:225
double NekDouble
void SplitComm(int pRows, int pColumns)
Splits this communicator into a grid of size pRows*pColumns and creates row and column communicators...
Definition: Comm.h:389
Base communications class.
Definition: Comm.h:75
virtual void v_AlltoAll(void *sendbuf, int sendcount, CommDataType sendtype, void *recvbuf, int recvcount, CommDataType recvtype)=0
void Block()
Block execution until all processes reach this point.
Definition: Comm.h:212
T Gather(const int rootProc, T &val)
Definition: Comm.h:338
#define dest(otri, vertexptr)
void SendRecvReplace(int pSendProc, int pRecvProc, T &pData)
Definition: Comm.h:257
int GetSize()
Returns number of processes.
Definition: Comm.h:188
virtual void v_SplitComm(int pRows, int pColumns)=0
CommSharedPtr CommCreateIf(int flag)
If the flag is non-zero create a new communicator.
Definition: Comm.h:379
virtual NekDouble v_Wtime()=0
virtual void v_Block()=0
virtual void v_SendRecvReplace(void *buf, int count, CommDataType dt, int pSendProc, int pRecvProc)=0
int m_size
Number of processes.
Definition: Comm.h:130
bool RemoveExistingFiles(void)
Definition: Comm.h:430
Provides a generic Factory class.
Definition: NekFactory.hpp:116
CommSharedPtr GetColumnComm()
Retrieve the column communicator to which this process belongs.
Definition: Comm.h:413