Nektar++
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Comm.h
Go to the documentation of this file.
1 ///////////////////////////////////////////////////////////////////////////////
2 //
3 // File Comm.h
4 //
5 // For more information, please see: http://www.nektar.info
6 //
7 // The MIT License
8 //
9 // Copyright (c) 2006 Division of Applied Mathematics, Brown University (USA),
10 // Department of Aeronautics, Imperial College London (UK), and Scientific
11 // Computing and Imaging Institute, University of Utah (USA).
12 //
13 // License for the specific language governing rights and limitations under
14 // Permission is hereby granted, free of charge, to any person obtaining a
15 // copy of this software and associated documentation files (the "Software"),
16 // to deal in the Software without restriction, including without limitation
17 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
18 // and/or sell copies of the Software, and to permit persons to whom the
19 // Software is furnished to do so, subject to the following conditions:
20 //
21 // The above copyright notice and this permission notice shall be included
22 // in all copies or substantial portions of the Software.
23 //
24 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
25 // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
26 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
27 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
28 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
29 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
30 // DEALINGS IN THE SOFTWARE.
31 //
32 // Description: Base communication class
33 //
34 ///////////////////////////////////////////////////////////////////////////////
35 #ifndef NEKTAR_LIB_UTILITIES_COMM_H
36 #define NEKTAR_LIB_UTILITIES_COMM_H
37 
38 #include <vector>
39 
40 #include <boost/enable_shared_from_this.hpp>
43 
45 namespace Nektar { template <typename Dim, typename DataType> class Array; }
46 
47 
48 namespace Nektar
49 {
50  namespace LibUtilities
51  {
52  // Forward declarations
53  class Comm;
54 
55  /// Pointer to a Communicator object.
56  typedef boost::shared_ptr<Comm> CommSharedPtr;
57 
58  /// Datatype of the NekFactory used to instantiate classes derived from
59  /// the EquationSystem class.
61 
63 
64  /// Type of operation to perform in AllReduce.
66  {
70  };
71 
72  /// Base communications class
73  class Comm: public boost::enable_shared_from_this<Comm>
74  {
75  public:
76  LIB_UTILITIES_EXPORT Comm(int narg, char* arg[]);
77  LIB_UTILITIES_EXPORT virtual ~Comm();
78 
79  LIB_UTILITIES_EXPORT inline void Finalise();
80 
81  /// Returns number of processes
82  LIB_UTILITIES_EXPORT inline int GetSize();
83  LIB_UTILITIES_EXPORT inline int GetRank();
84  LIB_UTILITIES_EXPORT inline const std::string& GetType() const;
85 
86  /// Block execution until all processes reach this point
87  LIB_UTILITIES_EXPORT inline void Block();
88  LIB_UTILITIES_EXPORT inline void Send(int pProc, Array<OneD, NekDouble>& pData);
89  LIB_UTILITIES_EXPORT inline void Send(int pProc, Array<OneD, int>& pData);
90  LIB_UTILITIES_EXPORT inline void Send(int pProc, std::vector<unsigned int>& pData);
91  LIB_UTILITIES_EXPORT inline void Recv(int pProc, Array<OneD, NekDouble>& pData);
92  LIB_UTILITIES_EXPORT inline void Recv(int pProc, Array<OneD, int>& pData);
93  LIB_UTILITIES_EXPORT inline void Recv(int pProc, std::vector<unsigned int>& pData);
94  LIB_UTILITIES_EXPORT inline void SendRecv(int pSendProc,
95  Array<OneD, NekDouble>& pSendData,
96  int pRecvProc,
97  Array<OneD, NekDouble>& pRecvData);
98  LIB_UTILITIES_EXPORT inline void SendRecv(int pSendProc,
99  Array<OneD, int>& pSendData,
100  int pRecvProc,
101  Array<OneD, int>& pRecvData);
102  LIB_UTILITIES_EXPORT inline void SendRecvReplace(int pSendProc,
103  int pRecvProc,
104  Array<OneD, NekDouble>& pSendData);
105  LIB_UTILITIES_EXPORT inline void SendRecvReplace(int pSendProc,
106  int pRecvProc,
107  Array<OneD, int>& pSendData);
108  LIB_UTILITIES_EXPORT inline void AllReduce(NekDouble& pData, enum ReduceOperator pOp);
109  LIB_UTILITIES_EXPORT inline void AllReduce(int& pData, enum ReduceOperator pOp);
110  LIB_UTILITIES_EXPORT inline void AllReduce(Array<OneD, NekDouble>& pData,
111  enum ReduceOperator pOp);
112  LIB_UTILITIES_EXPORT inline void AllReduce(Array<OneD, int >& pData,
113  enum ReduceOperator pOp);
114  LIB_UTILITIES_EXPORT inline void AllReduce(std::vector<unsigned int>& pData,
115  enum ReduceOperator pOp);
116  LIB_UTILITIES_EXPORT inline void AlltoAll(Array<OneD, NekDouble>& pSendData,
117  Array<OneD, NekDouble>& pRecvData);
118  LIB_UTILITIES_EXPORT inline void AlltoAll(Array<OneD, int>& pSendData,
119  Array<OneD, int>& pRecvData);
120  LIB_UTILITIES_EXPORT inline void AlltoAllv(Array<OneD, NekDouble>& pSendData,
121  Array<OneD, int>& pSendDataSizeMap,
122  Array<OneD, int>& pSendDataOffsetMap,
123  Array<OneD, NekDouble>& pRecvData,
124  Array<OneD, int>& pRecvDataSizeMap,
125  Array<OneD, int>& pRecvDataOffsetMap);
126  LIB_UTILITIES_EXPORT inline void AlltoAllv(Array<OneD, int>& pSendData,
127  Array<OneD, int>& pSendDataSizeMap,
128  Array<OneD, int>& pSendDataOffsetMap,
129  Array<OneD, int>& pRecvData,
130  Array<OneD, int>& pRecvDataSizeMap,
131  Array<OneD, int>& pRecvDataOffsetMap);
132 
133  LIB_UTILITIES_EXPORT inline void SplitComm(int pRows, int pColumns);
136 
137 
138  LIB_UTILITIES_EXPORT inline bool TreatAsRankZero(void);
139  LIB_UTILITIES_EXPORT inline bool RemoveExistingFiles(void);
140 
141  protected:
142  int m_size; ///< Number of processes
143  std::string m_type; ///< Type of communication
144  CommSharedPtr m_commRow; ///< Row communicator
145  CommSharedPtr m_commColumn; ///< Column communicator
146 
147  Comm();
148 
149  virtual void v_Finalise() = 0;
150  virtual int v_GetRank() = 0;
151  virtual void v_Block() = 0;
152  virtual void v_Send(int pProc, Array<OneD, NekDouble>& pData) = 0;
153  virtual void v_Send(int pProc, Array<OneD, int>& pData) = 0;
154  virtual void v_Send(int pProc, std::vector<unsigned int>& pData) = 0;
155  virtual void v_Recv(int pProc, Array<OneD, NekDouble>& pData) = 0;
156  virtual void v_Recv(int pProc, Array<OneD, int>& pData) = 0;
157  virtual void v_Recv(int pProc, std::vector<unsigned int>& pData) = 0;
158  virtual void v_SendRecv(int pSendProc,
159  Array<OneD, NekDouble>& pSendData,
160  int pRecvProc,
161  Array<OneD, NekDouble>& pRecvData) = 0;
162  virtual void v_SendRecv(int pSendProc,
163  Array<OneD, int>& pSendData,
164  int pRecvProc,
165  Array<OneD, int>& pRecvData) = 0;
166  virtual void v_SendRecvReplace(int pSendProc,
167  int pRecvProc,
168  Array<OneD, NekDouble>& pSendData) = 0;
169  virtual void v_SendRecvReplace(int pSendProc,
170  int pRecvProc,
171  Array<OneD, int>& pSendData) = 0;
172  virtual void v_AllReduce(NekDouble& pData,
173  enum ReduceOperator pOp) = 0;
174  virtual void v_AllReduce(int& pData,
175  enum ReduceOperator pOp) = 0;
176  virtual void v_AllReduce(Array<OneD, NekDouble>& pData,
177  enum ReduceOperator pOp) = 0;
178  virtual void v_AllReduce(Array<OneD, int >& pData,
179  enum ReduceOperator pOp) = 0;
180  virtual void v_AllReduce(std::vector<unsigned int>& pData,
181  enum ReduceOperator pOp) = 0;
182  virtual void v_AlltoAll(Array<OneD, NekDouble>& pSendData,
183  Array<OneD, NekDouble>& pRecvData) = 0;
184  virtual void v_AlltoAll(Array<OneD, int>& pSendData,
185  Array<OneD, int>& pRecvData) = 0;
186  virtual void v_AlltoAllv(Array<OneD, NekDouble>& pSendData,
187  Array<OneD, int>& pSendDataSizeMap,
188  Array<OneD, int>& pSendDataOffsetMap,
189  Array<OneD, NekDouble>& pRecvData,
190  Array<OneD, int>& pRecvDataSizeMap,
191  Array<OneD, int>& pRecvDataOffsetMap) = 0;
192  virtual void v_AlltoAllv(Array<OneD, int>& pSendData,
193  Array<OneD, int>& pSendDataSizeMap,
194  Array<OneD, int>& pSendDataOffsetMap,
195  Array<OneD, int>& pRecvData,
196  Array<OneD, int>& pRecvDataSizeMap,
197  Array<OneD, int>& pRecvDataOffsetMap) = 0;
198  virtual void v_SplitComm(int pRows, int pColumns) = 0;
199  virtual bool v_TreatAsRankZero(void) = 0;
200  LIB_UTILITIES_EXPORT virtual bool v_RemoveExistingFiles(void);
201  };
202 
203 
204  /**
205  *
206  */
207  inline void Comm::Finalise()
208  {
209  v_Finalise();
210  }
211 
212  /**
213  *
214  */
215  inline int Comm::GetSize()
216  {
217  return m_size;
218  }
219 
220  /**
221  *
222  */
223  inline int Comm::GetRank()
224  {
225  return v_GetRank();
226  }
227 
228  /**
229  *
230  */
231  inline const std::string& Comm::GetType() const
232  {
233  return m_type;
234  }
235 
236  /**
237  *
238  */
239  inline void Comm::Block()
240  {
241  v_Block();
242  }
243 
244  /**
245  *
246  */
247  inline void Comm::Send(int pProc, Array<OneD, NekDouble>& pData)
248  {
249  v_Send(pProc, pData);
250  }
251 
252  /**
253  *
254  */
255  inline void Comm::Recv(int pProc, Array<OneD, NekDouble>& pData)
256  {
257  v_Recv(pProc, pData);
258  }
259 
260 
261  /**
262  *
263  */
264  inline void Comm::Send(int pProc, Array<OneD, int>& pData)
265  {
266  v_Send(pProc, pData);
267  }
268 
269  /**
270  *
271  */
272  inline void Comm::Recv(int pProc, Array<OneD, int>& pData)
273  {
274  v_Recv(pProc, pData);
275  }
276 
277  /**
278  *
279  */
280  inline void Comm::Send(int pProc, std::vector<unsigned int>& pData)
281  {
282  v_Send(pProc, pData);
283  }
284 
285  /**
286  *
287  */
288  inline void Comm::Recv(int pProc, std::vector<unsigned int>& pData)
289  {
290  v_Recv(pProc, pData);
291  }
292 
293  /**
294  *
295  */
296  inline void Comm::SendRecv(int pSendProc,
297  Array<OneD, NekDouble>& pSendData,
298  int pRecvProc,
299  Array<OneD, NekDouble>& pRecvData)
300  {
301  v_SendRecv(pSendProc, pSendData, pRecvProc, pRecvData);
302  }
303 
304 
305  /**
306  *
307  */
308  inline void Comm::SendRecv(int pSendProc,
309  Array<OneD, int>& pSendData,
310  int pRecvProc,
311  Array<OneD, int>& pRecvData)
312  {
313  v_SendRecv(pSendProc, pSendData, pRecvProc, pRecvData);
314  }
315 
316  /**
317  *
318  */
319  inline void Comm::SendRecvReplace(int pSendProc,
320  int pRecvProc,
321  Array<OneD, NekDouble>& pSendData)
322  {
323  v_SendRecvReplace(pSendProc,pRecvProc,pSendData);
324  }
325 
326 
327  /**
328  *
329  */
330  inline void Comm::SendRecvReplace(int pSendProc,
331  int pRecvProc,
332  Array<OneD, int>& pSendData)
333  {
334  v_SendRecvReplace(pSendProc,pRecvProc,pSendData);
335  }
336 
337 
338  /**
339  *
340  */
341  inline void Comm::AllReduce(NekDouble& pData, enum ReduceOperator pOp)
342  {
343  v_AllReduce(pData, pOp);
344  }
345 
346 
347  /**
348  *
349  */
350  inline void Comm::AllReduce(int& pData, enum ReduceOperator pOp)
351  {
352  v_AllReduce(pData, pOp);
353  }
354 
355 
356  /**
357  *
358  */
359  inline void Comm::AllReduce(Array<OneD, NekDouble>& pData, enum ReduceOperator pOp)
360  {
361  v_AllReduce(pData, pOp);
362  }
363 
364 
365  /**
366  *
367  */
368  inline void Comm::AllReduce(Array<OneD, int>& pData, enum ReduceOperator pOp)
369  {
370  v_AllReduce(pData, pOp);
371  }
372 
373 
374  /**
375  *
376  */
377  inline void Comm::AllReduce(std::vector<unsigned int>& pData, enum ReduceOperator pOp)
378  {
379  v_AllReduce(pData, pOp);
380  }
381 
382 
383  /**
384  *
385  */
386  inline void Comm::AlltoAll(Array<OneD, NekDouble>& pSendData,Array<OneD, NekDouble>& pRecvData)
387  {
388  v_AlltoAll(pSendData,pRecvData);
389  }
390 
391 
392  /**
393  *
394  */
395  inline void Comm::AlltoAll(Array<OneD, int>& pSendData,Array<OneD, int>& pRecvData)
396  {
397  v_AlltoAll(pSendData,pRecvData);
398  }
399 
400 
401  /**
402  *
403  */
404  inline void Comm::AlltoAllv(Array<OneD, NekDouble>& pSendData,
405  Array<OneD, int>& pSendDataSizeMap,
406  Array<OneD, int>& pSendDataOffsetMap,
407  Array<OneD, NekDouble>& pRecvData,
408  Array<OneD, int>& pRecvDataSizeMap,
409  Array<OneD, int>& pRecvDataOffsetMap)
410  {
411  v_AlltoAllv(pSendData,pSendDataSizeMap,pSendDataOffsetMap,pRecvData,pRecvDataSizeMap,pRecvDataOffsetMap);
412  }
413 
414  /**
415  *
416  */
417  inline void Comm::AlltoAllv(Array<OneD, int>& pSendData,
418  Array<OneD, int>& pSendDataSizeMap,
419  Array<OneD, int>& pSendDataOffsetMap,
420  Array<OneD, int>& pRecvData,
421  Array<OneD, int>& pRecvDataSizeMap,
422  Array<OneD, int>& pRecvDataOffsetMap)
423  {
424  v_AlltoAllv(pSendData,pSendDataSizeMap,pSendDataOffsetMap,pRecvData,pRecvDataSizeMap,pRecvDataOffsetMap);
425  }
426 
427 
428  /**
429  * @brief Splits this communicator into a grid of size pRows*pColumns
430  * and creates row and column communicators. By default the communicator
431  * is a single row.
432  */
433  inline void Comm::SplitComm(int pRows, int pColumns)
434  {
435  v_SplitComm(pRows, pColumns);
436  }
437 
438 
439  /**
440  * @brief Retrieve the row communicator to which this process belongs.
441  */
443  {
444  if (!m_commRow.get())
445  {
446  return shared_from_this();
447  }
448  else
449  {
450  return m_commRow;
451  }
452  }
453 
454 
455  /**
456  * @brief Retrieve the column communicator to which this process
457  * belongs.
458  */
460  {
461  if (!m_commColumn.get())
462  {
463  return shared_from_this();
464  }
465  else
466  {
467  return m_commColumn;
468  }
469  }
470 
471  inline bool Comm::TreatAsRankZero(void)
472  {
473  return v_TreatAsRankZero();
474  }
475 
476  inline bool Comm::RemoveExistingFiles(void)
477  {
478  return v_RemoveExistingFiles();
479  }
480 
481 
482  }
483 }
484 
485 #endif