Nektar++
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
GsLib.hpp
Go to the documentation of this file.
1 ///////////////////////////////////////////////////////////////////////////////
2 //
3 // File GsLib.hpp
4 //
5 // For more information, please see: http://www.nektar.info
6 //
7 // The MIT License
8 //
9 // Copyright (c) 2006 Division of Applied Mathematics, Brown University (USA),
10 // Department of Aeronautics, Imperial College London (UK), and Scientific
11 // Computing and Imaging Institute, University of Utah (USA).
12 //
13 // License for the specific language governing rights and limitations under
14 // Permission is hereby granted, free of charge, to any person obtaining a
15 // copy of this software and associated documentation files (the "Software"),
16 // to deal in the Software without restriction, including without limitation
17 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
18 // and/or sell copies of the Software, and to permit persons to whom the
19 // Software is furnished to do so, subject to the following conditions:
20 //
21 // The above copyright notice and this permission notice shall be included
22 // in all copies or substantial portions of the Software.
23 //
24 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
25 // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
26 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
27 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
28 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
29 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
30 // DEALINGS IN THE SOFTWARE.
31 //
32 // Description: wrapper of functions around GSLib routines
33 //
34 ///////////////////////////////////////////////////////////////////////////////
35 
36 #ifndef NEKTAR_LIB_UTILITIES_COMMUNICATION_GSLIB_HPP
37 #define NEKTAR_LIB_UTILITIES_COMMUNICATION_GSLIB_HPP
38 
39 #include <iostream>
40 
43 #ifdef NEKTAR_USE_MPI
45 #endif
46 
47 
48 namespace Gs
49 {
50  using namespace std;
51  using namespace Nektar;
53  typedef enum { gs_add, gs_mul, gs_min, gs_max, gs_bpr, gs_op_n } gs_op;
55 
56  typedef struct { void *ptr; size_t n,max; } array;
57  typedef array buffer;
58 #ifdef NEKTAR_USE_MPI
59  typedef MPI_Comm comm_ext;
60  typedef MPI_Request comm_req;
61 #else
62  typedef int comm_ext;
63  typedef int comm_req;
64 #endif
65 
66  struct comm {
67  unsigned int id;
68  unsigned int np;
69  comm_ext c;
70  };
71 
72  typedef struct {
73  unsigned int n; /* number of messages */
74  unsigned int *p; /* message source/dest proc */
75  unsigned int *size; /* size of message */
76  unsigned int total; /* sum of message sizes */
77  } pw_comm_data;
78 
79  typedef struct {
81  const unsigned int *map[2];
82  comm_req *req;
83  unsigned int buffer_size;
84  } pw_data;
85 
86  typedef struct {
87  const unsigned int *scatter_map, *gather_map;
88  unsigned int size_r, size_r1, size_r2;
89  unsigned int size_sk, size_s, size_total;
90  unsigned int p1, p2;
91  unsigned int nrecvn;
92  } cr_stage;
93 
94  typedef struct {
95  cr_stage *stage[2];
96  unsigned int nstages;
97  unsigned int buffer_size, stage_buffer_size;
98  } cr_data;
99 
100  typedef struct {
101  const unsigned int *map_to_buf[2], *map_from_buf[2];
102  unsigned int buffer_size;
103  } allreduce_data;
104 
105  typedef void exec_fun(
106  void *data, gs_mode mode, unsigned vn, gs_dom dom, gs_op op,
107  unsigned transpose, const void *execdata, const struct comm *comm, char *buf);
108  typedef void fin_fun(void *data);
109 
110  typedef struct {
111  unsigned int buffer_size, mem_size;
112  void *data;
115  } gs_remote;
116 
117  typedef struct {
118  struct comm comm;
119  const unsigned int *map_local[2]; /* 0=unflagged, 1=all */
120  const unsigned int *flagged_primaries;
122  unsigned int handle_size;
123  } gs_data;
124 
126 
127  extern "C"
128  {
129  void nektar_gs(void *u, gs_dom dom, gs_op op, unsigned transpose,
130  gs_data *gsh, buffer *buf);
131  gs_data *nektar_gs_setup(const long *id, unsigned int n, const struct comm *comm,
132  int unique, gs_method method, int verbose);
133  void nektar_gs_free(gs_data *gsh);
134  void nektar_gs_unique(const long *id, unsigned int n, const struct comm *comm);
135  }
136 
137 
138  /**
139  * @brief Initialise Gather-Scatter map.
140  *
141  * On each process an array of IDs for each global degree of freedom is
142  * supplied which corresponds to a unique numbering of universal degrees of
143  * freedom. This is used to initialise the GSLib mapping between process-
144  * boundary degrees of freedom on different processes.
145  * @param pId Array of integers providing universal IDs for each
146  * global DOF on the process.
147  * @param pComm Communication object used for inter-process
148  * communication.
149  * @return GSLib data structure containing mapping information.
150  */
151  static inline gs_data* Init ( const Nektar::Array<OneD, long> pId,
152  const LibUtilities::CommSharedPtr& pComm)
153  {
154 #ifdef NEKTAR_USE_MPI
155  if (pComm->GetSize() == 1)
156  {
157  return 0;
158  }
159  LibUtilities::CommMpiSharedPtr vCommMpi = boost::dynamic_pointer_cast<LibUtilities::CommMpi> (pComm);
160  ASSERTL1(vCommMpi, "Failed to cast MPI Comm object.");
161  comm vComm;
162  MPI_Comm_dup(vCommMpi->GetComm(), &vComm.c);
163  vComm.id = vCommMpi->GetRank();
164  vComm.np = vCommMpi->GetSize();
165  gs_data* result = nektar_gs_setup(pId.get(),pId.num_elements(),
166  &vComm, 0, gs_auto, 1);
167  MPI_Comm_free(&vComm.c);
168  return result;
169 #else
170  return 0;
171 #endif
172  }
173 
174 
175  /**
176  * @brief Updates pId to negate all-but-one references to each universal ID.
177  *
178  * The array of universal IDs corresponding to the process-local DOF are
179  * updated such that the ID of only one instance of each universal ID
180  * remains positive. This allows the consistent formulation of universally
181  * -distributed dot products, for which the contributions of each DOF must
182  * be included only once.
183  */
184  static inline void Unique(const Nektar::Array<OneD, long> pId,
185  const LibUtilities::CommSharedPtr& pComm)
186  {
187 #ifdef NEKTAR_USE_MPI
188  if (pComm->GetSize() == 1)
189  {
190  return;
191  }
192  LibUtilities::CommMpiSharedPtr vCommMpi = boost::dynamic_pointer_cast<LibUtilities::CommMpi> (pComm);
193  ASSERTL1(vCommMpi, "Failed to cast MPI Comm object.");
194  comm vComm;
195  vComm.c = vCommMpi->GetComm();
196  vComm.id = vCommMpi->GetRank();
197  vComm.np = vCommMpi->GetSize();
198  nektar_gs_unique(pId.get(), pId.num_elements(), &vComm);
199 #endif
200  }
201 
202 
203  /**
204  * @brief Deallocates the GSLib mapping data.
205  */
206  static inline void Finalise (gs_data *pGsh)
207  {
208 #ifdef NEKTAR_USE_MPI
209  int finalized;
210  MPI_Finalized(&finalized);
211  if (pGsh && !finalized)
212  {
213  nektar_gs_free(pGsh);
214  }
215 #endif
216  }
217 
218 
219  /**
220  * @brief Performs a gather-scatter operation of the provided values.
221  *
222  * The
223  */
224  static inline void Gather(Nektar::Array<OneD, NekDouble> pU, gs_op pOp,
227  {
228 #ifdef NEKTAR_USE_MPI
229  if (!pGsh)
230  {
231  return;
232  }
233  if (pBuffer.num_elements() == 0)
234  {
235  nektar_gs(pU.get(), gs_double, pOp, false, pGsh, 0);
236  }
237  else
238  {
239  array buf;
240  buf.ptr = &pBuffer[0];
241  buf.n = pBuffer.num_elements();
242  nektar_gs(pU.get(), gs_double, pOp, false, pGsh, &buf);
243  }
244 #endif
245  }
246 
247 }
248 
249 #endif
gs_dom
Definition: GsLib.hpp:52
unsigned int nstages
Definition: GsLib.hpp:96
int comm_ext
Definition: GsLib.hpp:62
Definition: GsLib.hpp:48
static void Gather(Nektar::Array< OneD, NekDouble > pU, gs_op pOp, gs_data *pGsh, Nektar::Array< OneD, NekDouble > pBuffer=NullNekDouble1DArray)
Performs a gather-scatter operation of the provided values.
Definition: GsLib.hpp:224
static Array< OneD, NekDouble > NullNekDouble1DArray
unsigned int buffer_size
Definition: GsLib.hpp:83
static void Finalise(gs_data *pGsh)
Deallocates the GSLib mapping data.
Definition: GsLib.hpp:206
unsigned int buffer_size
Definition: GsLib.hpp:102
static gs_data * Init(const Nektar::Array< OneD, long > pId, const LibUtilities::CommSharedPtr &pComm)
Initialise Gather-Scatter map.
Definition: GsLib.hpp:151
void nektar_gs_unique(const long *id, unsigned int n, const struct comm *comm)
array buffer
Definition: GsLib.hpp:57
STL namespace.
unsigned int np
Definition: GsLib.hpp:68
int comm_req
Definition: GsLib.hpp:63
unsigned int size_r2
Definition: GsLib.hpp:88
gs_method
Definition: GsLib.hpp:125
void exec_fun(void *data, gs_mode mode, unsigned vn, gs_dom dom, gs_op op, unsigned transpose, const void *execdata, const struct comm *comm, char *buf)
Definition: GsLib.hpp:105
unsigned int * p
Definition: GsLib.hpp:74
unsigned int nrecvn
Definition: GsLib.hpp:91
unsigned int handle_size
Definition: GsLib.hpp:122
unsigned int size_total
Definition: GsLib.hpp:89
boost::shared_ptr< Comm > CommSharedPtr
Pointer to a Communicator object.
Definition: Comm.h:53
unsigned int id
Definition: GsLib.hpp:67
unsigned int * size
Definition: GsLib.hpp:75
void * ptr
Definition: GsLib.hpp:56
gs_data * nektar_gs_setup(const long *id, unsigned int n, const struct comm *comm, int unique, gs_method method, int verbose)
boost::shared_ptr< CommMpi > CommMpiSharedPtr
Pointer to a Communicator object.
Definition: CommMpi.h:55
unsigned int stage_buffer_size
Definition: GsLib.hpp:97
static void Unique(const Nektar::Array< OneD, long > pId, const LibUtilities::CommSharedPtr &pComm)
Updates pId to negate all-but-one references to each universal ID.
Definition: GsLib.hpp:184
unsigned int n
Definition: GsLib.hpp:73
A global linear system.
Definition: CommMpi.h:61
void nektar_gs_free(gs_data *gsh)
void fin_fun(void *data)
Definition: GsLib.hpp:108
const unsigned int * scatter_map
Definition: GsLib.hpp:87
void * data
Definition: GsLib.hpp:112
const unsigned int * flagged_primaries
Definition: GsLib.hpp:120
comm_ext c
Definition: GsLib.hpp:69
exec_fun * exec
Definition: GsLib.hpp:113
gs_remote r
Definition: GsLib.hpp:121
fin_fun * fin
Definition: GsLib.hpp:114
comm_req * req
Definition: GsLib.hpp:82
unsigned int total
Definition: GsLib.hpp:76
gs_mode
Definition: GsLib.hpp:54
unsigned int p2
Definition: GsLib.hpp:90
void nektar_gs(void *u, gs_dom dom, gs_op op, unsigned transpose, gs_data *gsh, buffer *buf)
gs_op
Definition: GsLib.hpp:53
#define ASSERTL1(condition, msg)
Assert Level 1 – Debugging which is used whether in FULLDEBUG or DEBUG compilation mode...
Definition: ErrorUtil.hpp:191
unsigned int mem_size
Definition: GsLib.hpp:111