Nektar++
GsLib.hpp
Go to the documentation of this file.
1 ///////////////////////////////////////////////////////////////////////////////
2 //
3 // File GsLib.hpp
4 //
5 // For more information, please see: http://www.nektar.info
6 //
7 // The MIT License
8 //
9 // Copyright (c) 2006 Division of Applied Mathematics, Brown University (USA),
10 // Department of Aeronautics, Imperial College London (UK), and Scientific
11 // Computing and Imaging Institute, University of Utah (USA).
12 //
13 // License for the specific language governing rights and limitations under
14 // Permission is hereby granted, free of charge, to any person obtaining a
15 // copy of this software and associated documentation files (the "Software"),
16 // to deal in the Software without restriction, including without limitation
17 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
18 // and/or sell copies of the Software, and to permit persons to whom the
19 // Software is furnished to do so, subject to the following conditions:
20 //
21 // The above copyright notice and this permission notice shall be included
22 // in all copies or substantial portions of the Software.
23 //
24 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
25 // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
26 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
27 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
28 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
29 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
30 // DEALINGS IN THE SOFTWARE.
31 //
32 // Description: wrapper of functions around GSLib routines
33 //
34 ///////////////////////////////////////////////////////////////////////////////
35 
36 #ifndef NEKTAR_LIB_UTILITIES_COMMUNICATION_GSLIB_HPP
37 #define NEKTAR_LIB_UTILITIES_COMMUNICATION_GSLIB_HPP
38 
39 #include <iostream>
40 using namespace std;
41 
44 #ifdef NEKTAR_USE_MPI
46 #endif
47 using namespace Nektar;
48 
49 namespace Gs
50 {
52  typedef enum { gs_add, gs_mul, gs_min, gs_max, gs_bpr, gs_op_n } gs_op;
54 
55  typedef struct { void *ptr; size_t n,max; } array;
56  typedef array buffer;
57 #ifdef NEKTAR_USE_MPI
58  typedef MPI_Comm comm_ext;
59  typedef MPI_Request comm_req;
60 #else
61  typedef int comm_ext;
62  typedef int comm_req;
63 #endif
64 
65  struct comm {
66  unsigned int id;
67  unsigned int np;
68  comm_ext c;
69  };
70 
71  typedef struct {
72  unsigned int n; /* number of messages */
73  unsigned int *p; /* message source/dest proc */
74  unsigned int *size; /* size of message */
75  unsigned int total; /* sum of message sizes */
76  } pw_comm_data;
77 
78  typedef struct {
80  const unsigned int *map[2];
81  comm_req *req;
82  unsigned int buffer_size;
83  } pw_data;
84 
85  typedef struct {
86  const unsigned int *scatter_map, *gather_map;
87  unsigned int size_r, size_r1, size_r2;
88  unsigned int size_sk, size_s, size_total;
89  unsigned int p1, p2;
90  unsigned int nrecvn;
91  } cr_stage;
92 
93  typedef struct {
94  cr_stage *stage[2];
95  unsigned int nstages;
96  unsigned int buffer_size, stage_buffer_size;
97  } cr_data;
98 
99  typedef struct {
100  const unsigned int *map_to_buf[2], *map_from_buf[2];
101  unsigned int buffer_size;
102  } allreduce_data;
103 
104  typedef void exec_fun(
105  void *data, gs_mode mode, unsigned vn, gs_dom dom, gs_op op,
106  unsigned transpose, const void *execdata, const struct comm *comm, char *buf);
107  typedef void fin_fun(void *data);
108 
109  typedef struct {
110  unsigned int buffer_size, mem_size;
111  void *data;
114  } gs_remote;
115 
116  typedef struct {
117  struct comm comm;
118  const unsigned int *map_local[2]; /* 0=unflagged, 1=all */
119  const unsigned int *flagged_primaries;
121  unsigned int handle_size;
122  } gs_data;
123 
125 
126  extern "C"
127  {
128  void nektar_gs(void *u, gs_dom dom, gs_op op, unsigned transpose,
129  gs_data *gsh, buffer *buf);
130  gs_data *nektar_gs_setup(const long *id, unsigned int n, const struct comm *comm,
131  int unique, gs_method method, int verbose);
132  void nektar_gs_free(gs_data *gsh);
133  void nektar_gs_unique(const long *id, unsigned int n, const struct comm *comm);
134  }
135 
136 
137  /**
138  * @brief Initialise Gather-Scatter map.
139  *
140  * On each process an array of IDs for each global degree of freedom is
141  * supplied which corresponds to a unique numbering of universal degrees of
142  * freedom. This is used to initialise the GSLib mapping between process-
143  * boundary degrees of freedom on different processes.
144  * @param pId Array of integers providing universal IDs for each
145  * global DOF on the process.
146  * @param pComm Communication object used for inter-process
147  * communication.
148  * @return GSLib data structure containing mapping information.
149  */
150  static inline gs_data* Init ( const Nektar::Array<OneD, long> pId,
151  const LibUtilities::CommSharedPtr& pComm)
152  {
153 #ifdef NEKTAR_USE_MPI
154  if (pComm->GetSize() == 1)
155  {
156  return 0;
157  }
158  LibUtilities::CommMpiSharedPtr vCommMpi = boost::dynamic_pointer_cast<LibUtilities::CommMpi> (pComm);
159  ASSERTL1(vCommMpi, "Failed to cast MPI Comm object.");
160  comm vComm;
161  MPI_Comm_dup(vCommMpi->GetComm(), &vComm.c);
162  vComm.id = vCommMpi->GetRank();
163  vComm.np = vCommMpi->GetSize();
164  return nektar_gs_setup(pId.get(),pId.num_elements(), &vComm, 0, gs_auto, 1);
165 #else
166  return 0;
167 #endif
168  }
169 
170 
171  /**
172  * @brief Updates pId to negate all-but-one references to each universal ID.
173  *
174  * The array of universal IDs corresponding to the process-local DOF are
175  * updated such that the ID of only one instance of each universal ID
176  * remains positive. This allows the consistent formulation of universally
177  * -distributed dot products, for which the contributions of each DOF must
178  * be included only once.
179  */
180  static inline void Unique(const Nektar::Array<OneD, long> pId,
181  const LibUtilities::CommSharedPtr& pComm)
182  {
183 #ifdef NEKTAR_USE_MPI
184  if (pComm->GetSize() == 1)
185  {
186  return;
187  }
188  LibUtilities::CommMpiSharedPtr vCommMpi = boost::dynamic_pointer_cast<LibUtilities::CommMpi> (pComm);
189  ASSERTL1(vCommMpi, "Failed to cast MPI Comm object.");
190  comm vComm;
191  vComm.c = vCommMpi->GetComm();
192  vComm.id = vCommMpi->GetRank();
193  vComm.np = vCommMpi->GetSize();
194  nektar_gs_unique(pId.get(), pId.num_elements(), &vComm);
195 #endif
196  }
197 
198 
199  /**
200  * @brief Deallocates the GSLib mapping data.
201  */
202  static inline void Finalise (gs_data *pGsh)
203  {
204 #ifdef NEKTAR_USE_MPI
205  if (pGsh)
206  {
207  nektar_gs_free(pGsh);
208  }
209 #endif
210  }
211 
212 
213  /**
214  * @brief Performs a gather-scatter operation of the provided values.
215  *
216  * The
217  */
218  static inline void Gather(Nektar::Array<OneD, NekDouble> pU, gs_op pOp,
221  {
222 #ifdef NEKTAR_USE_MPI
223  if (!pGsh)
224  {
225  return;
226  }
227  if (pBuffer.num_elements() == 0)
228  {
229  nektar_gs(pU.get(), gs_double, pOp, false, pGsh, 0);
230  }
231  else
232  {
233  array buf;
234  buf.ptr = &pBuffer[0];
235  buf.n = pBuffer.num_elements();
236  nektar_gs(pU.get(), gs_double, pOp, false, pGsh, &buf);
237  }
238 #endif
239  }
240 
241 }
242 
243 #endif
gs_dom
Definition: GsLib.hpp:51
unsigned int nstages
Definition: GsLib.hpp:95
int comm_ext
Definition: GsLib.hpp:61
Definition: GsLib.hpp:49
static void Gather(Nektar::Array< OneD, NekDouble > pU, gs_op pOp, gs_data *pGsh, Nektar::Array< OneD, NekDouble > pBuffer=NullNekDouble1DArray)
Performs a gather-scatter operation of the provided values.
Definition: GsLib.hpp:218
static Array< OneD, NekDouble > NullNekDouble1DArray
unsigned int buffer_size
Definition: GsLib.hpp:82
static void Finalise(gs_data *pGsh)
Deallocates the GSLib mapping data.
Definition: GsLib.hpp:202
unsigned int buffer_size
Definition: GsLib.hpp:101
static gs_data * Init(const Nektar::Array< OneD, long > pId, const LibUtilities::CommSharedPtr &pComm)
Initialise Gather-Scatter map.
Definition: GsLib.hpp:150
void nektar_gs_unique(const long *id, unsigned int n, const struct comm *comm)
array buffer
Definition: GsLib.hpp:56
STL namespace.
unsigned int np
Definition: GsLib.hpp:67
int comm_req
Definition: GsLib.hpp:62
unsigned int size_r2
Definition: GsLib.hpp:87
gs_method
Definition: GsLib.hpp:124
void exec_fun(void *data, gs_mode mode, unsigned vn, gs_dom dom, gs_op op, unsigned transpose, const void *execdata, const struct comm *comm, char *buf)
Definition: GsLib.hpp:104
unsigned int * p
Definition: GsLib.hpp:73
unsigned int nrecvn
Definition: GsLib.hpp:90
unsigned int handle_size
Definition: GsLib.hpp:121
unsigned int size_total
Definition: GsLib.hpp:88
boost::shared_ptr< Comm > CommSharedPtr
Pointer to a Communicator object.
Definition: Comm.h:53
unsigned int id
Definition: GsLib.hpp:66
unsigned int * size
Definition: GsLib.hpp:74
void * ptr
Definition: GsLib.hpp:55
gs_data * nektar_gs_setup(const long *id, unsigned int n, const struct comm *comm, int unique, gs_method method, int verbose)
boost::shared_ptr< CommMpi > CommMpiSharedPtr
Pointer to a Communicator object.
Definition: CommMpi.h:55
unsigned int stage_buffer_size
Definition: GsLib.hpp:96
static void Unique(const Nektar::Array< OneD, long > pId, const LibUtilities::CommSharedPtr &pComm)
Updates pId to negate all-but-one references to each universal ID.
Definition: GsLib.hpp:180
unsigned int n
Definition: GsLib.hpp:72
size_t n
Definition: GsLib.hpp:55
A global linear system.
Definition: CommMpi.h:61
void nektar_gs_free(gs_data *gsh)
void fin_fun(void *data)
Definition: GsLib.hpp:107
const unsigned int * scatter_map
Definition: GsLib.hpp:86
void * data
Definition: GsLib.hpp:111
const unsigned int * flagged_primaries
Definition: GsLib.hpp:119
comm_ext c
Definition: GsLib.hpp:68
exec_fun * exec
Definition: GsLib.hpp:112
gs_remote r
Definition: GsLib.hpp:120
fin_fun * fin
Definition: GsLib.hpp:113
comm_req * req
Definition: GsLib.hpp:81
unsigned int total
Definition: GsLib.hpp:75
gs_mode
Definition: GsLib.hpp:53
unsigned int p2
Definition: GsLib.hpp:89
void nektar_gs(void *u, gs_dom dom, gs_op op, unsigned transpose, gs_data *gsh, buffer *buf)
gs_op
Definition: GsLib.hpp:52
#define ASSERTL1(condition, msg)
Assert Level 1 – Debugging which is used whether in FULLDEBUG or DEBUG compilation mode...
Definition: ErrorUtil.hpp:191
unsigned int mem_size
Definition: GsLib.hpp:110