Nektar++
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
GsLib.hpp
Go to the documentation of this file.
1 ///////////////////////////////////////////////////////////////////////////////
2 //
3 // File GsLib.hpp
4 //
5 // For more information, please see: http://www.nektar.info
6 //
7 // The MIT License
8 //
9 // Copyright (c) 2006 Division of Applied Mathematics, Brown University (USA),
10 // Department of Aeronautics, Imperial College London (UK), and Scientific
11 // Computing and Imaging Institute, University of Utah (USA).
12 //
13 // License for the specific language governing rights and limitations under
14 // Permission is hereby granted, free of charge, to any person obtaining a
15 // copy of this software and associated documentation files (the "Software"),
16 // to deal in the Software without restriction, including without limitation
17 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
18 // and/or sell copies of the Software, and to permit persons to whom the
19 // Software is furnished to do so, subject to the following conditions:
20 //
21 // The above copyright notice and this permission notice shall be included
22 // in all copies or substantial portions of the Software.
23 //
24 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
25 // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
26 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
27 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
28 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
29 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
30 // DEALINGS IN THE SOFTWARE.
31 //
32 // Description: wrapper of functions around GSLib routines
33 //
34 ///////////////////////////////////////////////////////////////////////////////
35 
36 #ifndef NEKTAR_LIB_UTILITIES_COMMUNICATION_GSLIB_HPP
37 #define NEKTAR_LIB_UTILITIES_COMMUNICATION_GSLIB_HPP
38 
39 #include <iostream>
40 
43 #ifdef NEKTAR_USE_MPI
45 #endif
46 
47 namespace Gs
48 {
49 using namespace std;
50 using namespace Nektar;
52 typedef enum { gs_add, gs_mul, gs_min, gs_max, gs_bpr, gs_op_n } gs_op;
54 
55 typedef struct
56 {
57  void *ptr;
58  size_t n, max;
59 } array;
60 typedef array buffer;
61 #ifdef NEKTAR_USE_MPI
62 typedef MPI_Comm comm_ext;
63 typedef MPI_Request comm_req;
64 #else
65 typedef int comm_ext;
66 typedef int comm_req;
67 #endif
68 
69 struct comm
70 {
71  unsigned int id;
72  unsigned int np;
73  comm_ext c;
74 };
75 
76 typedef struct
77 {
78  unsigned int n; /* number of messages */
79  unsigned int *p; /* message source/dest proc */
80  unsigned int *size; /* size of message */
81  unsigned int total; /* sum of message sizes */
82 } pw_comm_data;
83 
84 typedef struct
85 {
87  const unsigned int *map[2];
88  comm_req *req;
89  unsigned int buffer_size;
90 } pw_data;
91 
92 typedef struct
93 {
94  const unsigned int *scatter_map, *gather_map;
95  unsigned int size_r, size_r1, size_r2;
96  unsigned int size_sk, size_s, size_total;
97  unsigned int p1, p2;
98  unsigned int nrecvn;
99 } cr_stage;
100 
101 typedef struct
102 {
103  cr_stage *stage[2];
104  unsigned int nstages;
105  unsigned int buffer_size, stage_buffer_size;
106 } cr_data;
107 
108 typedef struct
109 {
110  const unsigned int *map_to_buf[2], *map_from_buf[2];
111  unsigned int buffer_size;
113 
114 typedef void exec_fun(void *data, gs_mode mode, unsigned vn, gs_dom dom,
115  gs_op op, unsigned transpose, const void *execdata,
116  const struct comm *comm, char *buf);
117 typedef void fin_fun(void *data);
118 
119 typedef struct
120 {
121  unsigned int buffer_size, mem_size;
122  void *data;
125 } gs_remote;
126 
127 typedef struct
128 {
129  struct comm comm;
130  const unsigned int *map_local[2]; /* 0=unflagged, 1=all */
131  const unsigned int *flagged_primaries;
133  unsigned int handle_size;
134 } gs_data;
135 
136 typedef enum {
141 } gs_method;
142 
143 extern "C" {
144 void nektar_gs(void *u, gs_dom dom, gs_op op, unsigned transpose, gs_data *gsh,
145  buffer *buf);
146 gs_data *nektar_gs_setup(const long *id, unsigned int n,
147  const struct comm *comm, int unique, gs_method method,
148  int verbose);
149 void nektar_gs_free(gs_data *gsh);
150 void nektar_gs_unique(const long *id, unsigned int n, const struct comm *comm);
151 }
152 
153 /**
154  * @brief Initialise Gather-Scatter map.
155  *
156  * On each process an array of IDs for each global degree of freedom is
157  * supplied which corresponds to a unique numbering of universal degrees of
158  * freedom. This is used to initialise the GSLib mapping between process-
159  * boundary degrees of freedom on different processes.
160  * @param pId Array of integers providing universal IDs for each
161  * global DOF on the process.
162  * @param pComm Communication object used for inter-process
163  * communication.
164  * @return GSLib data structure containing mapping information.
165  */
166 static inline gs_data *Init(const Nektar::Array<OneD, long> pId,
167  const LibUtilities::CommSharedPtr &pComm,
168  bool verbose = true)
169 {
170 #ifdef NEKTAR_USE_MPI
171  if (pComm->GetSize() == 1)
172  {
173  return 0;
174  }
176  boost::dynamic_pointer_cast<LibUtilities::CommMpi>(pComm);
177  ASSERTL1(vCommMpi, "Failed to cast MPI Comm object.");
178  comm vComm;
179  MPI_Comm_dup(vCommMpi->GetComm(), &vComm.c);
180  vComm.id = vCommMpi->GetRank();
181  vComm.np = vCommMpi->GetSize();
182  gs_data *result = nektar_gs_setup(pId.get(), pId.num_elements(), &vComm, 0,
183  gs_auto, (int)verbose);
184  MPI_Comm_free(&vComm.c);
185  return result;
186 #else
187  return 0;
188 #endif
189 }
190 
191 /**
192  * @brief Updates pId to negate all-but-one references to each universal ID.
193  *
194  * The array of universal IDs corresponding to the process-local DOF are
195  * updated such that the ID of only one instance of each universal ID
196  * remains positive. This allows the consistent formulation of universally
197  * -distributed dot products, for which the contributions of each DOF must
198  * be included only once.
199  */
200 static inline void Unique(const Nektar::Array<OneD, long> pId,
201  const LibUtilities::CommSharedPtr &pComm)
202 {
203 #ifdef NEKTAR_USE_MPI
204  if (pComm->GetSize() == 1)
205  {
206  return;
207  }
209  boost::dynamic_pointer_cast<LibUtilities::CommMpi>(pComm);
210  ASSERTL1(vCommMpi, "Failed to cast MPI Comm object.");
211  comm vComm;
212  vComm.c = vCommMpi->GetComm();
213  vComm.id = vCommMpi->GetRank();
214  vComm.np = vCommMpi->GetSize();
215  nektar_gs_unique(pId.get(), pId.num_elements(), &vComm);
216 #endif
217 }
218 
219 /**
220  * @brief Deallocates the GSLib mapping data.
221  */
222 static inline void Finalise(gs_data *pGsh)
223 {
224 #ifdef NEKTAR_USE_MPI
225  int finalized;
226  MPI_Finalized(&finalized);
227  if (pGsh && !finalized)
228  {
229  nektar_gs_free(pGsh);
230  }
231 #endif
232 }
233 
234 /**
235  * @brief Performs a gather-scatter operation of the provided values.
236  *
237  * The
238  */
239 static inline void Gather(
242 {
243 #ifdef NEKTAR_USE_MPI
244  if (!pGsh)
245  {
246  return;
247  }
248  if (pBuffer.num_elements() == 0)
249  {
250  nektar_gs(pU.get(), gs_double, pOp, false, pGsh, 0);
251  }
252  else
253  {
254  array buf;
255  buf.ptr = &pBuffer[0];
256  buf.n = pBuffer.num_elements();
257  nektar_gs(pU.get(), gs_double, pOp, false, pGsh, &buf);
258  }
259 #endif
260 }
261 }
262 
263 #endif
gs_dom
Definition: GsLib.hpp:51
unsigned int nstages
Definition: GsLib.hpp:104
int comm_ext
Definition: GsLib.hpp:65
Definition: GsLib.hpp:47
static void Gather(Nektar::Array< OneD, NekDouble > pU, gs_op pOp, gs_data *pGsh, Nektar::Array< OneD, NekDouble > pBuffer=NullNekDouble1DArray)
Performs a gather-scatter operation of the provided values.
Definition: GsLib.hpp:239
static Array< OneD, NekDouble > NullNekDouble1DArray
unsigned int buffer_size
Definition: GsLib.hpp:89
static void Finalise(gs_data *pGsh)
Deallocates the GSLib mapping data.
Definition: GsLib.hpp:222
unsigned int buffer_size
Definition: GsLib.hpp:111
void nektar_gs_unique(const long *id, unsigned int n, const struct comm *comm)
array buffer
Definition: GsLib.hpp:60
STL namespace.
unsigned int np
Definition: GsLib.hpp:72
int comm_req
Definition: GsLib.hpp:66
unsigned int size_r2
Definition: GsLib.hpp:95
gs_method
Definition: GsLib.hpp:136
void exec_fun(void *data, gs_mode mode, unsigned vn, gs_dom dom, gs_op op, unsigned transpose, const void *execdata, const struct comm *comm, char *buf)
Definition: GsLib.hpp:114
unsigned int * p
Definition: GsLib.hpp:79
static gs_data * Init(const Nektar::Array< OneD, long > pId, const LibUtilities::CommSharedPtr &pComm, bool verbose=true)
Initialise Gather-Scatter map.
Definition: GsLib.hpp:166
unsigned int nrecvn
Definition: GsLib.hpp:98
unsigned int handle_size
Definition: GsLib.hpp:133
unsigned int size_total
Definition: GsLib.hpp:96
boost::shared_ptr< Comm > CommSharedPtr
Pointer to a Communicator object.
Definition: Comm.h:55
unsigned int id
Definition: GsLib.hpp:71
unsigned int * size
Definition: GsLib.hpp:80
void * ptr
Definition: GsLib.hpp:57
gs_data * nektar_gs_setup(const long *id, unsigned int n, const struct comm *comm, int unique, gs_method method, int verbose)
boost::shared_ptr< CommMpi > CommMpiSharedPtr
Pointer to a Communicator object.
Definition: CommMpi.h:55
unsigned int stage_buffer_size
Definition: GsLib.hpp:105
static void Unique(const Nektar::Array< OneD, long > pId, const LibUtilities::CommSharedPtr &pComm)
Updates pId to negate all-but-one references to each universal ID.
Definition: GsLib.hpp:200
unsigned int n
Definition: GsLib.hpp:78
size_t n
Definition: GsLib.hpp:58
A global linear system.
Definition: CommMpi.h:61
void nektar_gs_free(gs_data *gsh)
void fin_fun(void *data)
Definition: GsLib.hpp:117
const unsigned int * scatter_map
Definition: GsLib.hpp:94
void * data
Definition: GsLib.hpp:122
const unsigned int * flagged_primaries
Definition: GsLib.hpp:131
comm_ext c
Definition: GsLib.hpp:73
exec_fun * exec
Definition: GsLib.hpp:123
gs_remote r
Definition: GsLib.hpp:132
fin_fun * fin
Definition: GsLib.hpp:124
comm_req * req
Definition: GsLib.hpp:88
unsigned int total
Definition: GsLib.hpp:81
gs_mode
Definition: GsLib.hpp:53
unsigned int p2
Definition: GsLib.hpp:97
void nektar_gs(void *u, gs_dom dom, gs_op op, unsigned transpose, gs_data *gsh, buffer *buf)
gs_op
Definition: GsLib.hpp:52
#define ASSERTL1(condition, msg)
Assert Level 1 – Debugging which is used whether in FULLDEBUG or DEBUG compilation mode...
Definition: ErrorUtil.hpp:228
unsigned int mem_size
Definition: GsLib.hpp:121