57#include <boost/algorithm/string.hpp>
58#include <boost/format.hpp>
59#include <boost/graph/adjacency_iterator.hpp>
60#include <boost/graph/adjacency_list.hpp>
61#include <boost/graph/detail/edge.hpp>
74 std::map<int, MeshEntity> element,
76 : m_session(session), m_comm(comm), m_dim(meshDim), m_numFields(0),
77 m_elements(element), m_compMap(compMap), m_fieldNameToId(),
78 m_weightingRequired(false), m_weightBnd(false), m_weightDofs(false),
87 if (elIt->second.ghost)
100 [[maybe_unused]]
int nLocal)
103 "Too few elements for this many processes.");
107 "Parallel partitioning requires shared filesystem.");
121 TiXmlElement *expansionTypes =
m_session->GetElement(
"Nektar/Expansions");
127 TiXmlElement *expansion = expansionTypes->FirstChildElement();
129 std::string expType = expansion->Value();
139 std::vector<unsigned int> composite;
140 std::vector<unsigned int> nummodes;
141 std::vector<std::string> fieldName;
143 const char *nModesStr = expansion->Attribute(
"NUMMODES");
145 "NUMMODES was not defined in EXPANSION section of input");
146 std::string numModesStr = nModesStr;
149 ASSERTL0(valid,
"Unable to correctly parse the number of modes.");
151 if (nummodes.size() == 1)
153 for (
int i = 1; i <
m_dim; i++)
155 nummodes.push_back(nummodes[0]);
159 "Number of modes should match mesh dimension");
161 const char *fStr = expansion->Attribute(
"FIELDS");
164 std::string fieldStr = fStr;
167 ASSERTL0(valid,
"Unable to correctly parse the field string in "
170 for (
int i = 0; i < fieldName.size(); ++i)
182 fieldName.push_back(
"DefaultVar");
189 "Omitting field variables and explicitly listing "
190 "them in different ExpansionTypes is wrong practise");
197 std::string compositeStr = expansion->Attribute(
"COMPOSITE");
199 "COMPOSITE must be specified in expansion definition");
200 int beg = compositeStr.find_first_of(
"[");
201 int end = compositeStr.find_first_of(
"]");
202 std::string compositeListStr =
203 compositeStr.substr(beg + 1, end - beg - 1);
205 compositeListStr.c_str(), composite);
206 ASSERTL0(parseGood && !composite.empty(),
207 (std::string(
"Unable to read composite index range: ") +
212 for (
int i = 0; i < composite.size(); ++i)
214 auto &shapeType =
m_compMap[composite[i]].first;
215 auto &elmtIds =
m_compMap[composite[i]].second;
217 for (
int j = 0; j < fieldName.size(); j++)
219 for (
auto &elid : elmtIds)
227 expansion = expansion->NextSiblingElement(
"E");
230 else if (expType ==
"F")
232 ASSERTL0(expansion->Attribute(
"FILE"),
233 "Attribute FILE expected for type F expansion");
234 std::string filenameStr = expansion->Attribute(
"FILE");
236 "A filename must be specified for the FILE "
237 "attribute of expansion");
248 iofmt, comm,
m_session->GetSharedFilesystem());
251 std::vector<LibUtilities::FieldDefinitionsSharedPtr> fielddefs;
252 f->Import(filenameStr, fielddefs);
255 for (
int i = 0; i < fielddefs.size(); ++i)
258 for (
int j = 0; j < fielddefs[i]->m_fields.size(); ++j)
260 std::string fieldName = fielddefs[i]->m_fields[j];
269 int numHomoDir = fielddefs[i]->m_numHomogeneousDir;
271 for (
int j = 0; j < fielddefs[i]->m_elementIDs.size(); ++j)
273 int elid = fielddefs[i]->m_elementIDs[j];
274 std::vector<unsigned int> nummodes;
275 for (
int k = 0; k <
m_dim; k++)
277 nummodes.push_back(fielddefs[i]->m_numModes[cnt++]);
279 if (fielddefs[i]->m_uniOrder)
287 for (
int k = 0; k < fielddefs[i]->m_fields.size(); k++)
289 std::string fieldName = fielddefs[i]->m_fields[k];
292 m_shape[elid] = fielddefs[i]->m_shapeType;
299 "Expansion type not defined or not supported at the moment");
305 int nElmt = boost::num_vertices(
m_graph);
308 out <<
"# Partition information:" << std::endl;
309 out <<
"# No. elements : " << nElmt << std::endl;
310 out <<
"# No. partitions: " << nPart << std::endl;
311 out <<
"# ID nElmt nLocDof nBndDof" << std::endl;
314 std::vector<int> partElmtCount(nPart, 0);
315 std::vector<int> partLocCount(nPart, 0);
316 std::vector<int> partBndCount(nPart, 0);
318 std::map<int, int> elmtSizes;
319 std::map<int, int> elmtBndSizes;
321 for (std::map<int, NummodesPerField>::iterator expIt =
m_expansions.begin();
324 int elid = expIt->first;
327 for (NummodesPerField::iterator it = npf.begin(); it != npf.end(); ++it)
330 " Number of directional"
331 " modes in expansion spec for element id = " +
332 std::to_string(elid) +
" and field " + it->first +
333 " does not correspond to mesh dimension");
335 int na = it->second[0];
354 for (boost::tie(vertit, vertit_end) = boost::vertices(
m_graph);
355 vertit != vertit_end; ++vertit)
357 int partId =
m_graph[*vertit].partition;
358 partElmtCount[partId]++;
359 partLocCount[partId] += elmtSizes[
m_graph[*vertit].id];
360 partBndCount[partId] += elmtBndSizes[
m_graph[*vertit].id];
363 for (
int i = 0; i < nPart; ++i)
365 out << i <<
" " << partElmtCount[i] <<
" " << partLocCount[i] <<
" "
366 << partBndCount[i] << std::endl;
372 if (!
m_session->DefinesElement(
"Nektar/Conditions/SolverInfo"))
379 TiXmlElement *solverInfoElement =
380 m_session->GetElement(
"Nektar/Conditions/SolverInfo");
385 TiXmlElement *solverInfo = solverInfoElement->FirstChildElement(
"I");
386 ASSERTL0(solverInfo,
"Cannot read SolverInfo tags");
391 ASSERTL0(solverInfo->Attribute(
"PROPERTY"),
392 "Missing PROPERTY attribute in solver info "
394 std::string solverProperty = solverInfo->Attribute(
"PROPERTY");
396 "Solver info properties must have a non-empty "
399 std::string solverPropertyUpper = boost::to_upper_copy(solverProperty);
402 ASSERTL0(solverInfo->Attribute(
"VALUE"),
403 "Missing VALUE attribute in solver info section. ");
404 std::string solverValue = solverInfo->Attribute(
"VALUE");
406 "Solver info properties must have a non-empty value");
408 std::string propertyValueUpper = boost::to_upper_copy(solverValue);
410 if (solverPropertyUpper ==
"WEIGHTPARTITIONS")
412 if (propertyValueUpper ==
"DOF")
417 else if (propertyValueUpper ==
"BOUNDARY")
422 else if (propertyValueUpper ==
"BOTH")
430 solverInfo = solverInfo->NextSiblingElement(
"I");
451 std::map<int, MeshEntity>::iterator eIt;
459 for (std::map<int, NummodesPerField>::iterator expIt =
m_expansions.begin();
462 int elid = expIt->first;
465 for (NummodesPerField::iterator it = npf.begin(); it != npf.end(); ++it)
468 " Number of directional"
469 " modes in expansion spec for element id = " +
470 std::to_string(elid) +
" and field " + it->first +
471 " does not correspond to mesh dimension");
473 int na = it->second[0];
506 std::unordered_map<int, std::vector<int>> vGraphEdges;
511 auto vert = boost::add_vertex(
m_graph);
523 for (
auto &eId : elmt.second.list)
527 auto edgeIt = vGraphEdges.find(eId);
528 if (edgeIt != vGraphEdges.end())
530 for (
auto &iId : edgeIt->second)
535 vGraphEdges[eId].push_back(vcnt);
541 vGraphEdges[eId] = Id;
552 auto vert = boost::add_vertex(
m_graph);
553 m_graph[vert].id = ghost.first;
556 for (
auto &facet : ghost.second.list)
558 auto edgeIt = vGraphEdges.find(facet);
559 if (edgeIt != vGraphEdges.end())
561 for (
auto &iId : edgeIt->second)
566 vGraphEdges[facet].push_back(vcnt);
593 int nGraphVerts = boost::num_vertices(
m_graph);
595 int nLocal = nGraphVerts - nGhost;
612 int nWeight = ncon * nLocal;
615 std::vector<int> adjncy_tmp, adjwgt_tmp;
622 for (boost::tie(vertit, vertit_end) = boost::vertices(
m_graph);
623 vertit != vertit_end && vcnt < nLocal; ++vertit)
625 for (boost::tie(adjvertit, adjvertit_end) =
626 boost::adjacent_vertices(*vertit,
m_graph);
627 adjvertit != adjvertit_end; ++adjvertit, ++acnt)
629 adjncy_tmp.push_back(
m_graph[*adjvertit].
id);
632 adjwgt_tmp.push_back(
m_graph[*vertit].edgeWeight[0]);
636 adjwgt_tmp.push_back(1);
647 vwgt[ncon * (vcnt - 1) + ccnt] =
m_graph[*vertit].weight[0];
652 vwgt[ncon * (vcnt - 1) + ccnt] =
671 if (
m_comm->GetColumnComm()->GetRank() == 0)
675 adjwgt, nParts, vol, part);
687 for (i = 1; i <
m_comm->GetColumnComm()->GetSize(); ++i)
689 m_comm->GetColumnComm()->Send(i, part);
695 m_comm->GetColumnComm()->Recv(0, part);
705 m_comm->GetRowComm()->Recv(0, part);
714 for (boost::tie(vertit, vertit_end) = boost::vertices(
m_graph);
715 vertit != vertit_end; ++vertit, ++i)
723 int nproc =
m_comm->GetSpaceComm()->GetSize(),
724 rank =
m_comm->GetSpaceComm()->GetRank();
725 std::vector<int> numToSend(nproc, 0), numToRecv(nproc);
726 std::map<int, std::vector<int>> procMap;
728 for (boost::tie(vertit, vertit_end) = boost::vertices(
m_graph);
729 vertit != vertit_end && i < nLocal; ++vertit, ++i)
731 int toProc = part[i];
733 procMap[toProc].push_back(
m_graph[*vertit].
id);
736 m_comm->GetSpaceComm()->AlltoAll(numToSend, numToRecv);
739 std::vector<int> sendOffsetMap(nproc), recvOffsetMap(nproc);
741 sendOffsetMap[0] = 0;
742 recvOffsetMap[0] = 0;
743 for (
int i = 1; i < nproc; ++i)
745 sendOffsetMap[i] = sendOffsetMap[i - 1] + numToSend[i - 1];
746 recvOffsetMap[i] = recvOffsetMap[i - 1] + numToRecv[i - 1];
750 int totalSend =
Vmath::Vsum(nproc, &numToSend[0], 1);
751 int totalRecv =
Vmath::Vsum(nproc, &numToRecv[0], 1);
753 std::vector<int> sendData(totalSend), recvData(totalRecv);
756 for (
auto &verts : procMap)
758 for (
auto &vert : verts.second)
760 sendData[cnt++] = vert;
765 m_comm->GetSpaceComm()->AlltoAllv(sendData, numToSend, sendOffsetMap,
766 recvData, numToRecv, recvOffsetMap);
770 std::unordered_set<int> uniqueIDs;
771 for (
auto &
id : recvData)
773 uniqueIDs.insert(
id);
777 uniqueIDs.begin(), uniqueIDs.end());
785 "parallel execution");
787 for (boost::tie(vertit, vertit_end) = boost::vertices(
m_graph);
788 vertit != vertit_end; ++vertit)
790 for (boost::tie(adjvertit, adjvertit_end) =
791 boost::adjacent_vertices(*vertit,
m_graph);
792 adjvertit != adjvertit_end; ++adjvertit)
794 if (part[*adjvertit] != part[*vertit])
807 unsigned int cnt = 0;
811 for (i = 0; i < nParts; ++i)
813 cnt = std::count(pPart.begin(), pPart.end(), i);
827 for (i = 0; i < pPart.size(); ++i)
829 pPart[i] = i % nParts;
835 std::vector<unsigned int> &elmtid)
847 bool bndWeight,
int na,
int nb,
924 int n = std::max(na, std::max(nb, nc));
#define ASSERTL0(condition, msg)
#define NEKERROR(type, msg)
Assert Level 0 – Fundamental assert which is used whether in FULLDEBUG, DEBUG or OPT compilation mode...
#define SPATIAL_DOMAINS_EXPORT
static const std::string GetFileType(const std::string &filename, CommSharedPtr comm)
Determine file type of given input file.
Provides a generic Factory class.
tBaseSharedPtr CreateInstance(tKey idKey, tParam... args)
Create an instance of the class referred to by idKey.
static void GetXMLElementTimeLevel(TiXmlElement *&element, const size_t timeLevel, const bool enableCheck=true)
Get XML elment time level (Parallel-in-Time)
static bool GenerateVector(const std::string &str, std::vector< T > &out)
Takes a comma-separated string and converts it to entries in a vector.
static bool GenerateSeqVector(const std::string &str, std::vector< unsigned int > &out)
Takes a comma-separated compressed string and converts it to entries in a vector.
boost::graph_traits< BoostGraph >::vertex_iterator BoostVertexIterator
virtual void v_PartitionGraphImpl(int &nVerts, int &nVertConds, Nektar::Array< Nektar::OneD, int > &xadj, Nektar::Array< Nektar::OneD, int > &adjcy, Nektar::Array< Nektar::OneD, int > &vertWgt, Nektar::Array< Nektar::OneD, int > &vertSize, Nektar::Array< Nektar::OneD, int > &edgeWgt, int &nparts, int &volume, Nektar::Array< Nektar::OneD, int > &part)=0
void PartitionGraph(int nParts, bool overlapping=false)
Partition the graph.
std::map< int, NummodesPerField > m_expansions
boost::graph_traits< BoostGraph >::adjacency_iterator BoostAdjacencyIterator
std::map< int, MultiWeight > m_edgeWeights
CompositeDescriptor m_compMap
std::map< std::string, int > m_fieldNameToId
std::map< int, MeshEntity > m_ghostElmts
LibUtilities::SessionReaderSharedPtr m_session
std::map< int, MultiWeight > m_vertBndWeights
std::map< int, MultiWeight > m_vertWeights
void PrintPartInfo(std::ostream &out)
void GetElementIDs(const int procid, std::vector< unsigned int > &tmp)
boost::graph_traits< BoostGraph >::edge_descriptor BoostEdge
std::map< int, std::vector< unsigned int > > m_localPartition
int CalculateElementWeight(LibUtilities::ShapeType elmtType, bool bndWeight, int na, int nb, int nc)
std::map< std::string, NumModes > NummodesPerField
int CalculateEdgeWeight(LibUtilities::ShapeType elmtType, int na, int nb, int nc)
void PartitionMesh(int nParts, bool shared=false, bool overlapping=false, int nLocal=0)
std::map< int, MeshEntity > m_elements
MeshPartition(const LibUtilities::SessionReaderSharedPtr session, LibUtilities::CommSharedPtr comm, int meshDim, std::map< int, MeshEntity > element, CompositeDescriptor compMap)
LibUtilities::CommSharedPtr m_comm
void CheckPartitions(int nParts, Array< OneD, int > &pPart)
std::map< int, LibUtilities::ShapeType > m_shape
int getNumberOfCoefficients(int Na, int Nb, int Nc)
int getNumberOfBndCoefficients(int Na, int Nb, int Nc)
int getNumberOfCoefficients(int Na, int Nb, int Nc)
int getNumberOfBndCoefficients(int Na, int Nb, int Nc)
int getNumberOfCoefficients(int Na, int Nb, int Nc)
int getNumberOfBndCoefficients(int Na, int Nb, int Nc)
int getNumberOfCoefficients(int Na, int Nb)
int getNumberOfBndCoefficients(int Na, int Nb)
int getNumberOfCoefficients(int Na)
int getNumberOfBndCoefficients(int Na)
int getNumberOfBndCoefficients(int Na, int Nb, int Nc)
int getNumberOfCoefficients(int Na, int Nb, int Nc)
int getNumberOfCoefficients(int Na, int Nb)
int getNumberOfBndCoefficients(int Na, int Nb)
std::shared_ptr< FieldIO > FieldIOSharedPtr
std::shared_ptr< SessionReader > SessionReaderSharedPtr
FieldIOFactory & GetFieldIOFactory()
Returns the FieldIO factory.
CommFactory & GetCommFactory()
std::shared_ptr< Comm > CommSharedPtr
Pointer to a Communicator object.
std::map< int, std::pair< LibUtilities::ShapeType, std::vector< int > > > CompositeDescriptor
MeshPartitionFactory & GetMeshPartitionFactory()
T Vsum(int n, const T *x, const int incx)
Subtract return sum(x)