#include <InterfaceMapDG.h>

Public Member Functions
virtual	~InterfaceExchange ()=default
	Default destructor. More...

	InterfaceExchange (SpatialDomains::MovementSharedPtr movement, const ExpListSharedPtr &trace, const LibUtilities::CommSharedPtr &comm, std::pair< int, std::vector< InterfaceTraceSharedPtr > > rankPair)
	Constructor. More...

void	RankFillSizes (LibUtilities::CommRequestSharedPtr &requestSend, LibUtilities::CommRequestSharedPtr &requestRecv, int requestNum)

void	SendMissing (LibUtilities::CommRequestSharedPtr &requestSend, LibUtilities::CommRequestSharedPtr &requestRecv, int requestNum)

void	CalcRankDistances ()
	Populates m_foundRankCoords using the FindDistance function. More...

void	SendFwdTrace (LibUtilities::CommRequestSharedPtr &requestSend, LibUtilities::CommRequestSharedPtr &requestRecv, int requestNum, Array< OneD, NekDouble > &Fwd)

void	FillRankBwdTraceExchange (Array< OneD, NekDouble > &Bwd)

Private Attributes
SpatialDomains::MovementSharedPtr	m_movement
	Movement object associated with the non-conformal interfaces. More...

std::map< int, SpatialDomains::ZoneBaseShPtr >	m_zones
	Map of zone IDs to zone bases. More...

const ExpListSharedPtr	m_trace
	Trace expansion list. More...

const LibUtilities::CommSharedPtr	m_comm
	Communicator. More...

int	m_rank
	Process rank. More...

const std::vector< InterfaceTraceSharedPtr >	m_interfaceTraces
	Vector of interface traces i.e. every interface side present on m_rank. More...

Array< OneD, NekDouble >	m_send
	Send buffer for coord exchange. More...

Array< OneD, NekDouble >	m_recv
	Receive buffer for coord exchange. More...

Array< OneD, NekDouble >	m_recvTrace
	Receive buffer for trace exchange. More...

Array< OneD, NekDouble >	m_sendTrace
	Send buffer for trace exchange. More...

std::map< int, int >	m_totSendSize
	Map of rank to total size of send buffer for all interfaces. More...

std::map< int, int >	m_totRecvSize
	Map of rank to total size of receive buffer for all interfaces. More...

std::map< int, Array< OneD, int > >	m_sendSize
	Map of rank to array of size of send buffer for each interface. More...

std::map< int, Array< OneD, int > >	m_recvSize
	Map of rank to array of size of receive buffer for each interface. More...

std::map< int, std::map< int, std::pair< int, Array< OneD, NekDouble > > > >	m_foundRankCoords

Detailed Description

Object for one rank-to-rank communication for all interfaces shared between those ranks. e.g. if on rank 1 and there are interfaces shared with rank 2 and 3, there will be two InterfaceExchange objects in m_exchange in InterfaceMapDG. This holds the InterfaceTrace objects in m_interfaceTraces.

Definition at line 111 of file InterfaceMapDG.h.

Constructor & Destructor Documentation

◆ ~InterfaceExchange()

virtual Nektar::MultiRegions::InterfaceExchange::~InterfaceExchange ( )

virtualdefault

Default destructor.

◆ InterfaceExchange()

Nektar::MultiRegions::InterfaceExchange::InterfaceExchange	(	SpatialDomains::MovementSharedPtr	movement,
		const ExpListSharedPtr &	trace,
		const LibUtilities::CommSharedPtr &	comm,
		std::pair< int, std::vector< InterfaceTraceSharedPtr > >	rankPair
	)

inline

Constructor.

Definition at line 118 of file InterfaceMapDG.h.

        : m_movement(movement), m_zones(movement->GetZones()), m_trace(trace),
          m_comm(comm), m_rank(rankPair.first),
          m_interfaceTraces(rankPair.second)
    {
    }

Member Function Documentation

◆ CalcRankDistances()

void Nektar::MultiRegions::InterfaceExchange::CalcRankDistances ( )

Populates m_foundRankCoords using the FindDistance function.

Check coords in m_recv from other rank to see if present on this rank, and then populates m_foundRankCoords if found.

Loops over all interface traces present in the exchange, i.e. every interface needed on this rank-to-rank communication object
Then loops over all the quadrature points missing from the other side in that interface using global coordinates
Loop over each local edge and use MinMaxCheck on the point as a fast comparison on whether the point lies on the edge
If true then use the more expensive FindDistance function to find the closest local coordinate of the global 'missing' point on the local edge and the distance from the edge that corresponds to
If distance is less than 5e-5 save found coordinate in m_foundRankCoords

Definition at line 562 of file InterfaceMapDG.cpp.

{
    Array<OneD, int> disp(m_recvSize[m_rank].size() + 1, 0.0);
    std::partial_sum(m_recvSize[m_rank].begin(), m_recvSize[m_rank].end(),
                     &disp[1]);
 
    for (int i = 0; i < m_interfaceTraces.size(); ++i)
    {
        auto localEdge = m_interfaceTraces[i]->GetInterface()->GetEdge();
 
        for (int j = disp[i]; j < disp[i + 1]; j += 3)
        {
            Array<OneD, NekDouble> foundLocCoord;
            Array<OneD, NekDouble> xs(3);
            xs[0] = m_recv[j];
            xs[1] = m_recv[j + 1];
            xs[2] = m_recv[j + 2];
 
            for (auto &edge : localEdge)
            {
                // First check if inside the edge bounding box
                if (!edge.second->MinMaxCheck(xs))
                {
                    continue;
                }
 
                NekDouble dist = edge.second->FindDistance(xs, foundLocCoord);
 
                if (dist < NekConstants::kFindDistanceMin)
                {
                    m_foundRankCoords[m_rank][j / 3] = std::make_pair(
                        edge.second->GetGlobalID(), foundLocCoord);
                    break;
                }
            }
        }
    }
}

References Nektar::NekConstants::kFindDistanceMin, m_foundRankCoords, m_interfaceTraces, m_rank, m_recv, and m_recvSize.

◆ FillRankBwdTraceExchange()

void Nektar::MultiRegions::InterfaceExchange::FillRankBwdTraceExchange ( Array< OneD, NekDouble > & Bwd )

Loops over interfaces and partitions out the received trace from the other ranks for insertion into Bwd using FillRankBwdTrace

Parameters

Bwd	The Bwd trace to be filled from across the interface

Definition at line 492 of file InterfaceMapDG.cpp.

{
    int cnt = 0;
    for (int i = 0; i < m_interfaceTraces.size(); ++i)
    {
        Array<OneD, NekDouble> traceTmp(m_sendSize[m_rank][i] / 3, 0.0);
        for (int j = 0; j < m_sendSize[m_rank][i] / 3; ++j, ++cnt)
        {
            traceTmp[j] = m_recvTrace[cnt];
        }
 
        m_interfaceTraces[i]->FillRankBwdTrace(traceTmp, Bwd);
    }
}

References m_interfaceTraces, m_rank, m_recvTrace, and m_sendSize.

◆ RankFillSizes()

void Nektar::MultiRegions::InterfaceExchange::RankFillSizes	(	LibUtilities::CommRequestSharedPtr &	requestSend,
		LibUtilities::CommRequestSharedPtr &	requestRecv,
		int	requestNum
	)

Communicates with other ranks how many missing coordinates for each interface to expect

Parameters

requestSend	List of send requests
requestRecv	List of receive requests
requestNum	Index of request in list to use

Definition at line 349 of file InterfaceMapDG.cpp.

{
 
    // Recalc size is the number of interfaces missing points that need to be
    // communicated.
    int recalcSize = 0;
    for (auto &localInterface : m_interfaceTraces)
    {
        recalcSize +=
            m_zones[localInterface->GetInterface()->GetId()]->GetMoved();
    }
 
    m_sendSize[m_rank] = Array<OneD, int>(recalcSize);
    m_recvSize[m_rank] = Array<OneD, int>(recalcSize);
 
    int cnt = 0;
    for (auto &interface : m_interfaceTraces)
    {
        if (m_zones[interface->GetInterface()->GetId()]->GetMoved())
        {
            // Calculates the number of missing coordinates to be communicated
            // This size is 3 times the number of missing coordinates to allow
            // for 3D points
            m_sendSize[m_rank][cnt++] =
                interface->GetMissingCoords().size() * 3;
        }
    }
 
    // Uses non-blocking send/recvs to send communication sizes to other ranks
    m_comm->Isend(m_rank, m_sendSize[m_rank], m_interfaceTraces.size(),
                  requestSend, requestNum);
    m_comm->Irecv(m_rank, m_recvSize[m_rank], m_interfaceTraces.size(),
                  requestRecv, requestNum);
}

References m_comm, m_interfaceTraces, m_rank, m_recvSize, m_sendSize, and m_zones.

◆ SendFwdTrace()

void Nektar::MultiRegions::InterfaceExchange::SendFwdTrace	(	LibUtilities::CommRequestSharedPtr &	requestSend,
		LibUtilities::CommRequestSharedPtr &	requestRecv,
		int	requestNum,
		Array< OneD, NekDouble > &	Fwd
	)

Calculates and sends the trace to other rank from the m_foundRankCoords structure using non-blocking pairwise communication i.e. Isend & Irecv

Parameters

requestSend	List of send requests
requestRecv	List of receive requests
requestNum	Index of request in list to use
Fwd	The values to send across the interface

Definition at line 519 of file InterfaceMapDG.cpp.

{
    m_recvTrace =
        Array<OneD, NekDouble>(m_totSendSize[m_rank] / 3, std::nan(""));
    m_sendTrace =
        Array<OneD, NekDouble>(m_totRecvSize[m_rank] / 3, std::nan(""));
 
    for (auto &i : m_foundRankCoords[m_rank])
    {
        int traceId = m_trace->GetElmtToExpId(i.second.first);
        Array<OneD, NekDouble> locCoord = i.second.second;
 
        Array<OneD, NekDouble> edgePhys =
            Fwd + m_trace->GetPhys_Offset(traceId);
 
        m_sendTrace[i.first] =
            m_trace->GetExp(traceId)->StdPhysEvaluate(locCoord, edgePhys);
    }
 
    m_comm->Isend(m_rank, m_sendTrace, m_sendTrace.size(), requestSend,
                  requestNum);
    m_comm->Irecv(m_rank, m_recvTrace, m_recvTrace.size(), requestRecv,
                  requestNum);
}

References m_comm, m_foundRankCoords, m_rank, m_recvTrace, m_sendTrace, m_totRecvSize, m_totSendSize, and m_trace.

◆ SendMissing()

void Nektar::MultiRegions::InterfaceExchange::SendMissing	(	LibUtilities::CommRequestSharedPtr &	requestSend,
		LibUtilities::CommRequestSharedPtr &	requestRecv,
		int	requestNum
	)

Sends/receives the missing coordinates to/from other ranks

Parameters

requestSend	List of send requests
requestRecv	List of receive requests
requestNum	Index of request in list to use

Definition at line 388 of file InterfaceMapDG.cpp.

{
    m_totSendSize[m_rank] = std::accumulate(m_sendSize[m_rank].begin(),
                                            m_sendSize[m_rank].end(), 0);
    m_totRecvSize[m_rank] = std::accumulate(m_recvSize[m_rank].begin(),
                                            m_recvSize[m_rank].end(), 0);
 
    m_send = Array<OneD, NekDouble>(m_totSendSize[m_rank]);
    m_recv = Array<OneD, NekDouble>(m_totRecvSize[m_rank]);
 
    int cnt = 0;
    for (auto &interface : m_interfaceTraces)
    {
        if (m_zones[interface->GetInterface()->GetId()]->GetMoved())
        {
            auto missing = interface->GetMissingCoords();
            for (auto coord : missing)
            {
                // Points are organised in blocks of three
                // x coord, then y coord, then z coord.
                for (int k = 0; k < 3; ++k, ++cnt)
                {
                    m_send[cnt] = coord[k];
                }
            }
        }
    }
 
    // Uses non-blocking send/recvs to send missing points to other ranks
    m_comm->Isend(m_rank, m_send, m_totSendSize[m_rank], requestSend,
                  requestNum);
    m_comm->Irecv(m_rank, m_recv, m_totRecvSize[m_rank], requestRecv,
                  requestNum);
}

References m_comm, m_interfaceTraces, m_rank, m_recv, m_recvSize, m_send, m_sendSize, m_totRecvSize, m_totSendSize, and m_zones.

Member Data Documentation

◆ m_comm

const LibUtilities::CommSharedPtr Nektar::MultiRegions::InterfaceExchange::m_comm

private

Communicator.

Definition at line 185 of file InterfaceMapDG.h.

Referenced by RankFillSizes(), SendFwdTrace(), and SendMissing().

◆ m_foundRankCoords

std::map<int, std::map<int, std::pair<int, Array<OneD, NekDouble> > > > Nektar::MultiRegions::InterfaceExchange::m_foundRankCoords

private

Caches the found coordinates to reuse when exchanging the trace in a map of integer rank to a map of integer missing coordinate location to a pair of local edge ID and found local coordinate

Definition at line 213 of file InterfaceMapDG.h.

Referenced by CalcRankDistances(), and SendFwdTrace().

◆ m_interfaceTraces

const std::vector<InterfaceTraceSharedPtr> Nektar::MultiRegions::InterfaceExchange::m_interfaceTraces

private

Vector of interface traces i.e. every interface side present on m_rank.

Definition at line 189 of file InterfaceMapDG.h.

Referenced by CalcRankDistances(), FillRankBwdTraceExchange(), RankFillSizes(), and SendMissing().

◆ m_movement

SpatialDomains::MovementSharedPtr Nektar::MultiRegions::InterfaceExchange::m_movement

private

Movement object associated with the non-conformal interfaces.

Definition at line 179 of file InterfaceMapDG.h.

◆ m_rank

int Nektar::MultiRegions::InterfaceExchange::m_rank

private

Process rank.

Definition at line 187 of file InterfaceMapDG.h.

Referenced by CalcRankDistances(), FillRankBwdTraceExchange(), RankFillSizes(), SendFwdTrace(), and SendMissing().

◆ m_recv

Array<OneD, NekDouble> Nektar::MultiRegions::InterfaceExchange::m_recv

private

Receive buffer for coord exchange.

Definition at line 193 of file InterfaceMapDG.h.

Referenced by CalcRankDistances(), and SendMissing().

◆ m_recvSize

std::map<int, Array<OneD, int> > Nektar::MultiRegions::InterfaceExchange::m_recvSize

private

Map of rank to array of size of receive buffer for each interface.

Definition at line 205 of file InterfaceMapDG.h.

Referenced by CalcRankDistances(), RankFillSizes(), and SendMissing().

◆ m_recvTrace

Array<OneD, NekDouble> Nektar::MultiRegions::InterfaceExchange::m_recvTrace

private

Receive buffer for trace exchange.

Definition at line 195 of file InterfaceMapDG.h.

Referenced by FillRankBwdTraceExchange(), and SendFwdTrace().

◆ m_send

Array<OneD, NekDouble> Nektar::MultiRegions::InterfaceExchange::m_send

private

Send buffer for coord exchange.

Definition at line 191 of file InterfaceMapDG.h.

Referenced by SendMissing().

◆ m_sendSize

std::map<int, Array<OneD, int> > Nektar::MultiRegions::InterfaceExchange::m_sendSize

private

Map of rank to array of size of send buffer for each interface.

Definition at line 203 of file InterfaceMapDG.h.

Referenced by FillRankBwdTraceExchange(), RankFillSizes(), and SendMissing().

◆ m_sendTrace

Array<OneD, NekDouble> Nektar::MultiRegions::InterfaceExchange::m_sendTrace

private

Send buffer for trace exchange.

Definition at line 197 of file InterfaceMapDG.h.

Referenced by SendFwdTrace().

◆ m_totRecvSize

std::map<int, int> Nektar::MultiRegions::InterfaceExchange::m_totRecvSize

private

Map of rank to total size of receive buffer for all interfaces.

Definition at line 201 of file InterfaceMapDG.h.

Referenced by SendFwdTrace(), and SendMissing().

◆ m_totSendSize

std::map<int, int> Nektar::MultiRegions::InterfaceExchange::m_totSendSize

private

Map of rank to total size of send buffer for all interfaces.

Definition at line 199 of file InterfaceMapDG.h.

Referenced by SendFwdTrace(), and SendMissing().

◆ m_trace

const ExpListSharedPtr Nektar::MultiRegions::InterfaceExchange::m_trace

private

Trace expansion list.

Definition at line 183 of file InterfaceMapDG.h.

Referenced by SendFwdTrace().

◆ m_zones

std::map<int, SpatialDomains::ZoneBaseShPtr> Nektar::MultiRegions::InterfaceExchange::m_zones

private

Map of zone IDs to zone bases.

Definition at line 181 of file InterfaceMapDG.h.

Referenced by RankFillSizes(), and SendMissing().

Public Member Functions

Private Attributes

Detailed Description

Constructor & Destructor Documentation

◆ ~InterfaceExchange()

◆ InterfaceExchange()

Member Function Documentation

◆ CalcRankDistances()

◆ FillRankBwdTraceExchange()

◆ RankFillSizes()

◆ SendFwdTrace()

◆ SendMissing()

Member Data Documentation

◆ m_comm

◆ m_foundRankCoords

◆ m_interfaceTraces

◆ m_movement

◆ m_rank

◆ m_recv

◆ m_recvSize

◆ m_recvTrace

◆ m_send

◆ m_sendSize

◆ m_sendTrace

◆ m_totRecvSize

◆ m_totSendSize

◆ m_trace

◆ m_zones