#include <InterfaceMapDG.h>

Public Member Functions
virtual	~InterfaceExchange ()=default
	Default destructor.

	InterfaceExchange (SpatialDomains::MovementSharedPtr movement, const ExpListSharedPtr &trace, const LibUtilities::CommSharedPtr &comm, std::pair< int, std::vector< InterfaceTraceSharedPtr > > rankPair)
	Constructor.

void	RankFillSizes (LibUtilities::CommRequestSharedPtr &requestSend, LibUtilities::CommRequestSharedPtr &requestRecv, int requestNum)

void	SendMissing (LibUtilities::CommRequestSharedPtr &requestSend, LibUtilities::CommRequestSharedPtr &requestRecv, int requestNum)

void	CalcRankDistances ()
	Populates m_foundRankCoords using the FindDistance function.

void	SendFwdTrace (LibUtilities::CommRequestSharedPtr &requestSend, LibUtilities::CommRequestSharedPtr &requestRecv, int requestNum, Array< OneD, NekDouble > &Fwd)

void	FillRankBwdTraceExchange (Array< OneD, NekDouble > &Bwd)

Private Attributes
SpatialDomains::MovementSharedPtr	m_movement
	Movement object associated with the non-conformal interfaces.

std::map< int, SpatialDomains::ZoneBaseShPtr >	m_zones
	Map of zone IDs to zone bases.

const ExpListSharedPtr	m_trace
	Trace expansion list.

const LibUtilities::CommSharedPtr	m_comm
	Communicator.

int	m_rank
	Process rank.

const std::vector< InterfaceTraceSharedPtr >	m_interfaceTraces
	Vector of interface traces i.e. every interface side present on m_rank.

Array< OneD, NekDouble >	m_send
	Send buffer for coord exchange.

Array< OneD, NekDouble >	m_recv
	Receive buffer for coord exchange.

Array< OneD, NekDouble >	m_recvTrace
	Receive buffer for trace exchange.

Array< OneD, NekDouble >	m_sendTrace
	Send buffer for trace exchange.

std::map< int, int >	m_totSendSize
	Map of rank to total size of send buffer for all interfaces.

std::map< int, int >	m_totRecvSize
	Map of rank to total size of receive buffer for all interfaces.

std::map< int, Array< OneD, int > >	m_sendSize
	Map of rank to array of size of send buffer for each interface.

std::map< int, Array< OneD, int > >	m_recvSize
	Map of rank to array of size of receive buffer for each interface.

std::map< int, std::map< int, std::pair< std::pair< int, int >, Array< OneD, NekDouble > > > >	m_foundRankCoords

Detailed Description

Object for one rank-to-rank communication for all interfaces shared between those ranks. e.g. if on rank 1 and there are interfaces shared with rank 2 and 3, there will be two InterfaceExchange objects in m_exchange in InterfaceMapDG. This holds the InterfaceTrace objects in m_interfaceTraces.

Definition at line 136 of file InterfaceMapDG.h.

Constructor & Destructor Documentation

◆ ~InterfaceExchange()

virtual Nektar::MultiRegions::InterfaceExchange::~InterfaceExchange ( )

virtualdefault

Default destructor.

◆ InterfaceExchange()

Nektar::MultiRegions::InterfaceExchange::InterfaceExchange	(	SpatialDomains::MovementSharedPtr	movement,
		const ExpListSharedPtr &	trace,
		const LibUtilities::CommSharedPtr &	comm,
		std::pair< int, std::vector< InterfaceTraceSharedPtr > >	rankPair
	)

inline

Constructor.

Definition at line 143 of file InterfaceMapDG.h.

        : m_movement(movement), m_zones(movement->GetZones()), m_trace(trace),
          m_comm(comm), m_rank(rankPair.first),
          m_interfaceTraces(rankPair.second)
    {
    }

Member Function Documentation

◆ CalcRankDistances()

void Nektar::MultiRegions::InterfaceExchange::CalcRankDistances ( )

Populates m_foundRankCoords using the FindDistance function.

Check coords in m_recv from other rank to see if present on this rank, and then populates m_foundRankCoords if found.

Loops over all interface traces present in the exchange, i.e. every interface needed on this rank-to-rank communication object
Then loops over all the quadrature points missing from the other side in that interface using global coordinates
Loop over each local edge and use MinMaxCheck on the point as a fast comparison on whether the point lies on the edge
If true then use the more expensive FindDistance function to find the closest local coordinate of the global 'missing' point on the local edge and the distance from the edge that corresponds to
If distance is less than 5e-5 save found coordinate in m_foundRankCoords

Definition at line 741 of file InterfaceMapDG.cpp.

{
    // Clear old found coordinates
    auto foundRankCoordsCopy = m_foundRankCoords[m_rank];
    m_foundRankCoords[m_rank]
        .clear(); // @TODO: This may cause problems with 2 interfaces and one is
                  // fixed? With the skip below.
 
    Array<OneD, int> disp(m_recvSize[m_rank].size() + 1, 0.0);
    std::partial_sum(m_recvSize[m_rank].begin(), m_recvSize[m_rank].end(),
                     &disp[1]);
 
    for (int i = 0; i < m_interfaceTraces.size(); ++i)
    {
        if (!m_zones[m_interfaceTraces[i]->GetInterface()->GetId()]->GetMoved())
        {
            // If zone is not moved then skip
            continue;
        }
 
        auto localEdge = m_interfaceTraces[i]->GetInterface()->GetEdge();
 
        for (int j = disp[i]; j < disp[i + 1]; j += 3)
        {
            Array<OneD, NekDouble> foundLocCoord;
            Array<OneD, NekDouble> xs(3);
            xs[0] = m_recv[j];
            xs[1] = m_recv[j + 1];
            xs[2] = m_recv[j + 2];
 
            // First search the edge the point was found in last timestep
            if ((foundRankCoordsCopy.find(j / 3) !=
                 foundRankCoordsCopy.end()) &&
                (foundRankCoordsCopy[j / 3].first.first == i))
            {
                auto edge = m_interfaceTraces[i]->GetInterface()->GetEdge(
                    foundRankCoordsCopy[j / 3].first.second);
                NekDouble dist = edge->FindDistance(xs, foundLocCoord);
 
                if (dist < NekConstants::kFindDistanceMin)
                {
                    m_foundRankCoords[m_rank][j / 3] = std::make_pair(
                        std::make_pair(i, edge->GetGlobalID()), foundLocCoord);
                    continue;
                }
            }
 
            for (auto &edge : localEdge)
            {
                // First check if inside the edge bounding box
                if (!edge.second->MinMaxCheck(xs))
                {
                    continue;
                }
 
                NekDouble dist = edge.second->FindDistance(xs, foundLocCoord);
 
                if (dist < NekConstants::kFindDistanceMin)
                {
                    m_foundRankCoords[m_rank][j / 3] = std::make_pair(
                        std::make_pair(i, edge.second->GetGlobalID()),
                        foundLocCoord);
                    break;
                }
            }
        }
    }
}

References Nektar::NekConstants::kFindDistanceMin, m_foundRankCoords, m_interfaceTraces, m_rank, m_recv, m_recvSize, and m_zones.

◆ FillRankBwdTraceExchange()

void Nektar::MultiRegions::InterfaceExchange::FillRankBwdTraceExchange ( Array< OneD, NekDouble > & Bwd )

Loops over interfaces and partitions out the received trace from the other ranks for insertion into Bwd using FillRankBwdTrace

Parameters

Bwd	The Bwd trace to be filled from across the interface

Definition at line 671 of file InterfaceMapDG.cpp.

{
    int cnt = 0;
    for (int i = 0; i < m_interfaceTraces.size(); ++i)
    {
        Array<OneD, NekDouble> traceTmp(m_sendSize[m_rank][i] / 3, 0.0);
        for (int j = 0; j < m_sendSize[m_rank][i] / 3; ++j, ++cnt)
        {
            traceTmp[j] = m_recvTrace[cnt];
        }
 
        m_interfaceTraces[i]->FillRankBwdTrace(traceTmp, Bwd);
    }
}

References m_interfaceTraces, m_rank, m_recvTrace, and m_sendSize.

◆ RankFillSizes()

void Nektar::MultiRegions::InterfaceExchange::RankFillSizes	(	LibUtilities::CommRequestSharedPtr &	requestSend,
		LibUtilities::CommRequestSharedPtr &	requestRecv,
		int	requestNum
	)

Communicates with other ranks how many missing coordinates for each interface to expect

Parameters

requestSend	List of send requests
requestRecv	List of receive requests
requestNum	Index of request in list to use

Definition at line 420 of file InterfaceMapDG.cpp.

{
 
    // Recalc size is the number of interfaces missing points that need to be
    // communicated.
    int recalcSize = 0;
    for (auto &localInterface : m_interfaceTraces)
    {
        recalcSize +=
            m_zones[localInterface->GetInterface()->GetId()]->GetMoved();
    }
 
    m_sendSize[m_rank] = Array<OneD, int>(recalcSize);
    m_recvSize[m_rank] = Array<OneD, int>(recalcSize);
 
    int cnt = 0;
    for (auto &interface : m_interfaceTraces)
    {
        if (m_zones[interface->GetInterface()->GetId()]->GetMoved())
        {
            // Calculates the number of missing coordinates to be communicated
            // This size is 3 times the number of missing coordinates to allow
            // for 3D points
            m_sendSize[m_rank][cnt++] =
                interface->GetMissingCoords().size() * 3;
        }
    }
 
    // Uses non-blocking send/recvs to send communication sizes to other ranks
    m_comm->Isend(m_rank, m_sendSize[m_rank], m_interfaceTraces.size(),
                  requestSend, requestNum);
    m_comm->Irecv(m_rank, m_recvSize[m_rank], m_interfaceTraces.size(),
                  requestRecv, requestNum);
}

References m_comm, m_interfaceTraces, m_rank, m_recvSize, m_sendSize, and m_zones.

◆ SendFwdTrace()

void Nektar::MultiRegions::InterfaceExchange::SendFwdTrace	(	LibUtilities::CommRequestSharedPtr &	requestSend,
		LibUtilities::CommRequestSharedPtr &	requestRecv,
		int	requestNum,
		Array< OneD, NekDouble > &	Fwd
	)

Calculates and sends the trace to other rank from the m_foundRankCoords structure using non-blocking pairwise communication i.e. Isend & Irecv

Parameters

requestSend	List of send requests
requestRecv	List of receive requests
requestNum	Index of request in list to use
Fwd	The values to send across the interface

Definition at line 698 of file InterfaceMapDG.cpp.

{
    m_recvTrace =
        Array<OneD, NekDouble>(m_totSendSize[m_rank] / 3, std::nan(""));
    m_sendTrace =
        Array<OneD, NekDouble>(m_totRecvSize[m_rank] / 3, std::nan(""));
 
    for (auto &i : m_foundRankCoords[m_rank])
    {
        int traceId = m_trace->GetElmtToExpId(i.second.first.second);
        Array<OneD, NekDouble> locCoord = i.second.second;
 
        Array<OneD, NekDouble> edgePhys =
            Fwd + m_trace->GetPhys_Offset(traceId);
 
        m_sendTrace[i.first] =
            m_trace->GetExp(traceId)->StdPhysEvaluate(locCoord, edgePhys);
    }
 
    m_comm->Isend(m_rank, m_sendTrace, m_sendTrace.size(), requestSend,
                  requestNum);
    m_comm->Irecv(m_rank, m_recvTrace, m_recvTrace.size(), requestRecv,
                  requestNum);
}

References m_comm, m_foundRankCoords, m_rank, m_recvTrace, m_sendTrace, m_totRecvSize, m_totSendSize, and m_trace.

◆ SendMissing()

void Nektar::MultiRegions::InterfaceExchange::SendMissing	(	LibUtilities::CommRequestSharedPtr &	requestSend,
		LibUtilities::CommRequestSharedPtr &	requestRecv,
		int	requestNum
	)

Sends/receives the missing coordinates to/from other ranks

Parameters

requestSend	List of send requests
requestRecv	List of receive requests
requestNum	Index of request in list to use

Definition at line 459 of file InterfaceMapDG.cpp.

{
    m_totSendSize[m_rank] = std::accumulate(m_sendSize[m_rank].begin(),
                                            m_sendSize[m_rank].end(), 0);
    m_totRecvSize[m_rank] = std::accumulate(m_recvSize[m_rank].begin(),
                                            m_recvSize[m_rank].end(), 0);
 
    m_send = Array<OneD, NekDouble>(m_totSendSize[m_rank]);
    m_recv = Array<OneD, NekDouble>(m_totRecvSize[m_rank]);
 
    int cnt = 0;
    for (auto &interface : m_interfaceTraces)
    {
        if (m_zones[interface->GetInterface()->GetId()]->GetMoved())
        {
            auto missing = interface->GetMissingCoords();
            for (auto coord : missing)
            {
                // Points are organised in blocks of three
                // x coord, then y coord, then z coord.
                for (int k = 0; k < 3; ++k, ++cnt)
                {
                    m_send[cnt] = coord[k];
                }
            }
        }
    }
 
    // Uses non-blocking send/recvs to send missing points to other ranks
    m_comm->Isend(m_rank, m_send, m_totSendSize[m_rank], requestSend,
                  requestNum);
    m_comm->Irecv(m_rank, m_recv, m_totRecvSize[m_rank], requestRecv,
                  requestNum);
}

References m_comm, m_interfaceTraces, m_rank, m_recv, m_recvSize, m_send, m_sendSize, m_totRecvSize, m_totSendSize, and m_zones.

Member Data Documentation

◆ m_comm

const LibUtilities::CommSharedPtr Nektar::MultiRegions::InterfaceExchange::m_comm

private

Communicator.

Definition at line 210 of file InterfaceMapDG.h.

Referenced by RankFillSizes(), SendFwdTrace(), and SendMissing().

◆ m_foundRankCoords

std::map< int, std::map<int, std::pair<std::pair<int, int>, Array<OneD, NekDouble> > > > Nektar::MultiRegions::InterfaceExchange::m_foundRankCoords

private

Caches the found coordinates to reuse when exchanging the trace in a map of integer rank to a map of integer missing coordinate location to a pair of local edge ID and found local coordinate

Definition at line 240 of file InterfaceMapDG.h.

Referenced by CalcRankDistances(), and SendFwdTrace().

◆ m_interfaceTraces

const std::vector<InterfaceTraceSharedPtr> Nektar::MultiRegions::InterfaceExchange::m_interfaceTraces

private

Vector of interface traces i.e. every interface side present on m_rank.

Definition at line 214 of file InterfaceMapDG.h.

Referenced by CalcRankDistances(), FillRankBwdTraceExchange(), RankFillSizes(), and SendMissing().

◆ m_movement

SpatialDomains::MovementSharedPtr Nektar::MultiRegions::InterfaceExchange::m_movement

private

Movement object associated with the non-conformal interfaces.

Definition at line 204 of file InterfaceMapDG.h.

◆ m_rank

int Nektar::MultiRegions::InterfaceExchange::m_rank

private

Process rank.

Definition at line 212 of file InterfaceMapDG.h.

Referenced by CalcRankDistances(), FillRankBwdTraceExchange(), RankFillSizes(), SendFwdTrace(), and SendMissing().

◆ m_recv

Array<OneD, NekDouble> Nektar::MultiRegions::InterfaceExchange::m_recv

private

Receive buffer for coord exchange.

Definition at line 218 of file InterfaceMapDG.h.

Referenced by CalcRankDistances(), and SendMissing().

◆ m_recvSize

std::map<int, Array<OneD, int> > Nektar::MultiRegions::InterfaceExchange::m_recvSize

private

Map of rank to array of size of receive buffer for each interface.

Definition at line 230 of file InterfaceMapDG.h.

Referenced by CalcRankDistances(), RankFillSizes(), and SendMissing().

◆ m_recvTrace

Array<OneD, NekDouble> Nektar::MultiRegions::InterfaceExchange::m_recvTrace

private

Receive buffer for trace exchange.

Definition at line 220 of file InterfaceMapDG.h.

Referenced by FillRankBwdTraceExchange(), and SendFwdTrace().

◆ m_send

Array<OneD, NekDouble> Nektar::MultiRegions::InterfaceExchange::m_send

private

Send buffer for coord exchange.

Definition at line 216 of file InterfaceMapDG.h.

Referenced by SendMissing().

◆ m_sendSize

std::map<int, Array<OneD, int> > Nektar::MultiRegions::InterfaceExchange::m_sendSize

private

Map of rank to array of size of send buffer for each interface.

Definition at line 228 of file InterfaceMapDG.h.

Referenced by FillRankBwdTraceExchange(), RankFillSizes(), and SendMissing().

◆ m_sendTrace

Array<OneD, NekDouble> Nektar::MultiRegions::InterfaceExchange::m_sendTrace

private

Send buffer for trace exchange.

Definition at line 222 of file InterfaceMapDG.h.

Referenced by SendFwdTrace().

◆ m_totRecvSize

std::map<int, int> Nektar::MultiRegions::InterfaceExchange::m_totRecvSize

private

Map of rank to total size of receive buffer for all interfaces.

Definition at line 226 of file InterfaceMapDG.h.

Referenced by SendFwdTrace(), and SendMissing().

◆ m_totSendSize

std::map<int, int> Nektar::MultiRegions::InterfaceExchange::m_totSendSize

private

Map of rank to total size of send buffer for all interfaces.

Definition at line 224 of file InterfaceMapDG.h.

Referenced by SendFwdTrace(), and SendMissing().

◆ m_trace

const ExpListSharedPtr Nektar::MultiRegions::InterfaceExchange::m_trace

private

Trace expansion list.

Definition at line 208 of file InterfaceMapDG.h.

Referenced by SendFwdTrace().

◆ m_zones

std::map<int, SpatialDomains::ZoneBaseShPtr> Nektar::MultiRegions::InterfaceExchange::m_zones

private

Map of zone IDs to zone bases.

Definition at line 206 of file InterfaceMapDG.h.

Referenced by CalcRankDistances(), RankFillSizes(), and SendMissing().

Public Member Functions

Private Attributes

Detailed Description

Constructor & Destructor Documentation

◆ ~InterfaceExchange()

◆ InterfaceExchange()

Member Function Documentation

◆ CalcRankDistances()

◆ FillRankBwdTraceExchange()

◆ RankFillSizes()

◆ SendFwdTrace()

◆ SendMissing()

Member Data Documentation

◆ m_comm

◆ m_foundRankCoords

◆ m_interfaceTraces

◆ m_movement

◆ m_rank

◆ m_recv

◆ m_recvSize

◆ m_recvTrace

◆ m_send

◆ m_sendSize

◆ m_sendTrace

◆ m_totRecvSize

◆ m_totSendSize

◆ m_trace

◆ m_zones