DriverModifiedArnoldi.cpp

Go to the documentation of this file.

///////////////////////////////////////////////////////////////////////////////
//
// File: DriverModifiedArnoldi.cpp
//
// For more information, please see: http://www.nektar.info
//
// The MIT License
//
// Copyright (c) 2006 Division of Applied Mathematics, Brown University (USA),
// Department of Aeronautics, Imperial College London (UK), and Scientific
// Computing and Imaging Institute, University of Utah (USA).
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//
// Description: Driver for eigenvalue analysis using the modified Arnoldi
//              method.
//
///////////////////////////////////////////////////////////////////////////////
 
#include <iomanip>
 
#include <boost/core/ignore_unused.hpp>
 
#include <SolverUtils/DriverModifiedArnoldi.h>
 
using namespace std;
 
namespace Nektar
{
namespace SolverUtils
{
 
string DriverModifiedArnoldi::className =
    GetDriverFactory().RegisterCreatorFunction("ModifiedArnoldi",
                                               DriverModifiedArnoldi::create);
string DriverModifiedArnoldi::driverLookupId =
    LibUtilities::SessionReader::RegisterEnumValue("Driver", "ModifiedArnoldi",
                                                   0);
 
/**
 *
 */
DriverModifiedArnoldi::DriverModifiedArnoldi(
    const LibUtilities::SessionReaderSharedPtr pSession,
    const SpatialDomains::MeshGraphSharedPtr pGraph)
    : DriverArnoldi(pSession, pGraph)
{
}
 
/**
 *
 */
DriverModifiedArnoldi::~DriverModifiedArnoldi()
{
}
 
/**
 *
 */
void DriverModifiedArnoldi::v_InitObject(ostream &out)
{
    DriverArnoldi::v_InitObject(out);
 
    m_equ[0]->PrintSummary(out);
 
    // Print session parameters
    if (m_comm->GetRank() == 0)
    {
        out << "\tArnoldi solver type    : Modified Arnoldi" << endl;
    }
 
    DriverArnoldi::ArnoldiSummary(out);
 
    for (int i = 0; i < m_nequ; ++i)
    {
        m_equ[i]->DoInitialise();
    }
 
    // FwdTrans Initial conditions to be in Coefficient Space
    m_equ[m_nequ - 1]->TransPhysToCoeff();
}
 
/**
 *
 */
void DriverModifiedArnoldi::v_Execute(ostream &out)
{
    int i             = 0;
    int j             = 0;
    int nq            = m_equ[0]->UpdateFields()[0]->GetNcoeffs();
    int ntot          = m_nfields * nq;
    int converged     = 0;
    NekDouble resnorm = 0.0;
    ofstream evlout;
    std::string evlFile = m_session->GetSessionName() + ".evl";
 
    if (m_comm->GetRank() == 0)
    {
        evlout.open(evlFile.c_str());
    }
 
    // Allocate memory
    Array<OneD, NekDouble> alpha = Array<OneD, NekDouble>(m_kdim + 1, 0.0);
    Array<OneD, NekDouble> wr    = Array<OneD, NekDouble>(m_kdim, 0.0);
    Array<OneD, NekDouble> wi    = Array<OneD, NekDouble>(m_kdim, 0.0);
    Array<OneD, NekDouble> zvec  = Array<OneD, NekDouble>(m_kdim * m_kdim, 0.0);
 
    Array<OneD, Array<OneD, NekDouble>> Kseq =
        Array<OneD, Array<OneD, NekDouble>>(m_kdim + 1);
    Array<OneD, Array<OneD, NekDouble>> Kseqcopy =
        Array<OneD, Array<OneD, NekDouble>>(m_kdim + 1);
    Array<OneD, Array<OneD, NekDouble>> Tseq =
        Array<OneD, Array<OneD, NekDouble>>(m_kdim + 1);
    for (i = 0; i < m_kdim + 1; ++i)
    {
        Kseq[i] = Array<OneD, NekDouble>(ntot, 0.0);
        if (m_useMask)
        {
            Kseqcopy[i] = Array<OneD, NekDouble>(ntot, 0.0);
        }
        else
        {
            Kseqcopy[i] = Kseq[i];
        }
        Tseq[i] = Array<OneD, NekDouble>(ntot, 0.0);
    }
 
    // Copy starting vector into second sequence element (temporary).
    if (m_session->DefinesFunction("InitialConditions"))
    {
        if (m_comm->GetRank() == 0)
        {
            out << "\tInital vector       : specified in input file " << endl;
        }
        m_equ[0]->SetInitialConditions(0.0, false);
 
        CopyFieldToArnoldiArray(Kseq[1]);
    }
    else
    {
        if (m_comm->GetRank() == 0)
        {
            out << "\tInital vector       : random  " << endl;
        }
 
        NekDouble eps = 0.0001;
        Vmath::FillWhiteNoise(ntot, eps, &Kseq[1][0], 1);
        if (m_useMask)
        {
            Vmath::Vmul(ntot, Kseq[1], 1, GetMaskCoeff(), 1, Kseq[1], 1);
        }
    }
 
    // Perform one iteration to enforce boundary conditions.
    // Set this as the initial value in the sequence.
    EV_update(Kseq[1], Kseq[0]);
    if (m_comm->GetRank() == 0)
    {
        out << "Iteration: " << 0 << endl;
    }
 
    // Normalise first vector in sequence
    if (m_useMask)
    {
        Vmath::Vmul(ntot, Kseq[0], 1, GetMaskCoeff(), 1, Kseqcopy[0], 1);
    }
    alpha[0] = Blas::Ddot(ntot, &Kseqcopy[0][0], 1, &Kseqcopy[0][0], 1);
    m_comm->AllReduce(alpha[0], Nektar::LibUtilities::ReduceSum);
    alpha[0] = std::sqrt(alpha[0]);
    Vmath::Smul(ntot, 1.0 / alpha[0], Kseq[0], 1, Kseq[0], 1);
 
    // Fill initial krylov sequence
    NekDouble resid0;
    for (i = 1; !converged && i <= m_kdim; ++i)
    {
        // Compute next vector
        EV_update(Kseq[i - 1], Kseq[i]);
 
        // Normalise
        if (m_useMask)
        {
            Vmath::Vmul(ntot, Kseq[i], 1, GetMaskCoeff(), 1, Kseqcopy[i], 1);
        }
        alpha[i] = Blas::Ddot(ntot, &Kseqcopy[i][0], 1, &Kseqcopy[i][0], 1);
        m_comm->AllReduce(alpha[i], Nektar::LibUtilities::ReduceSum);
        alpha[i] = std::sqrt(alpha[i]);
 
        // alpha[i] = std::sqrt(alpha[i]);
        Vmath::Smul(ntot, 1.0 / alpha[i], Kseq[i], 1, Kseq[i], 1);
 
        // Copy Krylov sequence into temporary storage
        for (int k = 0; k < i + 1; ++k)
        {
            if (m_useMask)
            {
                Vmath::Vmul(ntot, Kseq[k], 1, GetMaskCoeff(), 1, Tseq[k], 1);
                Vmath::Vcopy(ntot, Kseq[k], 1, Kseqcopy[k], 1);
            }
            else
            {
                Vmath::Vcopy(ntot, Kseq[k], 1, Tseq[k], 1);
                ;
            }
        }
 
        // Generate Hessenberg matrix and compute eigenvalues of it.
        EV_small(Tseq, Kseqcopy, ntot, alpha, i, zvec, wr, wi, resnorm);
 
        // Test for convergence.
        converged = EV_test(i, i, zvec, wr, wi, resnorm, std::min(i, m_nvec),
                            evlout, resid0);
        if (i >= m_nvec)
        {
            converged = max(converged, 0);
        }
        else
        {
            converged = 0;
        }
 
        if (m_comm->GetRank() == 0)
        {
            out << "Iteration: " << i << " (residual : " << resid0 << ")"
                << endl;
        }
    }
 
    // Continue with full sequence
    if (!converged)
    {
        for (i = m_kdim + 1; !converged && i <= m_nits; ++i)
        {
            // Shift all the vectors in the sequence.
            // First vector is removed.
            for (int j = 1; j <= m_kdim; ++j)
            {
                alpha[j - 1] = alpha[j];
                Vmath::Vcopy(ntot, Kseq[j], 1, Kseq[j - 1], 1);
            }
 
            // Compute next vector
            EV_update(Kseq[m_kdim - 1], Kseq[m_kdim]);
 
            // Compute new scale factor
            if (m_useMask)
            {
                Vmath::Vmul(ntot, Kseq[m_kdim], 1, GetMaskCoeff(), 1,
                            Kseqcopy[m_kdim], 1);
            }
            alpha[m_kdim] = Blas::Ddot(ntot, &Kseqcopy[m_kdim][0], 1,
                                       &Kseqcopy[m_kdim][0], 1);
            m_comm->AllReduce(alpha[m_kdim], Nektar::LibUtilities::ReduceSum);
            alpha[m_kdim] = std::sqrt(alpha[m_kdim]);
            Vmath::Smul(ntot, 1.0 / alpha[m_kdim], Kseq[m_kdim], 1,
                        Kseq[m_kdim], 1);
 
            // Copy Krylov sequence into temporary storage
            for (int k = 0; k < m_kdim + 1; ++k)
            {
                if (m_useMask)
                {
                    Vmath::Vmul(ntot, Kseq[k], 1, GetMaskCoeff(), 1, Tseq[k],
                                1);
                    Vmath::Vcopy(ntot, Kseq[k], 1, Kseqcopy[k], 1);
                }
                else
                {
                    Vmath::Vcopy(ntot, Kseq[k], 1, Tseq[k], 1);
                    ;
                }
            }
 
            // Generate Hessenberg matrix and compute eigenvalues of it
            EV_small(Tseq, Kseqcopy, ntot, alpha, m_kdim, zvec, wr, wi,
                     resnorm);
 
            // Test for convergence.
            converged = EV_test(i, m_kdim, zvec, wr, wi, resnorm, m_nvec,
                                evlout, resid0);
 
            if (m_comm->GetRank() == 0)
            {
                out << "Iteration: " << i << " (residual : " << resid0 << ")"
                    << endl;
            }
        }
    }
 
    m_equ[0]->Output();
 
    // Evaluate and output computation time and solution accuracy.
    // The specific format of the error output is essential for the
    // regression tests to work.
    // Evaluate L2 Error
    for (j = 0; j < m_equ[0]->GetNvariables(); ++j)
    {
        NekDouble vL2Error   = m_equ[0]->L2Error(j, false);
        NekDouble vLinfError = m_equ[0]->LinfError(j);
        if (m_comm->GetRank() == 0)
        {
            out << "L 2 error (variable " << m_equ[0]->GetVariable(j)
                << ") : " << vL2Error << endl;
            out << "L inf error (variable " << m_equ[0]->GetVariable(j)
                << ") : " << vLinfError << endl;
        }
    }
 
    // Process eigenvectors and write out.
    m_nvec = converged;
    EV_post(Tseq, Kseqcopy, ntot, min(--i, m_kdim), m_nvec, zvec, wr, wi,
            converged);
 
    WARNINGL0(m_imagShift == 0, "Complex Shift applied. "
                                "Need to implement Ritz re-evaluation of"
                                "eigenvalue. Only one half of complex "
                                "value will be correct");
 
    // store eigenvalues so they can be accessed from driver class
    m_real_evl = wr;
    m_imag_evl = wi;
 
    // Close the runtime info file.
    if (m_comm->GetRank() == 0)
    {
        evlout.close();
    }
}
 
/**
 *
 */
void DriverModifiedArnoldi::EV_update(Array<OneD, NekDouble> &src,
                                      Array<OneD, NekDouble> &tgt)
{
    // Copy starting vector into first sequence element.
    CopyArnoldiArrayToField(src);
    m_equ[0]->TransCoeffToPhys();
 
    m_equ[0]->SetTime(0.);
    m_equ[0]->DoSolve();
 
    if (m_EvolutionOperator == eTransientGrowth)
    {
        Array<OneD, MultiRegions::ExpListSharedPtr> fields;
        fields = m_equ[0]->UpdateFields();
 
        // start Adjoint with latest fields of direct
        CopyFwdToAdj();
        m_equ[1]->TransCoeffToPhys();
 
        m_equ[1]->SetTime(0.);
        m_equ[1]->DoSolve();
    }
 
    // Copy starting vector into first sequence element.
    CopyFieldToArnoldiArray(tgt);
}
 
/**
 * Computes the Ritz eigenvalues and eigenvectors of the Hessenberg matrix
 * constructed from the Krylov sequence.
 */
void DriverModifiedArnoldi::EV_small(
    Array<OneD, Array<OneD, NekDouble>> &Kseq,
    Array<OneD, Array<OneD, NekDouble>> &Kseqcopy, const int ntot,
    const Array<OneD, NekDouble> &alpha, const int kdim,
    Array<OneD, NekDouble> &zvec, Array<OneD, NekDouble> &wr,
    Array<OneD, NekDouble> &wi, NekDouble &resnorm)
{
    int kdimp = kdim + 1;
    int lwork = 10 * kdim;
    int ier;
    Array<OneD, NekDouble> R(kdimp * kdimp, 0.0);
    Array<OneD, NekDouble> H(kdimp * kdim, 0.0);
    Array<OneD, NekDouble> rwork(lwork, 0.0);
 
    // Modified G-S orthonormalisation
    for (int i = 0; i < kdimp; ++i)
    {
        NekDouble gsc = Blas::Ddot(ntot, &Kseq[i][0], 1, &Kseq[i][0], 1);
        m_comm->AllReduce(gsc, Nektar::LibUtilities::ReduceSum);
        gsc = std::sqrt(gsc);
        ASSERTL0(gsc != 0.0, "Vectors are linearly independent.");
 
        R[i * kdimp + i] = gsc;
        Vmath::Smul(ntot, 1.0 / gsc, Kseq[i], 1, Kseq[i], 1);
        if (m_useMask)
        {
            Vmath::Smul(ntot, 1.0 / gsc, Kseqcopy[i], 1, Kseqcopy[i], 1);
        }
 
        for (int j = i + 1; j < kdimp; ++j)
        {
            gsc = Blas::Ddot(ntot, &Kseq[i][0], 1, &Kseq[j][0], 1);
            m_comm->AllReduce(gsc, Nektar::LibUtilities::ReduceSum);
            Vmath::Svtvp(ntot, -gsc, Kseq[i], 1, Kseq[j], 1, Kseq[j], 1);
            if (m_useMask)
            {
                Vmath::Svtvp(ntot, -gsc, Kseqcopy[i], 1, Kseqcopy[j], 1,
                             Kseqcopy[j], 1);
            }
            R[j * kdimp + i] = gsc;
        }
    }
 
    // Compute H matrix
    for (int i = 0; i < kdim; ++i)
    {
        for (int j = 0; j < kdim; ++j)
        {
            H[j * kdim + i] =
                alpha[j + 1] * R[(j + 1) * kdimp + i] -
                Vmath::Dot(j, &H[0] + i, kdim, &R[0] + j * kdimp, 1);
            H[j * kdim + i] /= R[j * kdimp + j];
        }
    }
 
    H[(kdim - 1) * kdim + kdim] =
        alpha[kdim] *
        std::fabs(R[kdim * kdimp + kdim] / R[(kdim - 1) * kdimp + kdim - 1]);
 
    Lapack::dgeev_('N', 'V', kdim, &H[0], kdim, &wr[0], &wi[0], 0, 1, &zvec[0],
                   kdim, &rwork[0], lwork, ier);
 
    ASSERTL0(!ier, "Error with dgeev");
 
    resnorm = H[(kdim - 1) * kdim + kdim];
}
 
/**
 * Check for convergence of the residuals of the eigenvalues of H.
 */
int DriverModifiedArnoldi::EV_test(const int itrn, const int kdim,
                                   Array<OneD, NekDouble> &zvec,
                                   Array<OneD, NekDouble> &wr,
                                   Array<OneD, NekDouble> &wi,
                                   const NekDouble resnorm, int nvec,
                                   ofstream &evlout, NekDouble &resid0)
{
    int idone = 0;
    // NekDouble period = 0.1;
 
    Array<OneD, NekDouble> resid(kdim);
    for (int i = 0; i < kdim; ++i)
    {
        NekDouble tmp = std::sqrt(
            Vmath::Dot(kdim, &zvec[0] + i * kdim, 1, &zvec[0] + i * kdim, 1));
        resid[i] = resnorm * std::fabs(zvec[kdim - 1 + i * kdim]) / tmp;
        if (wi[i] < 0.0)
        {
            resid[i - 1] = resid[i] = hypot(resid[i - 1], resid[i]);
        }
    }
 
    EV_sort(zvec, wr, wi, resid, kdim);
 
    while (nvec <= kdim && resid[nvec - 1] < m_evtol)
    {
        idone = nvec;
        ++nvec;
    }
    nvec -= (idone > 0);
 
    if (m_comm->GetRank() == 0)
    {
        evlout << "-- Iteration = " << itrn << ", H(k+1, k) = " << resnorm
               << endl;
        evlout.precision(4);
        evlout.setf(ios::scientific, ios::floatfield);
        if (m_timeSteppingAlgorithm)
        {
            evlout << "        Magnitude   Angle       Growth      "
                   << "Frequency   Residual" << endl;
        }
        else
        {
            evlout << "        Real        Imaginary   inverse real  "
                   << "inverse imag  Residual" << endl;
        }
 
        for (int i = 0; i < kdim; i++)
        {
            WriteEvs(evlout, i, wr[i], wi[i], resid[i]);
        }
    }
 
    resid0 = resid[nvec - 1];
    return idone;
}
 
/**
 * Sorts the computed eigenvalues by smallest residual first.
 */
void DriverModifiedArnoldi::EV_sort(Array<OneD, NekDouble> &evec,
                                    Array<OneD, NekDouble> &wr,
                                    Array<OneD, NekDouble> &wi,
                                    Array<OneD, NekDouble> &test, const int dim)
{
    Array<OneD, NekDouble> z_tmp(dim, 0.0);
    NekDouble wr_tmp, wi_tmp, te_tmp;
    for (int j = 1; j < dim; ++j)
    {
        wr_tmp = wr[j];
        wi_tmp = wi[j];
        te_tmp = test[j];
        Vmath::Vcopy(dim, &evec[0] + j * dim, 1, &z_tmp[0], 1);
        int i = j - 1;
        while (i >= 0 && test[i] > te_tmp)
        {
            wr[i + 1]   = wr[i];
            wi[i + 1]   = wi[i];
            test[i + 1] = test[i];
            Vmath::Vcopy(dim, &evec[0] + i * dim, 1, &evec[0] + (i + 1) * dim,
                         1);
            i--;
        }
        wr[i + 1]   = wr_tmp;
        wi[i + 1]   = wi_tmp;
        test[i + 1] = te_tmp;
        Vmath::Vcopy(dim, &z_tmp[0], 1, &evec[0] + (i + 1) * dim, 1);
    }
}
 
/**
 * Post-process the Ritz eigenvalues/eigenvectors of the matrix H, to compute
 * estimations of the leading eigenvalues and eigenvectors of the original
 * matrix.
 */
void DriverModifiedArnoldi::EV_post(Array<OneD, Array<OneD, NekDouble>> &Tseq,
                                    Array<OneD, Array<OneD, NekDouble>> &Kseq,
                                    const int ntot, const int kdim,
                                    const int nvec,
                                    Array<OneD, NekDouble> &zvec,
                                    Array<OneD, NekDouble> &wr,
                                    Array<OneD, NekDouble> &wi, const int icon)
{
    if (icon == 0)
    {
        // Not converged, write final Krylov vector
        ASSERTL0(false, "Convergence was not achieved within the "
                        "prescribed number of iterations.");
    }
    else if (icon < 0)
    {
        // Minimum residual reached
        ASSERTL0(false, "Minimum residual reached.");
    }
    else if (icon == nvec)
    {
        // Converged, write out eigenvectors
        EV_big(Tseq, Kseq, ntot, kdim, icon, zvec, wr, wi);
        Array<OneD, MultiRegions::ExpListSharedPtr> fields =
            m_equ[0]->UpdateFields();
 
        for (int j = 0; j < icon; ++j)
        {
            std::string file = m_session->GetSessionName() + "_eig_" +
                               boost::lexical_cast<std::string>(j) + ".fld";
 
            if (m_comm->GetRank() == 0)
            {
                WriteEvs(cout, j, wr[j], wi[j]);
            }
            WriteFld(file, Kseq[j]);
            if (m_useMask)
            {
                std::string fileunmask =
                    m_session->GetSessionName() + "_eig_masked_" +
                    boost::lexical_cast<std::string>(j) + ".fld";
                WriteFld(fileunmask, Tseq[j]);
            }
        }
    }
    else
    {
        // Not recognised
        ASSERTL0(false, "Unrecognised value.");
    }
}
 
/**
 * Compute estimates of the eigenvalues/eigenvectors of the original system.
 */
void DriverModifiedArnoldi::EV_big(Array<OneD, Array<OneD, NekDouble>> &bvecs,
                                   Array<OneD, Array<OneD, NekDouble>> &evecs,
                                   const int ntot, const int kdim,
                                   const int nvec, Array<OneD, NekDouble> &zvec,
                                   Array<OneD, NekDouble> &wr,
                                   Array<OneD, NekDouble> &wi)
{
    boost::ignore_unused(wr);
 
    NekDouble wgt, norm;
    Array<OneD, Array<OneD, NekDouble>> btmp(nvec);
    Array<OneD, Array<OneD, NekDouble>> etmp(nvec);
    for (int i = 0; i < nvec; ++i)
    {
        if (m_useMask)
        {
            btmp[i] = Array<OneD, NekDouble>(ntot, 0.);
            etmp[i] = Array<OneD, NekDouble>(ntot, 0.);
        }
        else
        {
            btmp[i] = evecs[i];
        }
    }
 
    // Generate big eigenvectors
    for (int j = 0; j < nvec; ++j)
    {
        Vmath::Zero(ntot, btmp[j], 1);
        if (m_useMask)
        {
            Vmath::Zero(ntot, etmp[j], 1);
        }
        for (int i = 0; i < kdim; ++i)
        {
            wgt = zvec[i + j * kdim];
            Vmath::Svtvp(ntot, wgt, bvecs[i], 1, btmp[j], 1, btmp[j], 1);
            if (m_useMask)
            {
                Vmath::Svtvp(ntot, wgt, evecs[i], 1, etmp[j], 1, etmp[j], 1);
            }
        }
    }
 
    // Normalise the big eigenvectors
    for (int i = 0; i < nvec; ++i)
    {
        if (wi[i] == 0.0) // Real mode
        {
            if (m_session->GetComm()->GetRank() == 0)
            {
                cout << "eigenvalue " << i << ": real mode" << endl;
            }
            norm = Blas::Ddot(ntot, &btmp[i][0], 1, &btmp[i][0], 1);
            m_comm->AllReduce(norm, Nektar::LibUtilities::ReduceSum);
            norm = std::sqrt(norm);
            if (m_useMask)
            {
                Vmath::Smul(ntot, 1.0 / norm, btmp[i], 1, bvecs[i], 1);
                Vmath::Smul(ntot, 1.0 / norm, etmp[i], 1, evecs[i], 1);
            }
            else
            {
                Vmath::Smul(ntot, 1.0 / norm, btmp[i], 1, evecs[i], 1);
            }
        }
        else
        {
            if (m_session->GetComm()->GetRank() == 0)
            {
                cout << "eigenvalues " << i << ", " << i + 1
                     << ": complex modes" << endl;
            }
            norm = Blas::Ddot(ntot, &btmp[i][0], 1, &btmp[i][0], 1);
            norm += Blas::Ddot(ntot, &btmp[i + 1][0], 1, &btmp[i + 1][0], 1);
            m_comm->AllReduce(norm, Nektar::LibUtilities::ReduceSum);
            norm = std::sqrt(norm);
 
            if (m_useMask)
            {
                Vmath::Smul(ntot, 1.0 / norm, btmp[i], 1, bvecs[i], 1);
                Vmath::Smul(ntot, 1.0 / norm, btmp[i + 1], 1, bvecs[i + 1], 1);
                Vmath::Smul(ntot, 1.0 / norm, etmp[i], 1, evecs[i], 1);
                Vmath::Smul(ntot, 1.0 / norm, etmp[i + 1], 1, evecs[i + 1], 1);
            }
            else
            {
                Vmath::Smul(ntot, 1.0 / norm, btmp[i], 1, evecs[i], 1);
                Vmath::Smul(ntot, 1.0 / norm, btmp[i + 1], 1, evecs[i + 1], 1);
            }
 
            i++;
        }
    }
}
 
} // namespace SolverUtils
} // namespace Nektar