GlobalLinSysPETSc.h

Go to the documentation of this file.

///////////////////////////////////////////////////////////////////////////////
//
// File: GlobalLinSysPETSc.h
//
// For more information, please see: http://www.nektar.info
//
// The MIT License
//
// Copyright (c) 2006 Division of Applied Mathematics, Brown University (USA),
// Department of Aeronautics, Imperial College London (UK), and Scientific
// Computing and Imaging Institute, University of Utah (USA).
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//
// Description: GlobalLinSysPETSc header
//
///////////////////////////////////////////////////////////////////////////////
#ifndef NEKTAR_LIB_MULTIREGIONS_GLOBALLINSYSPETSC_H
#define NEKTAR_LIB_MULTIREGIONS_GLOBALLINSYSPETSC_H
 
#include <MultiRegions/GlobalLinSys.h>
#include <MultiRegions/MultiRegionsDeclspec.h>
 
#include <petscksp.h>
#include <petscmat.h>
 
namespace Nektar::MultiRegions
{
// Forward declarations
class ExpList;
 
/// Enumerator
enum PETScMatMult
{
    ePETScMatMultSparse,
    ePETScMatMultShell
};
 
/// A PETSc global linear system.
class GlobalLinSysPETSc : virtual public GlobalLinSys
{
public:
    /// Constructor for full direct matrix solve.
    MULTI_REGIONS_EXPORT GlobalLinSysPETSc(
        const GlobalLinSysKey &pKey, const std::weak_ptr<ExpList> &pExp,
        const std::shared_ptr<AssemblyMap> &pLocToGloMap);
 
    MULTI_REGIONS_EXPORT ~GlobalLinSysPETSc() override;
 
protected:
    /// PETSc matrix object.
    Mat m_matrix;
    /// PETSc vector objects used for local storage.
    Vec m_x, m_b, m_locVec;
    /// KSP object that represents solver system.
    KSP m_ksp;
    /// PCShell for preconditioner.
    PC m_pc;
    /// Enumerator to select matrix multiplication type.
    PETScMatMult m_matMult;
    /// Reordering that takes universal IDs to a unique row in the PETSc
    /// matrix. @see GlobalLinSysPETSc::CalculateReordering
    std::vector<int> m_reorderedMap;
    /// PETSc scatter context that takes us between Nektar++ global
    /// ordering and PETSc vector ordering.
    VecScatter m_ctx;
    /// Number of unique degrees of freedom on this process.
    int m_nLocal;
 
    PreconditionerSharedPtr m_precon;
 
    /**
     * @brief Internal struct for MatShell and PCShell calls to store
     * current context for callback.
     *
     * To use the MatShell/PCShell representation inside PETSc KSP and
     * PC objects (so that we can use the local spectral element
     * approach) requires the use of a callback function, which must be
     * static. This is a lightweight wrapper allowing us to call a
     * virtual function so that we can handle the static
     * condensation/full variants of the global system.
     *
     * @see GlobalLinSysPETSc::DoMatrixMultiply
     */
    struct ShellCtx
    {
        /// Number of global degrees of freedom.
        int nGlobal;
        /// Number of Dirichlet degrees of freedom.
        int nDir;
        /// Pointer to the original calling object.
        GlobalLinSysPETSc *linSys;
    };
 
    void SetUpScatter();
    void SetUpMatVec(int nGlobal, int nDir);
    void SetUpSolver(NekDouble tolerance);
    void CalculateReordering(const Array<OneD, const int> &glo2uniMap,
                             const Array<OneD, const int> &glo2unique,
                             const AssemblyMapSharedPtr &pLocToGloMap);
 
    void v_SolveLinearSystem(const int pNumRows,
                             const Array<OneD, const NekDouble> &pInput,
                             Array<OneD, NekDouble> &pOutput,
                             const AssemblyMapSharedPtr &locToGloMap,
                             const int pNumDir) override;
 
    virtual void v_DoMatrixMultiply(const Array<OneD, const NekDouble> &pInput,
                                    Array<OneD, NekDouble> &pOutput) = 0;
 
private:
    static std::string matMult;
    static std::string matMultIds[];
 
    static PetscErrorCode DoMatrixMultiply(Mat M, Vec in, Vec out);
    static PetscErrorCode DoPreconditioner(PC pc, Vec in, Vec out);
    static void DoNekppOperation(Vec &in, Vec &out, ShellCtx *ctx, bool precon);
    static PetscErrorCode DoDestroyMatCtx(Mat M);
    static PetscErrorCode DoDestroyPCCtx(PC pc);
};
} // namespace Nektar::MultiRegions
 
#endif