MPIController Derived Type

type, public, extends(Object) :: MPIController

Provides a centralized interface with MPI and various support routines compatible with ReMKiT1D needs


Components

Type Visibility Attributes Name Initial
logical, public :: userDefined = .false.

True only if user explicitly sets it to true
type(MPI_Comm), private :: worldComm

Main MPI_Comm_World
type(MPI_Comm), private :: rowComm

Communicator for processors rows (those with the same x-domain but different h-domains)
type(MPI_Comm), private :: colComm

Communicator for processor columns (those with the same h-domain but different x-domains)
integer(kind=ik), private :: worldRank

Current process' world rank
integer(kind=ik), private :: worldSize

Size of worldComm
integer(kind=ik), private :: rowRank

Current process' row rank
integer(kind=ik), private :: rowSize

Size of rowComm
integer(kind=ik), private :: colRank

Current process' column rank
integer(kind=ik), private :: colSize

Size of colComm
integer(kind=ik), private :: rowNumX

Number of x-grid points in local processor row
integer(kind=ik), private, allocatable, dimension(:) :: colNumX

Number of x-grid points in each column processor
integer(kind=ik), private, allocatable, dimension(:) :: xDispls

Displacement in x-direction used for MPI gather
integer(kind=ik), private :: xHaloWidth

Halo width in x-direction

integer(kind=ik), private :: numV

Number of velocity cells in grid
integer(kind=ik), private :: numH

Number of harmonics in grid
integer(kind=ik), public, allocatable, dimension(:) :: rowNumH

locNumH for each processor in row
integer(kind=ik), public, allocatable, dimension(:) :: rowHOffset

harmonic offset for each processor in row
type(RealArray), public, allocatable, dimension(:) :: distBuffer

Distribution variable buffer used in halo exchange
type(IntArray), public, allocatable, dimension(:) :: neighbourPairs

Int array for each x-grid neighbour pair with entry of length 2 first element is left neighbour rank, second is right - allows for periodic domain

logical, public :: rowsSetUp

True if row data is set up and ready for communication

Type-Bound Procedures

procedure, public :: isDefined => isDefinedObject

  • interface

    private pure module function isDefinedObject(this) result(defined)

    Getter for userDefined

    Arguments

    Type IntentOptional Attributes Name
    class(Object), intent(in) :: this

    Return Value logical

procedure, public :: makeDefined => makeDefinedObject

  • interface

    private pure module subroutine makeDefinedObject(this)

    Set userDefined to .true.

    Arguments

    Type IntentOptional Attributes Name
    class(Object), intent(inout) :: this

procedure, public :: makeUndefined => makeUndefinedObject

  • interface

    private pure module subroutine makeUndefinedObject(this)

    Set userDefined to .false.

    Arguments

    Type IntentOptional Attributes Name
    class(Object), intent(inout) :: this

procedure, public :: getWorldRank

  • interface

    private pure module function getWorldRank(this) result(rank)

    Getter for worldRank

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(in) :: this

    Return Value integer(kind=ik)

procedure, public :: getWorldSize

  • interface

    private pure module function getWorldSize(this) result(size)

    Getter for worldSize

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(in) :: this

    Return Value integer(kind=ik)

procedure, public :: getRowRank

  • interface

    private pure module function getRowRank(this) result(rank)

    Getter for rowRank

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(in) :: this

    Return Value integer(kind=ik)

procedure, public :: getRowSize

  • interface

    private pure module function getRowSize(this) result(size)

    Getter for rowSize

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(in) :: this

    Return Value integer(kind=ik)

procedure, public :: getColRank

  • interface

    private pure module function getColRank(this) result(rank)

    Getter for columnRank

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(in) :: this

    Return Value integer(kind=ik)

procedure, public :: getColSize

  • interface

    private pure module function getColSize(this) result(size)

    Getter for columnSize

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(in) :: this

    Return Value integer(kind=ik)

procedure, public :: getXHaloWidth

  • interface

    private pure module function getXHaloWidth(this) result(xHaloWidth)

    Getter for xHaloWidth

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(in) :: this

    Return Value integer(kind=ik)

procedure, public :: calculateRowDistData

  • interface

    private module subroutine calculateRowDistData(this, partitionObj, xHaloWidth, numV)

    Initialize distribution DoF data used in exchanging distribution data in a row

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(inout) :: this
    type(Partition), intent(in) :: partitionObj

    Partition object used to retrieve DoF information
    integer(kind=ik), intent(in) :: xHaloWidth

    Halo width in x-direction
    integer(kind=ik), intent(in) :: numV

    Number of cells in v-grid

procedure, public :: initializeNeighbourPairs

  • interface

    private module subroutine initializeNeighbourPairs(this, periodic)

    Initialize neighbour pairs - if periodic in X adds additional pair to handle this

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(inout) :: this
    logical, intent(in) :: periodic

procedure, public :: broadcastVarInRow

  • interface

    private module subroutine broadcastVarInRow(this, varContainer, name)

    Broadcast variable with given name from row rank 0 process to other row processes - to be used only with non-distribution variables

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(inout) :: this
    type(VariableContainer), intent(inout) :: varContainer

    Variable container in which to broadcasty
    character(len=*), intent(in) :: name

    Name of variable to broadcast

procedure, public :: exchangeDistVarInRow

  • interface

    private module subroutine exchangeDistVarInRow(this, varContainer, name)

    Exchanges a distribution variable in a processor row

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(inout) :: this
    type(VariableContainer), intent(inout) :: varContainer

    Variable container in which to broadcast
    character(len=*), intent(in) :: name

    Name of variable to broadcast

procedure, public :: exchangeVarXHalos

  • interface

    private module subroutine exchangeVarXHalos(this, varContainer, name, varIsDist)

    Exchanges a halos in x direction for given variable

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(inout) :: this
    type(VariableContainer), intent(inout) :: varContainer

    Variable container in which to perform halo exchange
    character(len=*), intent(in) :: name

    Name of variable to exchange
    logical, intent(in) :: varIsDist

    Set to true if variable is a distribution

procedure, public :: broadcastReal

  • interface

    private module subroutine broadcastReal(this, vec, rootProc)

    Broadcast real vector from rank rootProc

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(inout) :: this
    real(kind=rk), intent(inout), dimension(:) :: vec
    integer(kind=ik), intent(in), optional :: rootProc

procedure, public :: broadcastInt

  • interface

    private module subroutine broadcastInt(this, vec)

    Broadcast integer vector from rank 0

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(inout) :: this
    integer(kind=ik), intent(inout), dimension(:) :: vec

procedure, public :: broadcastLogical

  • interface

    private module subroutine broadcastLogical(this, vec)

    Broadcast logical vector from rank 0

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(inout) :: this
    logical, intent(inout), dimension(:) :: vec

procedure, public :: broadcastCharacter

  • interface

    private module subroutine broadcastCharacter(this, vec)

    Broadcast character vector from rank 0

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(inout) :: this
    character(len=:), intent(inout), allocatable :: vec

procedure, public :: barrier

  • interface

    private module subroutine barrier(this)

    Calls MPI barrier on commWorld

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(inout) :: this

procedure, public :: gatherVar

  • interface

    private module subroutine gatherVar(this, localVec, globVec, varIsDist)

    Gather variable values into global vector for output on rank 0. If globVec in not allocated it will be allocated to the correct size, otherwise it is assumed it is of the correct size.

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(inout) :: this
    real(kind=rk), intent(in), dimension(:) :: localVec

    Local vector to gather
    real(kind=rk), intent(inout), allocatable, dimension(:) :: globVec

    Global vector on rank 0 to gather into
    logical, intent(in) :: varIsDist

    Set to true if the gathered variable is a distribution

procedure, public :: scatterVar

  • interface

    private module subroutine scatterVar(this, globVec, localVec, varIsDist)

    Scatter global variable values into local vector from rank 0.

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(inout) :: this
    real(kind=rk), intent(inout), dimension(:) :: globVec

    Global vector on rank 0 to scatter from
    real(kind=rk), intent(inout), allocatable, dimension(:) :: localVec

    Local vector to scatter into
    logical, intent(in) :: varIsDist

    Set to true if the scattered variable is a distribution

procedure, public :: setUpRows

  • interface

    private module subroutine setUpRows(this, numProcsX, numProcsH)

    Set up row/column communication in controller

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(inout) :: this
    integer(kind=ik), intent(in) :: numProcsX

    Number of processes in the x-direction
    integer(kind=ik), intent(in) :: numProcsH

    Number of processes in the h-direction

procedure, public :: isTrueEverywhere

  • interface

    private module function isTrueEverywhere(this, input) result(isTrue)

    Return true if input is true on every processor

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(inout) :: this
    logical, intent(inout) :: input

    Return Value logical

procedure, public :: allreduceMax

  • interface

    private module function allreduceMax(this, input) result(max)

    Return max value on all processors

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(inout) :: this
    real(kind=rk), intent(inout) :: input

    Return Value real(kind=rk)

procedure, public :: allreduceMin

  • interface

    private module function allreduceMin(this, input) result(min)

    Return min value on all processors

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(inout) :: this
    real(kind=rk), intent(inout) :: input

    Return Value real(kind=rk)

procedure, public :: init => initMPIController

  • interface

    private module subroutine initMPIController(this, numProcsX, numProcsH)

    MPI controller initialization routine - initializes communicators used outside of PETSc. Optionally sets up row/column communicators.

    Arguments

    Type IntentOptional Attributes Name
    class(MPIController), intent(inout) :: this
    integer(kind=ik), intent(in), optional :: numProcsX

    Number of processes in the x-direction
    integer(kind=ik), intent(in), optional :: numProcsH

    Number of processes in the h-direction